@particle-academy/prism-graph 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +140 -2
- package/dist/chunk.d.ts +29 -0
- package/dist/chunk.js +5 -0
- package/dist/chunking/markdown.d.ts +10 -0
- package/dist/chunking/markdown.js +64 -0
- package/dist/chunking/window.d.ts +23 -0
- package/dist/chunking/window.js +128 -0
- package/dist/digest.d.ts +8 -0
- package/dist/digest.js +25 -0
- package/dist/errors.d.ts +10 -0
- package/dist/errors.js +14 -0
- package/dist/fact.d.ts +44 -0
- package/dist/fact.js +1 -0
- package/dist/graph.d.ts +106 -0
- package/dist/graph.js +266 -0
- package/dist/index-spec.d.ts +14 -0
- package/dist/index-spec.js +31 -0
- package/dist/index.d.ts +18 -0
- package/dist/index.js +16 -0
- package/dist/scope.d.ts +28 -0
- package/dist/scope.js +70 -0
- package/dist/storage/driver.d.ts +49 -0
- package/dist/storage/driver.js +100 -0
- package/dist/storage/schema.d.ts +31 -0
- package/dist/storage/schema.js +102 -0
- package/dist/storage/store.d.ts +129 -0
- package/dist/storage/store.js +391 -0
- package/dist/tools.d.ts +28 -0
- package/dist/tools.js +166 -0
- package/dist/trust.d.ts +56 -0
- package/dist/trust.js +73 -0
- package/dist/vectors.d.ts +55 -0
- package/dist/vectors.js +155 -0
- package/dist/visible.d.ts +6 -0
- package/dist/visible.js +34 -0
- package/dist/wake.d.ts +66 -0
- package/dist/wake.js +126 -0
- package/package.json +34 -4
package/dist/graph.js
ADDED
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
import { digest, sha256 } from './digest.js';
|
|
2
|
+
import { GraphError, InvalidArgumentError } from './errors.js';
|
|
3
|
+
import { IndexSpec } from './index-spec.js';
|
|
4
|
+
import { DefaultAcceptancePolicy } from './trust.js';
|
|
5
|
+
/**
|
|
6
|
+
* Long-term memory for agents. This object only holds the wiring (store,
|
|
7
|
+
* optional vectors and embedder, acceptance policy); it has NO read or write
|
|
8
|
+
* methods of its own. Everything goes through a scope, so an unscoped query
|
|
9
|
+
* cannot be written, by an app or by a model.
|
|
10
|
+
*/
|
|
11
|
+
export class Graph {
|
|
12
|
+
store;
|
|
13
|
+
vectors;
|
|
14
|
+
embedder;
|
|
15
|
+
policy;
|
|
16
|
+
listMax;
|
|
17
|
+
constructor(store, vectors = null, embedder = null, policy = new DefaultAcceptancePolicy(),
|
|
18
|
+
/** The largest page listSources/listChunks will return (G-282). */
|
|
19
|
+
listMax = ScopedGraph.DEFAULT_LIST_MAX) {
|
|
20
|
+
this.store = store;
|
|
21
|
+
this.vectors = vectors;
|
|
22
|
+
this.embedder = embedder;
|
|
23
|
+
this.policy = policy;
|
|
24
|
+
this.listMax = listMax;
|
|
25
|
+
if (!Number.isInteger(listMax) || listMax < 1) {
|
|
26
|
+
throw new InvalidArgumentError('listMax must be a positive integer.');
|
|
27
|
+
}
|
|
28
|
+
if ((vectors === null) !== (embedder === null)) {
|
|
29
|
+
throw new InvalidArgumentError('Configure a VectorStore and an Embedder together, or neither.');
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
scope(scope) {
|
|
33
|
+
return new ScopedGraph(scope, this.store, this.vectors, this.embedder, this.policy, this.listMax);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Memory as seen from one scope. Reads return what this scope can see (its own
|
|
38
|
+
* rows and those shared from wider scopes above it); writes land at exactly
|
|
39
|
+
* this scope. Nothing here widens a write.
|
|
40
|
+
*/
|
|
41
|
+
export class ScopedGraph {
|
|
42
|
+
memoryScope;
|
|
43
|
+
store;
|
|
44
|
+
vectors;
|
|
45
|
+
embedder;
|
|
46
|
+
policy;
|
|
47
|
+
listMax;
|
|
48
|
+
static MAX_LIMIT = 200;
|
|
49
|
+
/** A query goes to the embedding provider and the full-text index, so it is bounded. */
|
|
50
|
+
static MAX_QUERY_BYTES = 4096;
|
|
51
|
+
static MAX_GET_IDS = 1000;
|
|
52
|
+
static RRF_K = 60;
|
|
53
|
+
/** The listing contract (G-282). A consumer refuses a version it does not know. */
|
|
54
|
+
static CONTRACT_VERSION = 'g282.1';
|
|
55
|
+
static DEFAULT_LIST_LIMIT = 50;
|
|
56
|
+
static DEFAULT_LIST_MAX = 200;
|
|
57
|
+
constructor(memoryScope, store, vectors, embedder, policy, listMax = ScopedGraph.DEFAULT_LIST_MAX) {
|
|
58
|
+
this.memoryScope = memoryScope;
|
|
59
|
+
this.store = store;
|
|
60
|
+
this.vectors = vectors;
|
|
61
|
+
this.embedder = embedder;
|
|
62
|
+
this.policy = policy;
|
|
63
|
+
this.listMax = listMax;
|
|
64
|
+
}
|
|
65
|
+
/** Sources this scope can see, newest first, a page at a time (G-282). */
|
|
66
|
+
async listSources(cursor = null, limit = null) {
|
|
67
|
+
const size = this.listLimit(limit);
|
|
68
|
+
let after = null;
|
|
69
|
+
if (cursor !== null) {
|
|
70
|
+
const key = this.readCursor(cursor, 'sources');
|
|
71
|
+
if (key.length !== 2 || typeof key[0] !== 'string' || typeof key[1] !== 'string') {
|
|
72
|
+
throw new InvalidArgumentError('Invalid cursor.');
|
|
73
|
+
}
|
|
74
|
+
after = [key[0], key[1]];
|
|
75
|
+
}
|
|
76
|
+
const rows = await this.store.listSources(this.memoryScope, after, size + 1);
|
|
77
|
+
const items = rows.slice(0, size);
|
|
78
|
+
const last = items.at(-1);
|
|
79
|
+
return {
|
|
80
|
+
contractVersion: ScopedGraph.CONTRACT_VERSION,
|
|
81
|
+
items,
|
|
82
|
+
nextCursor: rows.length > size && last !== undefined ? this.writeCursor('sources', [last.createdAt, last.sourceId]) : null,
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Live chunks of one source, in source order, a page at a time (G-282). A source outside this scope yields an
|
|
87
|
+
* empty page, never an error that would confirm it exists.
|
|
88
|
+
*/
|
|
89
|
+
async listChunks(sourceId, cursor = null, limit = null) {
|
|
90
|
+
const size = this.listLimit(limit);
|
|
91
|
+
let afterSeq = null;
|
|
92
|
+
if (cursor !== null) {
|
|
93
|
+
const key = this.readCursor(cursor, `chunks:${sourceId}`);
|
|
94
|
+
if (key.length !== 1 || typeof key[0] !== 'number' || !Number.isInteger(key[0]) || key[0] < 0) {
|
|
95
|
+
throw new InvalidArgumentError('Invalid cursor.');
|
|
96
|
+
}
|
|
97
|
+
afterSeq = key[0];
|
|
98
|
+
}
|
|
99
|
+
const rows = await this.store.listChunks(this.memoryScope, sourceId, afterSeq, size + 1);
|
|
100
|
+
const page = rows.slice(0, size);
|
|
101
|
+
const last = page.at(-1);
|
|
102
|
+
return {
|
|
103
|
+
contractVersion: ScopedGraph.CONTRACT_VERSION,
|
|
104
|
+
// seq is the store's paging key; the contract's order is `position`.
|
|
105
|
+
items: page.map(({ seq: _seq, ...chunk }) => chunk),
|
|
106
|
+
nextCursor: rows.length > size && last !== undefined ? this.writeCursor(`chunks:${sourceId}`, [last.seq]) : null,
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
listLimit(limit) {
|
|
110
|
+
const size = limit ?? Math.min(ScopedGraph.DEFAULT_LIST_LIMIT, this.listMax);
|
|
111
|
+
if (!Number.isInteger(size) || size < 1 || size > this.listMax) {
|
|
112
|
+
throw new InvalidArgumentError(`limit must be between 1 and ${this.listMax}.`);
|
|
113
|
+
}
|
|
114
|
+
return size;
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Opaque, and bound to this scope and listing: a cursor from another scope or listing is refused. It needs no
|
|
118
|
+
* secret, because every listing query applies the scope predicate regardless of the cursor.
|
|
119
|
+
*/
|
|
120
|
+
writeCursor(listing, key) {
|
|
121
|
+
return Buffer.from(JSON.stringify({ k: key, b: this.cursorBinding(listing) }), 'utf8').toString('base64url');
|
|
122
|
+
}
|
|
123
|
+
readCursor(cursor, listing) {
|
|
124
|
+
let data = null;
|
|
125
|
+
if (cursor.length <= 1024 && /^[A-Za-z0-9_-]*$/.test(cursor)) {
|
|
126
|
+
try {
|
|
127
|
+
data = JSON.parse(Buffer.from(cursor, 'base64url').toString('utf8'));
|
|
128
|
+
}
|
|
129
|
+
catch {
|
|
130
|
+
data = null;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
if (typeof data !== 'object' || data === null || !Array.isArray(data.k)
|
|
134
|
+
|| data.b !== this.cursorBinding(listing)) {
|
|
135
|
+
throw new InvalidArgumentError('Invalid cursor.');
|
|
136
|
+
}
|
|
137
|
+
return data.k;
|
|
138
|
+
}
|
|
139
|
+
cursorBinding(listing) {
|
|
140
|
+
return digest('prism-graph/cursor@1', [...this.memoryScope.toRow().map((v) => v ?? ''), listing]).slice(0, 16);
|
|
141
|
+
}
|
|
142
|
+
scope() {
|
|
143
|
+
return this.memoryScope;
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* Store a source and its verbatim chunks at this scope. Idempotent per key:
|
|
147
|
+
* unchanged text stores and embeds nothing new; text the source no longer
|
|
148
|
+
* contains is retired from search. Verbatim is stored first and never waits
|
|
149
|
+
* on embedding; a failed embedding heals on the next ingest.
|
|
150
|
+
*/
|
|
151
|
+
async ingest(key, uri, text, chunker, origin, options = {}) {
|
|
152
|
+
const actual = sha256(Buffer.from(text, 'utf8'));
|
|
153
|
+
if (options.sha256 !== undefined && options.sha256.toLowerCase() !== actual) {
|
|
154
|
+
throw new GraphError(`The provided sha256 for [${key}] does not match its text.`);
|
|
155
|
+
}
|
|
156
|
+
const chunks = chunker.chunk(this.memoryScope.qualify('source', key), text);
|
|
157
|
+
const { stored, retired } = await this.store.storeSource(this.memoryScope, key, uri, actual, options.metadata ?? {}, origin, options.occurredAt ?? null, chunks);
|
|
158
|
+
if (this.vectors !== null && this.embedder !== null && chunks.length > 0) {
|
|
159
|
+
const spec = this.spec();
|
|
160
|
+
await this.vectors.ensureIndex(spec);
|
|
161
|
+
const byId = new Map(chunks.map((c) => [c.id, c]));
|
|
162
|
+
const ids = [...byId.keys()];
|
|
163
|
+
for (let i = 0; i < ids.length; i += 500) {
|
|
164
|
+
const todo = await this.vectors.missing(spec, ids.slice(i, i + 500));
|
|
165
|
+
if (todo.length === 0) {
|
|
166
|
+
continue;
|
|
167
|
+
}
|
|
168
|
+
const vectors = await this.embedder.embed(todo.map((id) => byId.get(id).text));
|
|
169
|
+
if (vectors.length !== todo.length) {
|
|
170
|
+
throw new GraphError('The embedder returned a different number of vectors than chunks.');
|
|
171
|
+
}
|
|
172
|
+
await this.vectors.upsert(spec, new Map(todo.map((id, n) => [id, vectors[n]])));
|
|
173
|
+
// An erase that ran while the provider was embedding has already removed
|
|
174
|
+
// these chunks; erased text must not live on as an embedding.
|
|
175
|
+
const present = new Set((await this.store.getChunks(this.memoryScope, todo)).map((c) => c.id));
|
|
176
|
+
const gone = todo.filter((id) => !present.has(id));
|
|
177
|
+
if (gone.length > 0) {
|
|
178
|
+
await this.vectors.forget(gone);
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
return { source: key, sha256: actual, recipe: chunker.recipe(), chunks: chunks.length, stored, retired };
|
|
183
|
+
}
|
|
184
|
+
/** Take a source at this scope out of search; it stays readable by id. */
|
|
185
|
+
async retireSource(key) {
|
|
186
|
+
return this.store.retireSource(this.memoryScope, key);
|
|
187
|
+
}
|
|
188
|
+
/** Erase a source stored at this scope: chunks, vectors, evidence links and orphaned facts. */
|
|
189
|
+
async erase(key) {
|
|
190
|
+
const report = await this.store.eraseSource(this.memoryScope, key);
|
|
191
|
+
if (this.vectors === null || report.chunkIds.length === 0) {
|
|
192
|
+
return report;
|
|
193
|
+
}
|
|
194
|
+
return { ...report, vectors: await this.vectors.forget(report.chunkIds) };
|
|
195
|
+
}
|
|
196
|
+
async get(chunkIds) {
|
|
197
|
+
if (chunkIds.length > ScopedGraph.MAX_GET_IDS) {
|
|
198
|
+
throw new InvalidArgumentError(`Pass at most ${ScopedGraph.MAX_GET_IDS} chunk ids.`);
|
|
199
|
+
}
|
|
200
|
+
return this.store.getChunks(this.memoryScope, chunkIds);
|
|
201
|
+
}
|
|
202
|
+
/**
|
|
203
|
+
* Hybrid search over what this scope can see: full-text (BM25) and, when
|
|
204
|
+
* configured and `vectors` is true, vectors, fused by reciprocal rank. A
|
|
205
|
+
* chunk with no vector yet still ranks through text.
|
|
206
|
+
*/
|
|
207
|
+
async search(query, limit = 10, vectors = true) {
|
|
208
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > ScopedGraph.MAX_LIMIT) {
|
|
209
|
+
throw new InvalidArgumentError(`limit must be between 1 and ${ScopedGraph.MAX_LIMIT}.`);
|
|
210
|
+
}
|
|
211
|
+
if (Buffer.byteLength(query, 'utf8') > ScopedGraph.MAX_QUERY_BYTES) {
|
|
212
|
+
throw new InvalidArgumentError(`A query may be at most ${ScopedGraph.MAX_QUERY_BYTES} bytes.`);
|
|
213
|
+
}
|
|
214
|
+
if (!query.isWellFormed()) {
|
|
215
|
+
throw new InvalidArgumentError('A query must be valid UTF-8.');
|
|
216
|
+
}
|
|
217
|
+
const lists = new Map([['text', (await this.store.textSearch(this.memoryScope, query, limit * 3)).map((r) => r.chunkId)]]);
|
|
218
|
+
if (vectors && this.vectors !== null && this.embedder !== null && query.trim() !== '') {
|
|
219
|
+
const embedded = (await this.embedder.embed([query]))[0];
|
|
220
|
+
lists.set(this.vectors.method(), (await this.vectors.nearest(this.spec(), embedded, Math.min(1000, limit * 10))).map((r) => r.chunkId));
|
|
221
|
+
}
|
|
222
|
+
const fused = new Map();
|
|
223
|
+
const methods = new Map();
|
|
224
|
+
for (const [method, ids] of lists) {
|
|
225
|
+
ids.forEach((id, rank) => {
|
|
226
|
+
fused.set(id, (fused.get(id) ?? 0) + 1 / (ScopedGraph.RRF_K + rank + 1));
|
|
227
|
+
methods.set(id, [...(methods.get(id) ?? []), method]);
|
|
228
|
+
});
|
|
229
|
+
}
|
|
230
|
+
if (fused.size === 0) {
|
|
231
|
+
return [];
|
|
232
|
+
}
|
|
233
|
+
// Visibility and retirement are applied by the store, in SQL.
|
|
234
|
+
const chunks = (await this.store.getChunks(this.memoryScope, [...fused.keys()])).filter((c) => c.retiredAt == null);
|
|
235
|
+
return chunks
|
|
236
|
+
.map((chunk) => ({ chunk, score: fused.get(chunk.id), method: methods.get(chunk.id).length > 1 ? 'hybrid' : methods.get(chunk.id)[0] }))
|
|
237
|
+
.sort((a, b) => b.score - a.score || (a.chunk.id < b.chunk.id ? -1 : 1))
|
|
238
|
+
.slice(0, limit);
|
|
239
|
+
}
|
|
240
|
+
async facts(subject, options = {}) {
|
|
241
|
+
return this.store.facts(this.memoryScope, subject, options);
|
|
242
|
+
}
|
|
243
|
+
async timeline(subject) {
|
|
244
|
+
return this.store.timeline(this.memoryScope, subject);
|
|
245
|
+
}
|
|
246
|
+
/** Store a fact at this scope; its status comes from the acceptance policy. */
|
|
247
|
+
async assert(fact) {
|
|
248
|
+
return this.store.assertFact(this.memoryScope, fact, this.policy);
|
|
249
|
+
}
|
|
250
|
+
async supersede(oldKey, replacement) {
|
|
251
|
+
return this.store.supersede(this.memoryScope, oldKey, replacement, this.policy);
|
|
252
|
+
}
|
|
253
|
+
/** An app decision, never a tool. */
|
|
254
|
+
async retract(key) {
|
|
255
|
+
return this.store.retract(this.memoryScope, key);
|
|
256
|
+
}
|
|
257
|
+
/** An app decision, never a tool: a proposed fact becomes accepted. */
|
|
258
|
+
async accept(key) {
|
|
259
|
+
return this.store.accept(this.memoryScope, key);
|
|
260
|
+
}
|
|
261
|
+
/** The vector index for this tenant: another tenant's vectors can neither crowd nor appear in its results. */
|
|
262
|
+
spec() {
|
|
263
|
+
const base = this.embedder.spec();
|
|
264
|
+
return new IndexSpec(`${base.name.slice(0, 80)}.t${sha256(this.memoryScope.tenant).slice(0, 16)}`, base.provider, base.model, base.dimensions, base.recipe);
|
|
265
|
+
}
|
|
266
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The embedder an index is pinned to. Vectors from a different provider,
|
|
3
|
+
* model, dimension count or recipe are refused, never mixed into the index.
|
|
4
|
+
*/
|
|
5
|
+
export declare class IndexSpec {
|
|
6
|
+
readonly name: string;
|
|
7
|
+
readonly provider: string;
|
|
8
|
+
readonly model: string;
|
|
9
|
+
readonly dimensions: number;
|
|
10
|
+
readonly recipe: string;
|
|
11
|
+
constructor(name: string, provider: string, model: string, dimensions: number, recipe?: string);
|
|
12
|
+
/** Stable identity of the embedder, identical to the PHP reference's fingerprint(). */
|
|
13
|
+
fingerprint(): string;
|
|
14
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { digest } from './digest.js';
|
|
2
|
+
import { InvalidArgumentError } from './errors.js';
|
|
3
|
+
/**
|
|
4
|
+
* The embedder an index is pinned to. Vectors from a different provider,
|
|
5
|
+
* model, dimension count or recipe are refused, never mixed into the index.
|
|
6
|
+
*/
|
|
7
|
+
export class IndexSpec {
|
|
8
|
+
name;
|
|
9
|
+
provider;
|
|
10
|
+
model;
|
|
11
|
+
dimensions;
|
|
12
|
+
recipe;
|
|
13
|
+
constructor(name, provider, model, dimensions, recipe = 'raw@1') {
|
|
14
|
+
this.name = name;
|
|
15
|
+
this.provider = provider;
|
|
16
|
+
this.model = model;
|
|
17
|
+
this.dimensions = dimensions;
|
|
18
|
+
this.recipe = recipe;
|
|
19
|
+
if (!/^[A-Za-z0-9_.:-]{1,100}$/.test(name)) {
|
|
20
|
+
throw new InvalidArgumentError('Index names are 1-100 of A-Z a-z 0-9 _ . : -');
|
|
21
|
+
}
|
|
22
|
+
if (!Number.isInteger(dimensions) || dimensions < 1 || dimensions > 65536) {
|
|
23
|
+
throw new InvalidArgumentError('Dimensions must be between 1 and 65536.');
|
|
24
|
+
}
|
|
25
|
+
Object.freeze(this);
|
|
26
|
+
}
|
|
27
|
+
/** Stable identity of the embedder, identical to the PHP reference's fingerprint(). */
|
|
28
|
+
fingerprint() {
|
|
29
|
+
return digest('prism-graph/index@1', [this.provider, this.model, this.dimensions, this.recipe]);
|
|
30
|
+
}
|
|
31
|
+
}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
export { type Chunk, type Chunker, chunkId } from './chunk.js';
|
|
2
|
+
export { MarkdownChunker } from './chunking/markdown.js';
|
|
3
|
+
export { WindowChunker } from './chunking/window.js';
|
|
4
|
+
export { digest, sha256 } from './digest.js';
|
|
5
|
+
export { GraphError, InvalidArgumentError } from './errors.js';
|
|
6
|
+
export type { EraseReport, Fact, IngestReport } from './fact.js';
|
|
7
|
+
export { Graph, ScopedGraph, type Hit } from './graph.js';
|
|
8
|
+
export { IndexSpec } from './index-spec.js';
|
|
9
|
+
export { Scope } from './scope.js';
|
|
10
|
+
export { NodeSqliteDriver, type Row, type SqlDriver, type SqlValue } from './storage/driver.js';
|
|
11
|
+
export { baseSchema, fullTextSchema, repairFullTextSchema, TableMap, vectorSchema } from './storage/schema.js';
|
|
12
|
+
export { ftsMatch, GraphStore, normaliseTime } from './storage/store.js';
|
|
13
|
+
export type { ListedChunk, ListedSource, ListingPage } from './storage/store.js';
|
|
14
|
+
export { MAX_IDS, MAX_NOTE_BYTES, MAX_RESULTS, memoryTools, type MemoryToolDefinition } from './tools.js';
|
|
15
|
+
export { type AcceptancePolicy, DefaultAcceptancePolicy, FactStatus, isTrust, Origin, Trust } from './trust.js';
|
|
16
|
+
export { float32, MAX_K, SqliteVectorStore, type Embedder, type VectorStore } from './vectors.js';
|
|
17
|
+
export { markVisible, scrubUtf8 } from './visible.js';
|
|
18
|
+
export { FactsLayer, SearchLayer, Wake, WakeReport, type WakeIncluded, type WakeItem, type WakeLayer } from './wake.js';
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
export { chunkId } from './chunk.js';
|
|
2
|
+
export { MarkdownChunker } from './chunking/markdown.js';
|
|
3
|
+
export { WindowChunker } from './chunking/window.js';
|
|
4
|
+
export { digest, sha256 } from './digest.js';
|
|
5
|
+
export { GraphError, InvalidArgumentError } from './errors.js';
|
|
6
|
+
export { Graph, ScopedGraph } from './graph.js';
|
|
7
|
+
export { IndexSpec } from './index-spec.js';
|
|
8
|
+
export { Scope } from './scope.js';
|
|
9
|
+
export { NodeSqliteDriver } from './storage/driver.js';
|
|
10
|
+
export { baseSchema, fullTextSchema, repairFullTextSchema, TableMap, vectorSchema } from './storage/schema.js';
|
|
11
|
+
export { ftsMatch, GraphStore, normaliseTime } from './storage/store.js';
|
|
12
|
+
export { MAX_IDS, MAX_NOTE_BYTES, MAX_RESULTS, memoryTools } from './tools.js';
|
|
13
|
+
export { DefaultAcceptancePolicy, FactStatus, isTrust, Origin, Trust } from './trust.js';
|
|
14
|
+
export { float32, MAX_K, SqliteVectorStore } from './vectors.js';
|
|
15
|
+
export { markVisible, scrubUtf8 } from './visible.js';
|
|
16
|
+
export { FactsLayer, SearchLayer, Wake, WakeReport } from './wake.js';
|
package/dist/scope.d.ts
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Where a memory lives and who can read it. Every read and write goes through
|
|
3
|
+
* one; there is no unscoped query.
|
|
4
|
+
*
|
|
5
|
+
* Levels nest: tenant > owner > agent > thread. An agent needs an owner and a
|
|
6
|
+
* thread needs an agent, so no row can be visible across owners except a
|
|
7
|
+
* tenant-wide one written deliberately at tenant level.
|
|
8
|
+
*
|
|
9
|
+
* A reader at (t, o, a, th) sees rows whose tenant is t and whose every
|
|
10
|
+
* narrower level is either unset (shared upward) or exactly its own.
|
|
11
|
+
*/
|
|
12
|
+
export declare class Scope {
|
|
13
|
+
readonly tenant: string;
|
|
14
|
+
readonly owner: string | null;
|
|
15
|
+
readonly agent: string | null;
|
|
16
|
+
readonly thread: string | null;
|
|
17
|
+
constructor(tenant: string, owner?: string | null, agent?: string | null, thread?: string | null);
|
|
18
|
+
static tenant(tenant: string): Scope;
|
|
19
|
+
withOwner(owner: string): Scope;
|
|
20
|
+
withAgent(agent: string): Scope;
|
|
21
|
+
withThread(thread: string): Scope;
|
|
22
|
+
level(): 'tenant' | 'owner' | 'agent' | 'thread';
|
|
23
|
+
/** Can a reader at this scope see a row stored at `row`? */
|
|
24
|
+
canSee(row: Scope): boolean;
|
|
25
|
+
/** The internal id for an app-supplied key at this scope (identical to the PHP reference). */
|
|
26
|
+
qualify(kind: string, key: string): string;
|
|
27
|
+
toRow(): [string, string | null, string | null, string | null];
|
|
28
|
+
}
|
package/dist/scope.js
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { digest } from './digest.js';
|
|
2
|
+
import { InvalidArgumentError } from './errors.js';
|
|
3
|
+
const ID = /^[A-Za-z0-9_.:@/-]{1,191}$/;
|
|
4
|
+
/**
|
|
5
|
+
* Where a memory lives and who can read it. Every read and write goes through
|
|
6
|
+
* one; there is no unscoped query.
|
|
7
|
+
*
|
|
8
|
+
* Levels nest: tenant > owner > agent > thread. An agent needs an owner and a
|
|
9
|
+
* thread needs an agent, so no row can be visible across owners except a
|
|
10
|
+
* tenant-wide one written deliberately at tenant level.
|
|
11
|
+
*
|
|
12
|
+
* A reader at (t, o, a, th) sees rows whose tenant is t and whose every
|
|
13
|
+
* narrower level is either unset (shared upward) or exactly its own.
|
|
14
|
+
*/
|
|
15
|
+
export class Scope {
|
|
16
|
+
tenant;
|
|
17
|
+
owner;
|
|
18
|
+
agent;
|
|
19
|
+
thread;
|
|
20
|
+
constructor(tenant, owner = null, agent = null, thread = null) {
|
|
21
|
+
for (const [level, value] of [['tenant', tenant], ['owner', owner], ['agent', agent], ['thread', thread]]) {
|
|
22
|
+
if (value !== null && (typeof value !== 'string' || !ID.test(value))) {
|
|
23
|
+
throw new InvalidArgumentError(`Scope ${level} must be 1-191 of A-Z a-z 0-9 _ . : @ / -`);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
if (agent !== null && owner === null) {
|
|
27
|
+
throw new InvalidArgumentError('An agent scope needs an owner: agent memory is always about someone.');
|
|
28
|
+
}
|
|
29
|
+
if (thread !== null && agent === null) {
|
|
30
|
+
throw new InvalidArgumentError('A thread scope needs an agent.');
|
|
31
|
+
}
|
|
32
|
+
this.tenant = tenant;
|
|
33
|
+
this.owner = owner;
|
|
34
|
+
this.agent = agent;
|
|
35
|
+
this.thread = thread;
|
|
36
|
+
Object.freeze(this);
|
|
37
|
+
}
|
|
38
|
+
static tenant(tenant) {
|
|
39
|
+
return new Scope(tenant);
|
|
40
|
+
}
|
|
41
|
+
withOwner(owner) {
|
|
42
|
+
return new Scope(this.tenant, owner);
|
|
43
|
+
}
|
|
44
|
+
withAgent(agent) {
|
|
45
|
+
if (this.owner === null) {
|
|
46
|
+
throw new InvalidArgumentError('Set the owner before the agent.');
|
|
47
|
+
}
|
|
48
|
+
return new Scope(this.tenant, this.owner, agent);
|
|
49
|
+
}
|
|
50
|
+
withThread(thread) {
|
|
51
|
+
return new Scope(this.tenant, this.owner, this.agent, thread);
|
|
52
|
+
}
|
|
53
|
+
level() {
|
|
54
|
+
return this.thread !== null ? 'thread' : this.agent !== null ? 'agent' : this.owner !== null ? 'owner' : 'tenant';
|
|
55
|
+
}
|
|
56
|
+
/** Can a reader at this scope see a row stored at `row`? */
|
|
57
|
+
canSee(row) {
|
|
58
|
+
return row.tenant === this.tenant
|
|
59
|
+
&& (row.owner === null || row.owner === this.owner)
|
|
60
|
+
&& (row.agent === null || row.agent === this.agent)
|
|
61
|
+
&& (row.thread === null || row.thread === this.thread);
|
|
62
|
+
}
|
|
63
|
+
/** The internal id for an app-supplied key at this scope (identical to the PHP reference). */
|
|
64
|
+
qualify(kind, key) {
|
|
65
|
+
return digest(`prism-graph/${kind}@1`, [this.tenant, this.owner ?? '', this.agent ?? '', this.thread ?? '', key]);
|
|
66
|
+
}
|
|
67
|
+
toRow() {
|
|
68
|
+
return [this.tenant, this.owner, this.agent, this.thread];
|
|
69
|
+
}
|
|
70
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import type { DatabaseSync } from 'node:sqlite';
|
|
2
|
+
export type SqlValue = string | number | bigint | null | Uint8Array;
|
|
3
|
+
export type Row = Record<string, unknown>;
|
|
4
|
+
/**
|
|
5
|
+
* The SQL the stores need, and nothing else. Every value is a bound
|
|
6
|
+
* parameter ("?" placeholders); table names come only from a validated
|
|
7
|
+
* TableMap.
|
|
8
|
+
*
|
|
9
|
+
* `atomic()` is all-or-nothing INCLUDING inside a caller's own transaction:
|
|
10
|
+
* the work runs in a SAVEPOINT, so a refusal or failure rolls back exactly
|
|
11
|
+
* prism-graph's work and leaves the caller's transaction usable. (The PHP
|
|
12
|
+
* reference had to learn this the hard way: see its 0.2.1 notes.)
|
|
13
|
+
*/
|
|
14
|
+
export interface SqlDriver {
|
|
15
|
+
readonly dialect: 'sqlite';
|
|
16
|
+
run(sql: string, params?: readonly SqlValue[]): Promise<number>;
|
|
17
|
+
all(sql: string, params?: readonly SqlValue[]): Promise<Row[]>;
|
|
18
|
+
exec(sql: string): Promise<void>;
|
|
19
|
+
atomic<T>(work: () => Promise<T>): Promise<T>;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* SQLite through Node's built-in node:sqlite (Node 22.16+): no native module,
|
|
23
|
+
* no third-party dependency. Needs SQLite with FTS5, which node:sqlite has.
|
|
24
|
+
*
|
|
25
|
+
* GIVE IT A CONNECTION OF ITS OWN, or route every write on that connection
|
|
26
|
+
* through `atomic()`. SQLite transactions belong to the connection: a write
|
|
27
|
+
* another part of the app makes while one of our blocks is open lands inside
|
|
28
|
+
* our savepoint, and is rolled back with it if our block fails.
|
|
29
|
+
*
|
|
30
|
+
* Atomic blocks are serialised on the connection. A block opened inside
|
|
31
|
+
* another (the same async call chain) queues inside its parent; a callback
|
|
32
|
+
* that outlives its block (a timer, an un-awaited promise) is treated as a
|
|
33
|
+
* new caller, never as part of a block that has already closed.
|
|
34
|
+
*/
|
|
35
|
+
export declare class NodeSqliteDriver implements SqlDriver {
|
|
36
|
+
private readonly db;
|
|
37
|
+
readonly dialect: "sqlite";
|
|
38
|
+
private queue;
|
|
39
|
+
/** Only ever increases: savepoint names are unique for the life of the driver. */
|
|
40
|
+
private counter;
|
|
41
|
+
private readonly inside;
|
|
42
|
+
constructor(db: DatabaseSync);
|
|
43
|
+
run(sql: string, params?: readonly SqlValue[]): Promise<number>;
|
|
44
|
+
all(sql: string, params?: readonly SqlValue[]): Promise<Row[]>;
|
|
45
|
+
exec(sql: string): Promise<void>;
|
|
46
|
+
atomic<T>(work: () => Promise<T>): Promise<T>;
|
|
47
|
+
private savepoint;
|
|
48
|
+
private rollback;
|
|
49
|
+
}
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
2
|
+
import { GraphError } from '../errors.js';
|
|
3
|
+
/**
|
|
4
|
+
* SQLite through Node's built-in node:sqlite (Node 22.16+): no native module,
|
|
5
|
+
* no third-party dependency. Needs SQLite with FTS5, which node:sqlite has.
|
|
6
|
+
*
|
|
7
|
+
* GIVE IT A CONNECTION OF ITS OWN, or route every write on that connection
|
|
8
|
+
* through `atomic()`. SQLite transactions belong to the connection: a write
|
|
9
|
+
* another part of the app makes while one of our blocks is open lands inside
|
|
10
|
+
* our savepoint, and is rolled back with it if our block fails.
|
|
11
|
+
*
|
|
12
|
+
* Atomic blocks are serialised on the connection. A block opened inside
|
|
13
|
+
* another (the same async call chain) queues inside its parent; a callback
|
|
14
|
+
* that outlives its block (a timer, an un-awaited promise) is treated as a
|
|
15
|
+
* new caller, never as part of a block that has already closed.
|
|
16
|
+
*/
|
|
17
|
+
export class NodeSqliteDriver {
|
|
18
|
+
db;
|
|
19
|
+
dialect = 'sqlite';
|
|
20
|
+
queue = Promise.resolve();
|
|
21
|
+
/** Only ever increases: savepoint names are unique for the life of the driver. */
|
|
22
|
+
counter = 0;
|
|
23
|
+
inside = new AsyncLocalStorage();
|
|
24
|
+
constructor(db) {
|
|
25
|
+
this.db = db;
|
|
26
|
+
if (typeof db.isTransaction !== 'boolean') {
|
|
27
|
+
// Without it, a failed commit cannot be told apart from a caller's own
|
|
28
|
+
// transaction, and a stuck connection would silently lose later writes.
|
|
29
|
+
throw new GraphError('prism-graph needs node:sqlite with DatabaseSync.isTransaction (Node 22.16+).');
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
async run(sql, params = []) {
|
|
33
|
+
return Number(this.db.prepare(sql).run(...params.map(bind)).changes);
|
|
34
|
+
}
|
|
35
|
+
async all(sql, params = []) {
|
|
36
|
+
return this.db.prepare(sql).all(...params.map(bind));
|
|
37
|
+
}
|
|
38
|
+
async exec(sql) {
|
|
39
|
+
this.db.exec(sql);
|
|
40
|
+
}
|
|
41
|
+
atomic(work) {
|
|
42
|
+
const parent = this.inside.getStore();
|
|
43
|
+
if (parent?.open === true) {
|
|
44
|
+
// Nested in an open block: run inside it, one nested block at a time.
|
|
45
|
+
const result = parent.queue.then(() => this.savepoint(work, false));
|
|
46
|
+
parent.queue = result.catch(() => undefined);
|
|
47
|
+
return result;
|
|
48
|
+
}
|
|
49
|
+
const result = this.queue.then(() => this.savepoint(work, true));
|
|
50
|
+
this.queue = result.catch(() => undefined);
|
|
51
|
+
return result;
|
|
52
|
+
}
|
|
53
|
+
async savepoint(work, outermost) {
|
|
54
|
+
// In SQLite a SAVEPOINT outside a transaction opens one, and RELEASE
|
|
55
|
+
// commits it; inside the caller's transaction it nests. One rule for both.
|
|
56
|
+
// Remember whether THIS block opened the transaction: only then may a
|
|
57
|
+
// failure close it with a full ROLLBACK.
|
|
58
|
+
const opened = outermost && !this.db.isTransaction;
|
|
59
|
+
const name = `prism_graph_sp_${++this.counter}`;
|
|
60
|
+
this.db.exec(`SAVEPOINT ${name}`);
|
|
61
|
+
const block = { queue: Promise.resolve(), open: true };
|
|
62
|
+
let result;
|
|
63
|
+
try {
|
|
64
|
+
result = await this.inside.run(block, work);
|
|
65
|
+
await block.queue; // nested blocks the work started but did not await
|
|
66
|
+
}
|
|
67
|
+
catch (error) {
|
|
68
|
+
block.open = false;
|
|
69
|
+
this.rollback(name, opened);
|
|
70
|
+
throw error;
|
|
71
|
+
}
|
|
72
|
+
block.open = false;
|
|
73
|
+
try {
|
|
74
|
+
this.db.exec(`RELEASE SAVEPOINT ${name}`);
|
|
75
|
+
}
|
|
76
|
+
catch (error) {
|
|
77
|
+
// The commit itself failed (e.g. SQLITE_BUSY on the outermost RELEASE):
|
|
78
|
+
// undo our work and never leave the connection inside a transaction,
|
|
79
|
+
// where every later write would report success and be lost.
|
|
80
|
+
this.rollback(name, opened);
|
|
81
|
+
throw error;
|
|
82
|
+
}
|
|
83
|
+
return result;
|
|
84
|
+
}
|
|
85
|
+
rollback(name, opened) {
|
|
86
|
+
try {
|
|
87
|
+
this.db.exec(`ROLLBACK TO SAVEPOINT ${name}`);
|
|
88
|
+
this.db.exec(`RELEASE SAVEPOINT ${name}`);
|
|
89
|
+
}
|
|
90
|
+
catch {
|
|
91
|
+
// Fall through: if we opened the transaction, close it below.
|
|
92
|
+
}
|
|
93
|
+
if (opened && this.db.isTransaction) {
|
|
94
|
+
this.db.exec('ROLLBACK');
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
function bind(value) {
|
|
99
|
+
return value === undefined ? null : value;
|
|
100
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which app-owned tables play each role. Apps create and extend these tables
|
|
3
|
+
* in their own migrations; the package only reads and writes the core columns
|
|
4
|
+
* it documents. Names are capped at 50 characters so derived names stay inside
|
|
5
|
+
* PostgreSQL's 63-character identifier limit.
|
|
6
|
+
*/
|
|
7
|
+
export declare class TableMap {
|
|
8
|
+
readonly sources: string;
|
|
9
|
+
readonly chunks: string;
|
|
10
|
+
readonly facts: string;
|
|
11
|
+
readonly evidence: string;
|
|
12
|
+
constructor(sources?: string, chunks?: string, facts?: string, evidence?: string);
|
|
13
|
+
/** The full-text index over chunks (an FTS5 table on SQLite). */
|
|
14
|
+
fts(): string;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* The core schema, as plain SQL for SQLite, matching the PHP reference's
|
|
18
|
+
* BaseSchema. The package never creates tables at runtime: apps run these from
|
|
19
|
+
* their own migration tooling, and own them afterwards.
|
|
20
|
+
*/
|
|
21
|
+
export declare function baseSchema(t?: TableMap): string[];
|
|
22
|
+
/** External-content FTS5 kept in step by triggers; the text lives once, in the chunks table. */
|
|
23
|
+
export declare function fullTextSchema(t?: TableMap): string[];
|
|
24
|
+
/**
|
|
25
|
+
* Restore the full-text index after a migration that rebuilt the chunks table
|
|
26
|
+
* (which drops its triggers), and rebuild it from the stored chunks. Safe to
|
|
27
|
+
* run at any time.
|
|
28
|
+
*/
|
|
29
|
+
export declare function repairFullTextSchema(t?: TableMap): string[];
|
|
30
|
+
/** The optional vector tables (exact cosine search over packed float32). */
|
|
31
|
+
export declare function vectorSchema(indexes?: string, vectors?: string): string[];
|