@particle-academy/prism-graph 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +103 -2
- package/dist/chunk.d.ts +29 -0
- package/dist/chunk.js +5 -0
- package/dist/chunking/markdown.d.ts +10 -0
- package/dist/chunking/markdown.js +64 -0
- package/dist/chunking/window.d.ts +23 -0
- package/dist/chunking/window.js +128 -0
- package/dist/digest.d.ts +8 -0
- package/dist/digest.js +25 -0
- package/dist/errors.d.ts +10 -0
- package/dist/errors.js +14 -0
- package/dist/fact.d.ts +44 -0
- package/dist/fact.js +1 -0
- package/dist/graph.d.ts +82 -0
- package/dist/graph.js +176 -0
- package/dist/index-spec.d.ts +14 -0
- package/dist/index-spec.js +31 -0
- package/dist/index.d.ts +17 -0
- package/dist/index.js +16 -0
- package/dist/scope.d.ts +28 -0
- package/dist/scope.js +70 -0
- package/dist/storage/driver.d.ts +49 -0
- package/dist/storage/driver.js +100 -0
- package/dist/storage/schema.d.ts +31 -0
- package/dist/storage/schema.js +102 -0
- package/dist/storage/store.d.ts +85 -0
- package/dist/storage/store.js +331 -0
- package/dist/tools.d.ts +28 -0
- package/dist/tools.js +166 -0
- package/dist/trust.d.ts +56 -0
- package/dist/trust.js +73 -0
- package/dist/vectors.d.ts +55 -0
- package/dist/vectors.js +155 -0
- package/dist/visible.d.ts +6 -0
- package/dist/visible.js +34 -0
- package/dist/wake.d.ts +66 -0
- package/dist/wake.js +126 -0
- package/package.json +34 -4
package/dist/graph.js
ADDED
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
import { sha256 } from './digest.js';
|
|
2
|
+
import { GraphError, InvalidArgumentError } from './errors.js';
|
|
3
|
+
import { IndexSpec } from './index-spec.js';
|
|
4
|
+
import { DefaultAcceptancePolicy } from './trust.js';
|
|
5
|
+
/**
|
|
6
|
+
* Long-term memory for agents. This object only holds the wiring (store,
|
|
7
|
+
* optional vectors and embedder, acceptance policy); it has NO read or write
|
|
8
|
+
* methods of its own. Everything goes through a scope, so an unscoped query
|
|
9
|
+
* cannot be written, by an app or by a model.
|
|
10
|
+
*/
|
|
11
|
+
export class Graph {
|
|
12
|
+
store;
|
|
13
|
+
vectors;
|
|
14
|
+
embedder;
|
|
15
|
+
policy;
|
|
16
|
+
constructor(store, vectors = null, embedder = null, policy = new DefaultAcceptancePolicy()) {
|
|
17
|
+
this.store = store;
|
|
18
|
+
this.vectors = vectors;
|
|
19
|
+
this.embedder = embedder;
|
|
20
|
+
this.policy = policy;
|
|
21
|
+
if ((vectors === null) !== (embedder === null)) {
|
|
22
|
+
throw new InvalidArgumentError('Configure a VectorStore and an Embedder together, or neither.');
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
scope(scope) {
|
|
26
|
+
return new ScopedGraph(scope, this.store, this.vectors, this.embedder, this.policy);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Memory as seen from one scope. Reads return what this scope can see (its own
|
|
31
|
+
* rows and those shared from wider scopes above it); writes land at exactly
|
|
32
|
+
* this scope. Nothing here widens a write.
|
|
33
|
+
*/
|
|
34
|
+
export class ScopedGraph {
|
|
35
|
+
memoryScope;
|
|
36
|
+
store;
|
|
37
|
+
vectors;
|
|
38
|
+
embedder;
|
|
39
|
+
policy;
|
|
40
|
+
static MAX_LIMIT = 200;
|
|
41
|
+
/** A query goes to the embedding provider and the full-text index, so it is bounded. */
|
|
42
|
+
static MAX_QUERY_BYTES = 4096;
|
|
43
|
+
static MAX_GET_IDS = 1000;
|
|
44
|
+
static RRF_K = 60;
|
|
45
|
+
constructor(memoryScope, store, vectors, embedder, policy) {
|
|
46
|
+
this.memoryScope = memoryScope;
|
|
47
|
+
this.store = store;
|
|
48
|
+
this.vectors = vectors;
|
|
49
|
+
this.embedder = embedder;
|
|
50
|
+
this.policy = policy;
|
|
51
|
+
}
|
|
52
|
+
scope() {
|
|
53
|
+
return this.memoryScope;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Store a source and its verbatim chunks at this scope. Idempotent per key:
|
|
57
|
+
* unchanged text stores and embeds nothing new; text the source no longer
|
|
58
|
+
* contains is retired from search. Verbatim is stored first and never waits
|
|
59
|
+
* on embedding; a failed embedding heals on the next ingest.
|
|
60
|
+
*/
|
|
61
|
+
async ingest(key, uri, text, chunker, origin, options = {}) {
|
|
62
|
+
const actual = sha256(Buffer.from(text, 'utf8'));
|
|
63
|
+
if (options.sha256 !== undefined && options.sha256.toLowerCase() !== actual) {
|
|
64
|
+
throw new GraphError(`The provided sha256 for [${key}] does not match its text.`);
|
|
65
|
+
}
|
|
66
|
+
const chunks = chunker.chunk(this.memoryScope.qualify('source', key), text);
|
|
67
|
+
const { stored, retired } = await this.store.storeSource(this.memoryScope, key, uri, actual, options.metadata ?? {}, origin, options.occurredAt ?? null, chunks);
|
|
68
|
+
if (this.vectors !== null && this.embedder !== null && chunks.length > 0) {
|
|
69
|
+
const spec = this.spec();
|
|
70
|
+
await this.vectors.ensureIndex(spec);
|
|
71
|
+
const byId = new Map(chunks.map((c) => [c.id, c]));
|
|
72
|
+
const ids = [...byId.keys()];
|
|
73
|
+
for (let i = 0; i < ids.length; i += 500) {
|
|
74
|
+
const todo = await this.vectors.missing(spec, ids.slice(i, i + 500));
|
|
75
|
+
if (todo.length === 0) {
|
|
76
|
+
continue;
|
|
77
|
+
}
|
|
78
|
+
const vectors = await this.embedder.embed(todo.map((id) => byId.get(id).text));
|
|
79
|
+
if (vectors.length !== todo.length) {
|
|
80
|
+
throw new GraphError('The embedder returned a different number of vectors than chunks.');
|
|
81
|
+
}
|
|
82
|
+
await this.vectors.upsert(spec, new Map(todo.map((id, n) => [id, vectors[n]])));
|
|
83
|
+
// An erase that ran while the provider was embedding has already removed
|
|
84
|
+
// these chunks; erased text must not live on as an embedding.
|
|
85
|
+
const present = new Set((await this.store.getChunks(this.memoryScope, todo)).map((c) => c.id));
|
|
86
|
+
const gone = todo.filter((id) => !present.has(id));
|
|
87
|
+
if (gone.length > 0) {
|
|
88
|
+
await this.vectors.forget(gone);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
return { source: key, sha256: actual, recipe: chunker.recipe(), chunks: chunks.length, stored, retired };
|
|
93
|
+
}
|
|
94
|
+
/** Take a source at this scope out of search; it stays readable by id. */
|
|
95
|
+
async retireSource(key) {
|
|
96
|
+
return this.store.retireSource(this.memoryScope, key);
|
|
97
|
+
}
|
|
98
|
+
/** Erase a source stored at this scope: chunks, vectors, evidence links and orphaned facts. */
|
|
99
|
+
async erase(key) {
|
|
100
|
+
const report = await this.store.eraseSource(this.memoryScope, key);
|
|
101
|
+
if (this.vectors === null || report.chunkIds.length === 0) {
|
|
102
|
+
return report;
|
|
103
|
+
}
|
|
104
|
+
return { ...report, vectors: await this.vectors.forget(report.chunkIds) };
|
|
105
|
+
}
|
|
106
|
+
async get(chunkIds) {
|
|
107
|
+
if (chunkIds.length > ScopedGraph.MAX_GET_IDS) {
|
|
108
|
+
throw new InvalidArgumentError(`Pass at most ${ScopedGraph.MAX_GET_IDS} chunk ids.`);
|
|
109
|
+
}
|
|
110
|
+
return this.store.getChunks(this.memoryScope, chunkIds);
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Hybrid search over what this scope can see: full-text (BM25) and, when
|
|
114
|
+
* configured and `vectors` is true, vectors, fused by reciprocal rank. A
|
|
115
|
+
* chunk with no vector yet still ranks through text.
|
|
116
|
+
*/
|
|
117
|
+
async search(query, limit = 10, vectors = true) {
|
|
118
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > ScopedGraph.MAX_LIMIT) {
|
|
119
|
+
throw new InvalidArgumentError(`limit must be between 1 and ${ScopedGraph.MAX_LIMIT}.`);
|
|
120
|
+
}
|
|
121
|
+
if (Buffer.byteLength(query, 'utf8') > ScopedGraph.MAX_QUERY_BYTES) {
|
|
122
|
+
throw new InvalidArgumentError(`A query may be at most ${ScopedGraph.MAX_QUERY_BYTES} bytes.`);
|
|
123
|
+
}
|
|
124
|
+
if (!query.isWellFormed()) {
|
|
125
|
+
throw new InvalidArgumentError('A query must be valid UTF-8.');
|
|
126
|
+
}
|
|
127
|
+
const lists = new Map([['text', (await this.store.textSearch(this.memoryScope, query, limit * 3)).map((r) => r.chunkId)]]);
|
|
128
|
+
if (vectors && this.vectors !== null && this.embedder !== null && query.trim() !== '') {
|
|
129
|
+
const embedded = (await this.embedder.embed([query]))[0];
|
|
130
|
+
lists.set(this.vectors.method(), (await this.vectors.nearest(this.spec(), embedded, Math.min(1000, limit * 10))).map((r) => r.chunkId));
|
|
131
|
+
}
|
|
132
|
+
const fused = new Map();
|
|
133
|
+
const methods = new Map();
|
|
134
|
+
for (const [method, ids] of lists) {
|
|
135
|
+
ids.forEach((id, rank) => {
|
|
136
|
+
fused.set(id, (fused.get(id) ?? 0) + 1 / (ScopedGraph.RRF_K + rank + 1));
|
|
137
|
+
methods.set(id, [...(methods.get(id) ?? []), method]);
|
|
138
|
+
});
|
|
139
|
+
}
|
|
140
|
+
if (fused.size === 0) {
|
|
141
|
+
return [];
|
|
142
|
+
}
|
|
143
|
+
// Visibility and retirement are applied by the store, in SQL.
|
|
144
|
+
const chunks = (await this.store.getChunks(this.memoryScope, [...fused.keys()])).filter((c) => c.retiredAt == null);
|
|
145
|
+
return chunks
|
|
146
|
+
.map((chunk) => ({ chunk, score: fused.get(chunk.id), method: methods.get(chunk.id).length > 1 ? 'hybrid' : methods.get(chunk.id)[0] }))
|
|
147
|
+
.sort((a, b) => b.score - a.score || (a.chunk.id < b.chunk.id ? -1 : 1))
|
|
148
|
+
.slice(0, limit);
|
|
149
|
+
}
|
|
150
|
+
async facts(subject, options = {}) {
|
|
151
|
+
return this.store.facts(this.memoryScope, subject, options);
|
|
152
|
+
}
|
|
153
|
+
async timeline(subject) {
|
|
154
|
+
return this.store.timeline(this.memoryScope, subject);
|
|
155
|
+
}
|
|
156
|
+
/** Store a fact at this scope; its status comes from the acceptance policy. */
|
|
157
|
+
async assert(fact) {
|
|
158
|
+
return this.store.assertFact(this.memoryScope, fact, this.policy);
|
|
159
|
+
}
|
|
160
|
+
async supersede(oldKey, replacement) {
|
|
161
|
+
return this.store.supersede(this.memoryScope, oldKey, replacement, this.policy);
|
|
162
|
+
}
|
|
163
|
+
/** An app decision, never a tool. */
|
|
164
|
+
async retract(key) {
|
|
165
|
+
return this.store.retract(this.memoryScope, key);
|
|
166
|
+
}
|
|
167
|
+
/** An app decision, never a tool: a proposed fact becomes accepted. */
|
|
168
|
+
async accept(key) {
|
|
169
|
+
return this.store.accept(this.memoryScope, key);
|
|
170
|
+
}
|
|
171
|
+
/** The vector index for this tenant: another tenant's vectors can neither crowd nor appear in its results. */
|
|
172
|
+
spec() {
|
|
173
|
+
const base = this.embedder.spec();
|
|
174
|
+
return new IndexSpec(`${base.name.slice(0, 80)}.t${sha256(this.memoryScope.tenant).slice(0, 16)}`, base.provider, base.model, base.dimensions, base.recipe);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The embedder an index is pinned to. Vectors from a different provider,
|
|
3
|
+
* model, dimension count or recipe are refused, never mixed into the index.
|
|
4
|
+
*/
|
|
5
|
+
export declare class IndexSpec {
|
|
6
|
+
readonly name: string;
|
|
7
|
+
readonly provider: string;
|
|
8
|
+
readonly model: string;
|
|
9
|
+
readonly dimensions: number;
|
|
10
|
+
readonly recipe: string;
|
|
11
|
+
constructor(name: string, provider: string, model: string, dimensions: number, recipe?: string);
|
|
12
|
+
/** Stable identity of the embedder, identical to the PHP reference's fingerprint(). */
|
|
13
|
+
fingerprint(): string;
|
|
14
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { digest } from './digest.js';
|
|
2
|
+
import { InvalidArgumentError } from './errors.js';
|
|
3
|
+
/**
|
|
4
|
+
* The embedder an index is pinned to. Vectors from a different provider,
|
|
5
|
+
* model, dimension count or recipe are refused, never mixed into the index.
|
|
6
|
+
*/
|
|
7
|
+
export class IndexSpec {
|
|
8
|
+
name;
|
|
9
|
+
provider;
|
|
10
|
+
model;
|
|
11
|
+
dimensions;
|
|
12
|
+
recipe;
|
|
13
|
+
constructor(name, provider, model, dimensions, recipe = 'raw@1') {
|
|
14
|
+
this.name = name;
|
|
15
|
+
this.provider = provider;
|
|
16
|
+
this.model = model;
|
|
17
|
+
this.dimensions = dimensions;
|
|
18
|
+
this.recipe = recipe;
|
|
19
|
+
if (!/^[A-Za-z0-9_.:-]{1,100}$/.test(name)) {
|
|
20
|
+
throw new InvalidArgumentError('Index names are 1-100 of A-Z a-z 0-9 _ . : -');
|
|
21
|
+
}
|
|
22
|
+
if (!Number.isInteger(dimensions) || dimensions < 1 || dimensions > 65536) {
|
|
23
|
+
throw new InvalidArgumentError('Dimensions must be between 1 and 65536.');
|
|
24
|
+
}
|
|
25
|
+
Object.freeze(this);
|
|
26
|
+
}
|
|
27
|
+
/** Stable identity of the embedder, identical to the PHP reference's fingerprint(). */
|
|
28
|
+
fingerprint() {
|
|
29
|
+
return digest('prism-graph/index@1', [this.provider, this.model, this.dimensions, this.recipe]);
|
|
30
|
+
}
|
|
31
|
+
}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export { type Chunk, type Chunker, chunkId } from './chunk.js';
|
|
2
|
+
export { MarkdownChunker } from './chunking/markdown.js';
|
|
3
|
+
export { WindowChunker } from './chunking/window.js';
|
|
4
|
+
export { digest, sha256 } from './digest.js';
|
|
5
|
+
export { GraphError, InvalidArgumentError } from './errors.js';
|
|
6
|
+
export type { EraseReport, Fact, IngestReport } from './fact.js';
|
|
7
|
+
export { Graph, ScopedGraph, type Hit } from './graph.js';
|
|
8
|
+
export { IndexSpec } from './index-spec.js';
|
|
9
|
+
export { Scope } from './scope.js';
|
|
10
|
+
export { NodeSqliteDriver, type Row, type SqlDriver, type SqlValue } from './storage/driver.js';
|
|
11
|
+
export { baseSchema, fullTextSchema, repairFullTextSchema, TableMap, vectorSchema } from './storage/schema.js';
|
|
12
|
+
export { ftsMatch, GraphStore, normaliseTime } from './storage/store.js';
|
|
13
|
+
export { MAX_IDS, MAX_NOTE_BYTES, MAX_RESULTS, memoryTools, type MemoryToolDefinition } from './tools.js';
|
|
14
|
+
export { type AcceptancePolicy, DefaultAcceptancePolicy, FactStatus, isTrust, Origin, Trust } from './trust.js';
|
|
15
|
+
export { float32, MAX_K, SqliteVectorStore, type Embedder, type VectorStore } from './vectors.js';
|
|
16
|
+
export { markVisible, scrubUtf8 } from './visible.js';
|
|
17
|
+
export { FactsLayer, SearchLayer, Wake, WakeReport, type WakeIncluded, type WakeItem, type WakeLayer } from './wake.js';
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
export { chunkId } from './chunk.js';
|
|
2
|
+
export { MarkdownChunker } from './chunking/markdown.js';
|
|
3
|
+
export { WindowChunker } from './chunking/window.js';
|
|
4
|
+
export { digest, sha256 } from './digest.js';
|
|
5
|
+
export { GraphError, InvalidArgumentError } from './errors.js';
|
|
6
|
+
export { Graph, ScopedGraph } from './graph.js';
|
|
7
|
+
export { IndexSpec } from './index-spec.js';
|
|
8
|
+
export { Scope } from './scope.js';
|
|
9
|
+
export { NodeSqliteDriver } from './storage/driver.js';
|
|
10
|
+
export { baseSchema, fullTextSchema, repairFullTextSchema, TableMap, vectorSchema } from './storage/schema.js';
|
|
11
|
+
export { ftsMatch, GraphStore, normaliseTime } from './storage/store.js';
|
|
12
|
+
export { MAX_IDS, MAX_NOTE_BYTES, MAX_RESULTS, memoryTools } from './tools.js';
|
|
13
|
+
export { DefaultAcceptancePolicy, FactStatus, isTrust, Origin, Trust } from './trust.js';
|
|
14
|
+
export { float32, MAX_K, SqliteVectorStore } from './vectors.js';
|
|
15
|
+
export { markVisible, scrubUtf8 } from './visible.js';
|
|
16
|
+
export { FactsLayer, SearchLayer, Wake, WakeReport } from './wake.js';
|
package/dist/scope.d.ts
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Where a memory lives and who can read it. Every read and write goes through
|
|
3
|
+
* one; there is no unscoped query.
|
|
4
|
+
*
|
|
5
|
+
* Levels nest: tenant > owner > agent > thread. An agent needs an owner and a
|
|
6
|
+
* thread needs an agent, so no row can be visible across owners except a
|
|
7
|
+
* tenant-wide one written deliberately at tenant level.
|
|
8
|
+
*
|
|
9
|
+
* A reader at (t, o, a, th) sees rows whose tenant is t and whose every
|
|
10
|
+
* narrower level is either unset (shared upward) or exactly its own.
|
|
11
|
+
*/
|
|
12
|
+
export declare class Scope {
|
|
13
|
+
readonly tenant: string;
|
|
14
|
+
readonly owner: string | null;
|
|
15
|
+
readonly agent: string | null;
|
|
16
|
+
readonly thread: string | null;
|
|
17
|
+
constructor(tenant: string, owner?: string | null, agent?: string | null, thread?: string | null);
|
|
18
|
+
static tenant(tenant: string): Scope;
|
|
19
|
+
withOwner(owner: string): Scope;
|
|
20
|
+
withAgent(agent: string): Scope;
|
|
21
|
+
withThread(thread: string): Scope;
|
|
22
|
+
level(): 'tenant' | 'owner' | 'agent' | 'thread';
|
|
23
|
+
/** Can a reader at this scope see a row stored at `row`? */
|
|
24
|
+
canSee(row: Scope): boolean;
|
|
25
|
+
/** The internal id for an app-supplied key at this scope (identical to the PHP reference). */
|
|
26
|
+
qualify(kind: string, key: string): string;
|
|
27
|
+
toRow(): [string, string | null, string | null, string | null];
|
|
28
|
+
}
|
package/dist/scope.js
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { digest } from './digest.js';
|
|
2
|
+
import { InvalidArgumentError } from './errors.js';
|
|
3
|
+
const ID = /^[A-Za-z0-9_.:@/-]{1,191}$/;
|
|
4
|
+
/**
|
|
5
|
+
* Where a memory lives and who can read it. Every read and write goes through
|
|
6
|
+
* one; there is no unscoped query.
|
|
7
|
+
*
|
|
8
|
+
* Levels nest: tenant > owner > agent > thread. An agent needs an owner and a
|
|
9
|
+
* thread needs an agent, so no row can be visible across owners except a
|
|
10
|
+
* tenant-wide one written deliberately at tenant level.
|
|
11
|
+
*
|
|
12
|
+
* A reader at (t, o, a, th) sees rows whose tenant is t and whose every
|
|
13
|
+
* narrower level is either unset (shared upward) or exactly its own.
|
|
14
|
+
*/
|
|
15
|
+
export class Scope {
|
|
16
|
+
tenant;
|
|
17
|
+
owner;
|
|
18
|
+
agent;
|
|
19
|
+
thread;
|
|
20
|
+
constructor(tenant, owner = null, agent = null, thread = null) {
|
|
21
|
+
for (const [level, value] of [['tenant', tenant], ['owner', owner], ['agent', agent], ['thread', thread]]) {
|
|
22
|
+
if (value !== null && (typeof value !== 'string' || !ID.test(value))) {
|
|
23
|
+
throw new InvalidArgumentError(`Scope ${level} must be 1-191 of A-Z a-z 0-9 _ . : @ / -`);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
if (agent !== null && owner === null) {
|
|
27
|
+
throw new InvalidArgumentError('An agent scope needs an owner: agent memory is always about someone.');
|
|
28
|
+
}
|
|
29
|
+
if (thread !== null && agent === null) {
|
|
30
|
+
throw new InvalidArgumentError('A thread scope needs an agent.');
|
|
31
|
+
}
|
|
32
|
+
this.tenant = tenant;
|
|
33
|
+
this.owner = owner;
|
|
34
|
+
this.agent = agent;
|
|
35
|
+
this.thread = thread;
|
|
36
|
+
Object.freeze(this);
|
|
37
|
+
}
|
|
38
|
+
static tenant(tenant) {
|
|
39
|
+
return new Scope(tenant);
|
|
40
|
+
}
|
|
41
|
+
withOwner(owner) {
|
|
42
|
+
return new Scope(this.tenant, owner);
|
|
43
|
+
}
|
|
44
|
+
withAgent(agent) {
|
|
45
|
+
if (this.owner === null) {
|
|
46
|
+
throw new InvalidArgumentError('Set the owner before the agent.');
|
|
47
|
+
}
|
|
48
|
+
return new Scope(this.tenant, this.owner, agent);
|
|
49
|
+
}
|
|
50
|
+
withThread(thread) {
|
|
51
|
+
return new Scope(this.tenant, this.owner, this.agent, thread);
|
|
52
|
+
}
|
|
53
|
+
level() {
|
|
54
|
+
return this.thread !== null ? 'thread' : this.agent !== null ? 'agent' : this.owner !== null ? 'owner' : 'tenant';
|
|
55
|
+
}
|
|
56
|
+
/** Can a reader at this scope see a row stored at `row`? */
|
|
57
|
+
canSee(row) {
|
|
58
|
+
return row.tenant === this.tenant
|
|
59
|
+
&& (row.owner === null || row.owner === this.owner)
|
|
60
|
+
&& (row.agent === null || row.agent === this.agent)
|
|
61
|
+
&& (row.thread === null || row.thread === this.thread);
|
|
62
|
+
}
|
|
63
|
+
/** The internal id for an app-supplied key at this scope (identical to the PHP reference). */
|
|
64
|
+
qualify(kind, key) {
|
|
65
|
+
return digest(`prism-graph/${kind}@1`, [this.tenant, this.owner ?? '', this.agent ?? '', this.thread ?? '', key]);
|
|
66
|
+
}
|
|
67
|
+
toRow() {
|
|
68
|
+
return [this.tenant, this.owner, this.agent, this.thread];
|
|
69
|
+
}
|
|
70
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import type { DatabaseSync } from 'node:sqlite';
|
|
2
|
+
export type SqlValue = string | number | bigint | null | Uint8Array;
|
|
3
|
+
export type Row = Record<string, unknown>;
|
|
4
|
+
/**
|
|
5
|
+
* The SQL the stores need, and nothing else. Every value is a bound
|
|
6
|
+
* parameter ("?" placeholders); table names come only from a validated
|
|
7
|
+
* TableMap.
|
|
8
|
+
*
|
|
9
|
+
* `atomic()` is all-or-nothing INCLUDING inside a caller's own transaction:
|
|
10
|
+
* the work runs in a SAVEPOINT, so a refusal or failure rolls back exactly
|
|
11
|
+
* prism-graph's work and leaves the caller's transaction usable. (The PHP
|
|
12
|
+
* reference had to learn this the hard way: see its 0.2.1 notes.)
|
|
13
|
+
*/
|
|
14
|
+
export interface SqlDriver {
|
|
15
|
+
readonly dialect: 'sqlite';
|
|
16
|
+
run(sql: string, params?: readonly SqlValue[]): Promise<number>;
|
|
17
|
+
all(sql: string, params?: readonly SqlValue[]): Promise<Row[]>;
|
|
18
|
+
exec(sql: string): Promise<void>;
|
|
19
|
+
atomic<T>(work: () => Promise<T>): Promise<T>;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* SQLite through Node's built-in node:sqlite (Node 22.16+): no native module,
|
|
23
|
+
* no third-party dependency. Needs SQLite with FTS5, which node:sqlite has.
|
|
24
|
+
*
|
|
25
|
+
* GIVE IT A CONNECTION OF ITS OWN, or route every write on that connection
|
|
26
|
+
* through `atomic()`. SQLite transactions belong to the connection: a write
|
|
27
|
+
* another part of the app makes while one of our blocks is open lands inside
|
|
28
|
+
* our savepoint, and is rolled back with it if our block fails.
|
|
29
|
+
*
|
|
30
|
+
* Atomic blocks are serialised on the connection. A block opened inside
|
|
31
|
+
* another (the same async call chain) queues inside its parent; a callback
|
|
32
|
+
* that outlives its block (a timer, an un-awaited promise) is treated as a
|
|
33
|
+
* new caller, never as part of a block that has already closed.
|
|
34
|
+
*/
|
|
35
|
+
export declare class NodeSqliteDriver implements SqlDriver {
|
|
36
|
+
private readonly db;
|
|
37
|
+
readonly dialect: "sqlite";
|
|
38
|
+
private queue;
|
|
39
|
+
/** Only ever increases: savepoint names are unique for the life of the driver. */
|
|
40
|
+
private counter;
|
|
41
|
+
private readonly inside;
|
|
42
|
+
constructor(db: DatabaseSync);
|
|
43
|
+
run(sql: string, params?: readonly SqlValue[]): Promise<number>;
|
|
44
|
+
all(sql: string, params?: readonly SqlValue[]): Promise<Row[]>;
|
|
45
|
+
exec(sql: string): Promise<void>;
|
|
46
|
+
atomic<T>(work: () => Promise<T>): Promise<T>;
|
|
47
|
+
private savepoint;
|
|
48
|
+
private rollback;
|
|
49
|
+
}
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
2
|
+
import { GraphError } from '../errors.js';
|
|
3
|
+
/**
|
|
4
|
+
* SQLite through Node's built-in node:sqlite (Node 22.16+): no native module,
|
|
5
|
+
* no third-party dependency. Needs SQLite with FTS5, which node:sqlite has.
|
|
6
|
+
*
|
|
7
|
+
* GIVE IT A CONNECTION OF ITS OWN, or route every write on that connection
|
|
8
|
+
* through `atomic()`. SQLite transactions belong to the connection: a write
|
|
9
|
+
* another part of the app makes while one of our blocks is open lands inside
|
|
10
|
+
* our savepoint, and is rolled back with it if our block fails.
|
|
11
|
+
*
|
|
12
|
+
* Atomic blocks are serialised on the connection. A block opened inside
|
|
13
|
+
* another (the same async call chain) queues inside its parent; a callback
|
|
14
|
+
* that outlives its block (a timer, an un-awaited promise) is treated as a
|
|
15
|
+
* new caller, never as part of a block that has already closed.
|
|
16
|
+
*/
|
|
17
|
+
export class NodeSqliteDriver {
|
|
18
|
+
db;
|
|
19
|
+
dialect = 'sqlite';
|
|
20
|
+
queue = Promise.resolve();
|
|
21
|
+
/** Only ever increases: savepoint names are unique for the life of the driver. */
|
|
22
|
+
counter = 0;
|
|
23
|
+
inside = new AsyncLocalStorage();
|
|
24
|
+
constructor(db) {
|
|
25
|
+
this.db = db;
|
|
26
|
+
if (typeof db.isTransaction !== 'boolean') {
|
|
27
|
+
// Without it, a failed commit cannot be told apart from a caller's own
|
|
28
|
+
// transaction, and a stuck connection would silently lose later writes.
|
|
29
|
+
throw new GraphError('prism-graph needs node:sqlite with DatabaseSync.isTransaction (Node 22.16+).');
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
async run(sql, params = []) {
|
|
33
|
+
return Number(this.db.prepare(sql).run(...params.map(bind)).changes);
|
|
34
|
+
}
|
|
35
|
+
async all(sql, params = []) {
|
|
36
|
+
return this.db.prepare(sql).all(...params.map(bind));
|
|
37
|
+
}
|
|
38
|
+
async exec(sql) {
|
|
39
|
+
this.db.exec(sql);
|
|
40
|
+
}
|
|
41
|
+
atomic(work) {
|
|
42
|
+
const parent = this.inside.getStore();
|
|
43
|
+
if (parent?.open === true) {
|
|
44
|
+
// Nested in an open block: run inside it, one nested block at a time.
|
|
45
|
+
const result = parent.queue.then(() => this.savepoint(work, false));
|
|
46
|
+
parent.queue = result.catch(() => undefined);
|
|
47
|
+
return result;
|
|
48
|
+
}
|
|
49
|
+
const result = this.queue.then(() => this.savepoint(work, true));
|
|
50
|
+
this.queue = result.catch(() => undefined);
|
|
51
|
+
return result;
|
|
52
|
+
}
|
|
53
|
+
async savepoint(work, outermost) {
|
|
54
|
+
// In SQLite a SAVEPOINT outside a transaction opens one, and RELEASE
|
|
55
|
+
// commits it; inside the caller's transaction it nests. One rule for both.
|
|
56
|
+
// Remember whether THIS block opened the transaction: only then may a
|
|
57
|
+
// failure close it with a full ROLLBACK.
|
|
58
|
+
const opened = outermost && !this.db.isTransaction;
|
|
59
|
+
const name = `prism_graph_sp_${++this.counter}`;
|
|
60
|
+
this.db.exec(`SAVEPOINT ${name}`);
|
|
61
|
+
const block = { queue: Promise.resolve(), open: true };
|
|
62
|
+
let result;
|
|
63
|
+
try {
|
|
64
|
+
result = await this.inside.run(block, work);
|
|
65
|
+
await block.queue; // nested blocks the work started but did not await
|
|
66
|
+
}
|
|
67
|
+
catch (error) {
|
|
68
|
+
block.open = false;
|
|
69
|
+
this.rollback(name, opened);
|
|
70
|
+
throw error;
|
|
71
|
+
}
|
|
72
|
+
block.open = false;
|
|
73
|
+
try {
|
|
74
|
+
this.db.exec(`RELEASE SAVEPOINT ${name}`);
|
|
75
|
+
}
|
|
76
|
+
catch (error) {
|
|
77
|
+
// The commit itself failed (e.g. SQLITE_BUSY on the outermost RELEASE):
|
|
78
|
+
// undo our work and never leave the connection inside a transaction,
|
|
79
|
+
// where every later write would report success and be lost.
|
|
80
|
+
this.rollback(name, opened);
|
|
81
|
+
throw error;
|
|
82
|
+
}
|
|
83
|
+
return result;
|
|
84
|
+
}
|
|
85
|
+
rollback(name, opened) {
|
|
86
|
+
try {
|
|
87
|
+
this.db.exec(`ROLLBACK TO SAVEPOINT ${name}`);
|
|
88
|
+
this.db.exec(`RELEASE SAVEPOINT ${name}`);
|
|
89
|
+
}
|
|
90
|
+
catch {
|
|
91
|
+
// Fall through: if we opened the transaction, close it below.
|
|
92
|
+
}
|
|
93
|
+
if (opened && this.db.isTransaction) {
|
|
94
|
+
this.db.exec('ROLLBACK');
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
function bind(value) {
|
|
99
|
+
return value === undefined ? null : value;
|
|
100
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which app-owned tables play each role. Apps create and extend these tables
|
|
3
|
+
* in their own migrations; the package only reads and writes the core columns
|
|
4
|
+
* it documents. Names are capped at 50 characters so derived names stay inside
|
|
5
|
+
* PostgreSQL's 63-character identifier limit.
|
|
6
|
+
*/
|
|
7
|
+
export declare class TableMap {
|
|
8
|
+
readonly sources: string;
|
|
9
|
+
readonly chunks: string;
|
|
10
|
+
readonly facts: string;
|
|
11
|
+
readonly evidence: string;
|
|
12
|
+
constructor(sources?: string, chunks?: string, facts?: string, evidence?: string);
|
|
13
|
+
/** The full-text index over chunks (an FTS5 table on SQLite). */
|
|
14
|
+
fts(): string;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* The core schema, as plain SQL for SQLite, matching the PHP reference's
|
|
18
|
+
* BaseSchema. The package never creates tables at runtime: apps run these from
|
|
19
|
+
* their own migration tooling, and own them afterwards.
|
|
20
|
+
*/
|
|
21
|
+
export declare function baseSchema(t?: TableMap): string[];
|
|
22
|
+
/** External-content FTS5 kept in step by triggers; the text lives once, in the chunks table. */
|
|
23
|
+
export declare function fullTextSchema(t?: TableMap): string[];
|
|
24
|
+
/**
|
|
25
|
+
* Restore the full-text index after a migration that rebuilt the chunks table
|
|
26
|
+
* (which drops its triggers), and rebuild it from the stored chunks. Safe to
|
|
27
|
+
* run at any time.
|
|
28
|
+
*/
|
|
29
|
+
export declare function repairFullTextSchema(t?: TableMap): string[];
|
|
30
|
+
/** The optional vector tables (exact cosine search over packed float32). */
|
|
31
|
+
export declare function vectorSchema(indexes?: string, vectors?: string): string[];
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { InvalidArgumentError } from '../errors.js';
|
|
2
|
+
/**
|
|
3
|
+
* Which app-owned tables play each role. Apps create and extend these tables
|
|
4
|
+
* in their own migrations; the package only reads and writes the core columns
|
|
5
|
+
* it documents. Names are capped at 50 characters so derived names stay inside
|
|
6
|
+
* PostgreSQL's 63-character identifier limit.
|
|
7
|
+
*/
|
|
8
|
+
export class TableMap {
|
|
9
|
+
sources;
|
|
10
|
+
chunks;
|
|
11
|
+
facts;
|
|
12
|
+
evidence;
|
|
13
|
+
constructor(sources = 'graph_sources', chunks = 'graph_chunks', facts = 'graph_facts', evidence = 'graph_fact_evidence') {
|
|
14
|
+
this.sources = sources;
|
|
15
|
+
this.chunks = chunks;
|
|
16
|
+
this.facts = facts;
|
|
17
|
+
this.evidence = evidence;
|
|
18
|
+
for (const [role, table] of Object.entries({ sources, chunks, facts, evidence })) {
|
|
19
|
+
if (!/^[A-Za-z_][A-Za-z0-9_]{0,49}$/.test(table)) {
|
|
20
|
+
throw new InvalidArgumentError(`Invalid table name for ${role}.`);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
Object.freeze(this);
|
|
24
|
+
}
|
|
25
|
+
/** The full-text index over chunks (an FTS5 table on SQLite). */
|
|
26
|
+
fts() {
|
|
27
|
+
return `${this.chunks}_fts`;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* The core schema, as plain SQL for SQLite, matching the PHP reference's
|
|
32
|
+
* BaseSchema. The package never creates tables at runtime: apps run these from
|
|
33
|
+
* their own migration tooling, and own them afterwards.
|
|
34
|
+
*/
|
|
35
|
+
export function baseSchema(t = new TableMap()) {
|
|
36
|
+
const scope = 'tenant_id VARCHAR(191) NOT NULL, owner_id VARCHAR(191), agent_id VARCHAR(191), thread_id VARCHAR(191)';
|
|
37
|
+
return [
|
|
38
|
+
`CREATE TABLE ${t.sources} (id CHAR(64) PRIMARY KEY, source_key VARCHAR(191) NOT NULL, ${scope},
|
|
39
|
+
uri TEXT NOT NULL, sha256 CHAR(64) NOT NULL, metadata TEXT NOT NULL,
|
|
40
|
+
trust VARCHAR(20) NOT NULL, writer VARCHAR(191) NOT NULL,
|
|
41
|
+
occurred_at CHAR(27), recorded_at CHAR(27) NOT NULL)`,
|
|
42
|
+
`CREATE INDEX ${t.sources}_scope ON ${t.sources} (tenant_id, owner_id, agent_id, thread_id)`,
|
|
43
|
+
// seq is the explicit integer key the full-text index joins on: an implicit
|
|
44
|
+
// rowid is not guaranteed stable across VACUUM.
|
|
45
|
+
`CREATE TABLE ${t.chunks} (seq INTEGER PRIMARY KEY, id CHAR(64) NOT NULL UNIQUE, source_id CHAR(64) NOT NULL REFERENCES ${t.sources} (id),
|
|
46
|
+
text TEXT NOT NULL, byte_start INTEGER NOT NULL, byte_end INTEGER NOT NULL,
|
|
47
|
+
line_start INTEGER NOT NULL, line_end INTEGER NOT NULL, heading_path TEXT NOT NULL,
|
|
48
|
+
recipe VARCHAR(191) NOT NULL, retired_at CHAR(27))`,
|
|
49
|
+
`CREATE INDEX ${t.chunks}_source ON ${t.chunks} (source_id, byte_start)`,
|
|
50
|
+
`CREATE TABLE ${t.facts} (id CHAR(64) PRIMARY KEY, fact_key VARCHAR(191) NOT NULL, ${scope},
|
|
51
|
+
subject VARCHAR(191) NOT NULL, predicate VARCHAR(191) NOT NULL, object TEXT NOT NULL,
|
|
52
|
+
status VARCHAR(20) NOT NULL, valid_from CHAR(27) NOT NULL, valid_to CHAR(27),
|
|
53
|
+
superseded_by VARCHAR(191), recorded_at CHAR(27) NOT NULL, expired_at CHAR(27))`,
|
|
54
|
+
`CREATE INDEX ${t.facts}_subject ON ${t.facts} (tenant_id, subject, valid_to)`,
|
|
55
|
+
`CREATE TABLE ${t.evidence} (fact_id CHAR(64) NOT NULL REFERENCES ${t.facts} (id),
|
|
56
|
+
chunk_id CHAR(64) NOT NULL REFERENCES ${t.chunks} (id), PRIMARY KEY (fact_id, chunk_id))`,
|
|
57
|
+
`CREATE INDEX ${t.evidence}_chunk ON ${t.evidence} (chunk_id)`,
|
|
58
|
+
...fullTextSchema(t),
|
|
59
|
+
];
|
|
60
|
+
}
|
|
61
|
+
/** External-content FTS5 kept in step by triggers; the text lives once, in the chunks table. */
|
|
62
|
+
export function fullTextSchema(t = new TableMap()) {
|
|
63
|
+
const fts = t.fts();
|
|
64
|
+
return [
|
|
65
|
+
`CREATE VIRTUAL TABLE ${fts} USING fts5(text, content='${t.chunks}', content_rowid='seq', tokenize='unicode61')`,
|
|
66
|
+
`CREATE TRIGGER ${t.chunks}_ai AFTER INSERT ON ${t.chunks} BEGIN
|
|
67
|
+
INSERT INTO ${fts} (rowid, text) VALUES (new.seq, new.text); END`,
|
|
68
|
+
`CREATE TRIGGER ${t.chunks}_ad AFTER DELETE ON ${t.chunks} BEGIN
|
|
69
|
+
INSERT INTO ${fts} (${fts}, rowid, text) VALUES ('delete', old.seq, old.text); END`,
|
|
70
|
+
`CREATE TRIGGER ${t.chunks}_au AFTER UPDATE OF text ON ${t.chunks} BEGIN
|
|
71
|
+
INSERT INTO ${fts} (${fts}, rowid, text) VALUES ('delete', old.seq, old.text);
|
|
72
|
+
INSERT INTO ${fts} (rowid, text) VALUES (new.seq, new.text); END`,
|
|
73
|
+
];
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Restore the full-text index after a migration that rebuilt the chunks table
|
|
77
|
+
* (which drops its triggers), and rebuild it from the stored chunks. Safe to
|
|
78
|
+
* run at any time.
|
|
79
|
+
*/
|
|
80
|
+
export function repairFullTextSchema(t = new TableMap()) {
|
|
81
|
+
const fts = t.fts();
|
|
82
|
+
const [virtual, insert, del, update] = fullTextSchema(t);
|
|
83
|
+
return [
|
|
84
|
+
virtual.replace('CREATE VIRTUAL TABLE', 'CREATE VIRTUAL TABLE IF NOT EXISTS'),
|
|
85
|
+
`DROP TRIGGER IF EXISTS ${t.chunks}_ai`,
|
|
86
|
+
`DROP TRIGGER IF EXISTS ${t.chunks}_ad`,
|
|
87
|
+
`DROP TRIGGER IF EXISTS ${t.chunks}_au`,
|
|
88
|
+
insert,
|
|
89
|
+
del,
|
|
90
|
+
update,
|
|
91
|
+
`INSERT INTO ${fts} (${fts}) VALUES ('rebuild')`,
|
|
92
|
+
];
|
|
93
|
+
}
|
|
94
|
+
/** The optional vector tables (exact cosine search over packed float32). */
|
|
95
|
+
export function vectorSchema(indexes = 'graph_vector_indexes', vectors = 'graph_vectors') {
|
|
96
|
+
return [
|
|
97
|
+
`CREATE TABLE ${indexes} (name VARCHAR(100) PRIMARY KEY, fingerprint CHAR(64) NOT NULL, provider VARCHAR(100) NOT NULL,
|
|
98
|
+
model VARCHAR(191) NOT NULL, dimensions INTEGER NOT NULL, recipe VARCHAR(191) NOT NULL)`,
|
|
99
|
+
`CREATE TABLE ${vectors} (index_name VARCHAR(100) NOT NULL REFERENCES ${indexes} (name), chunk_id CHAR(64) NOT NULL,
|
|
100
|
+
vector BLOB NOT NULL, PRIMARY KEY (index_name, chunk_id))`,
|
|
101
|
+
];
|
|
102
|
+
}
|