@loomwork/plugin-sqlite 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/LICENSE +202 -0
  2. package/NOTICE +8 -0
  3. package/README.md +62 -2
  4. package/dist/SqliteArchiveStore.d.ts +22 -0
  5. package/dist/SqliteArchiveStore.d.ts.map +1 -0
  6. package/dist/SqliteArchiveStore.js +129 -0
  7. package/dist/SqliteArchiveStore.js.map +1 -0
  8. package/dist/SqliteBranchRegistry.d.ts +26 -0
  9. package/dist/SqliteBranchRegistry.d.ts.map +1 -0
  10. package/dist/SqliteBranchRegistry.js +154 -0
  11. package/dist/SqliteBranchRegistry.js.map +1 -0
  12. package/dist/SqliteContentStore.d.ts +55 -0
  13. package/dist/SqliteContentStore.d.ts.map +1 -0
  14. package/dist/SqliteContentStore.js +150 -0
  15. package/dist/SqliteContentStore.js.map +1 -0
  16. package/dist/SqliteDurableStateStore.d.ts +26 -0
  17. package/dist/SqliteDurableStateStore.d.ts.map +1 -0
  18. package/dist/SqliteDurableStateStore.js +81 -0
  19. package/dist/SqliteDurableStateStore.js.map +1 -0
  20. package/dist/SqliteEventStore.d.ts +55 -0
  21. package/dist/SqliteEventStore.d.ts.map +1 -0
  22. package/dist/SqliteEventStore.js +297 -0
  23. package/dist/SqliteEventStore.js.map +1 -0
  24. package/dist/SqliteLeaseStore.d.ts +25 -0
  25. package/dist/SqliteLeaseStore.d.ts.map +1 -0
  26. package/dist/SqliteLeaseStore.js +107 -0
  27. package/dist/SqliteLeaseStore.js.map +1 -0
  28. package/dist/SqliteMemoryStore.d.ts +52 -0
  29. package/dist/SqliteMemoryStore.d.ts.map +1 -0
  30. package/dist/SqliteMemoryStore.js +149 -0
  31. package/dist/SqliteMemoryStore.js.map +1 -0
  32. package/dist/SqliteReadModelStore.d.ts +38 -0
  33. package/dist/SqliteReadModelStore.d.ts.map +1 -0
  34. package/dist/SqliteReadModelStore.js +292 -0
  35. package/dist/SqliteReadModelStore.js.map +1 -0
  36. package/dist/SqliteSnapshotStore.d.ts +31 -0
  37. package/dist/SqliteSnapshotStore.d.ts.map +1 -0
  38. package/dist/SqliteSnapshotStore.js +101 -0
  39. package/dist/SqliteSnapshotStore.js.map +1 -0
  40. package/dist/SqliteTimerStore.d.ts +19 -0
  41. package/dist/SqliteTimerStore.d.ts.map +1 -0
  42. package/dist/SqliteTimerStore.js +106 -0
  43. package/dist/SqliteTimerStore.js.map +1 -0
  44. package/dist/index.d.ts +13 -0
  45. package/dist/index.d.ts.map +1 -0
  46. package/dist/index.js +26 -0
  47. package/dist/index.js.map +1 -0
  48. package/dist/preset.d.ts +31 -0
  49. package/dist/preset.d.ts.map +1 -0
  50. package/dist/preset.js +51 -0
  51. package/dist/preset.js.map +1 -0
  52. package/dist/schema.d.ts +3 -0
  53. package/dist/schema.d.ts.map +1 -0
  54. package/dist/schema.js +103 -0
  55. package/dist/schema.js.map +1 -0
  56. package/dist/snapshotSchema.d.ts +3 -0
  57. package/dist/snapshotSchema.d.ts.map +1 -0
  58. package/dist/snapshotSchema.js +19 -0
  59. package/dist/snapshotSchema.js.map +1 -0
  60. package/package.json +62 -3
  61. package/src/SqliteArchiveStore.ts +189 -0
  62. package/src/SqliteBranchRegistry.ts +199 -0
  63. package/src/SqliteContentStore.ts +195 -0
  64. package/src/SqliteDurableStateStore.ts +101 -0
  65. package/src/SqliteEventStore.ts +414 -0
  66. package/src/SqliteLeaseStore.ts +165 -0
  67. package/src/SqliteMemoryStore.ts +160 -0
  68. package/src/SqliteReadModelStore.ts +378 -0
  69. package/src/SqliteSnapshotStore.ts +157 -0
  70. package/src/SqliteTimerStore.ts +124 -0
  71. package/src/index.ts +13 -0
  72. package/src/preset.ts +66 -0
  73. package/src/schema.ts +100 -0
  74. package/src/snapshotSchema.ts +17 -0
@@ -0,0 +1,160 @@
1
+ import type { MemoryEntry, MemorySearchOptions, MemoryStore } from "@loomwork/memory";
2
+ import { clampSearchLimit, queryTerms, searchableText } from "@loomwork/memory";
3
+ import Database from "better-sqlite3";
4
+
5
+ interface EntryRow {
6
+ entry: string;
7
+ }
8
+
9
+ /**
10
+ * SQLite-backed {@link MemoryStore} — the zero-server durable agent memory. Point it
11
+ * at the same DB file as the SqliteEventStore so one backup captures events + memory.
12
+ * Own idempotent migration, same pattern as SqliteTimerStore/SqliteDurableStateStore.
13
+ *
14
+ * A key names an ordered list, so the table needs an insertion key of its own: entries
15
+ * routinely share a timestamp and `ORDER BY timestamp` would reshuffle a conversation.
16
+ * `AUTOINCREMENT` (not bare rowid) makes the monotonicity a schema-level guarantee
17
+ * rather than an argument about rowid reuse after a `clear`.
18
+ *
19
+ * `replace` is one transaction, which is the whole reason it is on the SPI: MemoryRuntime
20
+ * rewrites a list on every TTL/maxEntries eviction, and a crash between the delete and the
21
+ * re-insert would make routine eviction the thing that loses the scope.
22
+ *
23
+ * `remove` (ADR 0091) is one indexed `DELETE` matching `json_extract(entry, '$.id')` — the
24
+ * id is read from the stored JSON rather than duplicated into a column, so nothing can
25
+ * drift between what `read` returns and what a delete matches, and a database written
26
+ * before this shipped needs no migration.
27
+ *
28
+ * `search` rides an FTS5 index over `searchableText(entry)`, held in an EXTERNAL-CONTENT
29
+ * table so the text is indexed but not stored twice, and synchronised by triggers so a
30
+ * `clear`/`replace` cannot leave the index describing entries that no longer exist. FTS5's
31
+ * `rank` is a BM25 score where LOWER is better, hence `ORDER BY rank` ascending.
32
+ */
33
+ export class SqliteMemoryStore implements MemoryStore {
34
+ private readonly db: Database.Database;
35
+ private readonly appendStmt: Database.Statement;
36
+ private readonly readStmt: Database.Statement;
37
+ private readonly clearStmt: Database.Statement;
38
+ private readonly removeStmt: Database.Statement;
39
+ private readonly searchStmt: Database.Statement;
40
+ private readonly replaceTxn: (key: string, entries: readonly MemoryEntry[]) => void;
41
+
42
+ constructor(dbPath: string) {
43
+ this.db = new Database(dbPath);
44
+ try {
45
+ this.db.pragma("journal_mode = WAL");
46
+ this.db.pragma("synchronous = NORMAL");
47
+ this.db.exec(`
48
+ CREATE TABLE IF NOT EXISTS memory_entries (
49
+ seq INTEGER PRIMARY KEY AUTOINCREMENT,
50
+ key TEXT NOT NULL,
51
+ entry TEXT NOT NULL,
52
+ text TEXT NOT NULL
53
+ );
54
+ CREATE INDEX IF NOT EXISTS idx_memory_entries_key
55
+ ON memory_entries (key, seq ASC);
56
+ -- Removal by entry id (ADR 0091). An EXPRESSION index over the stored JSON, not
57
+ -- a column: the entry is already persisted whole, so an id round-trips with no
58
+ -- schema change and no migration of rows written before this shipped -- and a
59
+ -- column would be a second copy able to disagree with the one a read returns.
60
+ CREATE INDEX IF NOT EXISTS idx_memory_entries_id
61
+ ON memory_entries (key, json_extract(entry, '$.id'))
62
+ WHERE json_extract(entry, '$.id') IS NOT NULL;
63
+
64
+ CREATE VIRTUAL TABLE IF NOT EXISTS memory_fts USING fts5(
65
+ text, content='memory_entries', content_rowid='seq'
66
+ );
67
+ -- External content: the index holds no copy of the row, so it must be told about
68
+ -- every insert and delete. Entries are never updated, so there is no update trigger.
69
+ CREATE TRIGGER IF NOT EXISTS memory_entries_ai AFTER INSERT ON memory_entries BEGIN
70
+ INSERT INTO memory_fts(rowid, text) VALUES (new.seq, new.text);
71
+ END;
72
+ CREATE TRIGGER IF NOT EXISTS memory_entries_ad AFTER DELETE ON memory_entries BEGIN
73
+ INSERT INTO memory_fts(memory_fts, rowid, text) VALUES ('delete', old.seq, old.text);
74
+ END;
75
+ `);
76
+ this.appendStmt = this.db.prepare(
77
+ "INSERT INTO memory_entries (key, entry, text) VALUES (?, ?, ?)"
78
+ );
79
+ this.readStmt = this.db.prepare(
80
+ "SELECT entry FROM memory_entries WHERE key = ? ORDER BY seq ASC"
81
+ );
82
+ this.clearStmt = this.db.prepare("DELETE FROM memory_entries WHERE key = ?");
83
+ // `json_each` over ONE json-array parameter, not a generated `IN (?,?,?…)`: a
84
+ // document with 200 chunks would otherwise be a 200-placeholder statement that has
85
+ // never been prepared before, and better-sqlite3 caches by SQL text.
86
+ this.removeStmt = this.db.prepare(
87
+ `DELETE FROM memory_entries
88
+ WHERE key = ?
89
+ AND json_extract(entry, '$.id') IN (SELECT value FROM json_each(?))`
90
+ );
91
+ this.searchStmt = this.db.prepare(
92
+ `SELECT e.entry FROM memory_fts
93
+ JOIN memory_entries e ON e.seq = memory_fts.rowid
94
+ WHERE memory_fts MATCH ? AND e.key = ?
95
+ ORDER BY memory_fts.rank
96
+ LIMIT ?`
97
+ );
98
+ this.replaceTxn = this.db.transaction((key: string, entries: readonly MemoryEntry[]) => {
99
+ this.clearStmt.run(key);
100
+ for (const entry of entries)
101
+ this.appendStmt.run(key, JSON.stringify(entry), searchableText(entry));
102
+ });
103
+ } catch (err) {
104
+ this.db.close(); // don't leak the file handle if setup fails on a bad/locked file
105
+ throw err;
106
+ }
107
+ }
108
+
109
+ async append(key: string, entry: MemoryEntry): Promise<void> {
110
+ this.appendStmt.run(key, JSON.stringify(entry), searchableText(entry));
111
+ }
112
+
113
+ async read(key: string): Promise<MemoryEntry[]> {
114
+ const rows = this.readStmt.all(key) as EntryRow[];
115
+ return rows.map((row) => JSON.parse(row.entry) as MemoryEntry);
116
+ }
117
+
118
+ async clear(key: string): Promise<void> {
119
+ this.clearStmt.run(key);
120
+ }
121
+
122
+ async replace(key: string, entries: readonly MemoryEntry[]): Promise<void> {
123
+ this.replaceTxn(key, entries);
124
+ }
125
+
126
+ /**
127
+ * Delete by entry id. The FTS index needs no special handling: rows are deleted, so the
128
+ * existing AFTER DELETE trigger takes them out of `memory_fts` — which is the reason
129
+ * this is a row delete and not a rewrite of the `entry` JSON.
130
+ *
131
+ * An entry with no id has `json_extract(...) IS NULL`, and `NULL IN (...)` is NULL —
132
+ * never true — so anonymous entries cannot be removed by any list of ids, including one
133
+ * containing the strings "null" or "undefined".
134
+ */
135
+ async remove(key: string, ids: readonly string[]): Promise<void> {
136
+ if (ids.length === 0) return;
137
+ this.removeStmt.run(key, JSON.stringify(ids));
138
+ }
139
+
140
+ async search(key: string, query: string, opts: MemorySearchOptions = {}): Promise<MemoryEntry[]> {
141
+ const terms = queryTerms(query);
142
+ if (terms.length === 0) return [];
143
+ // `queryTerms`, not `tokenize`: OR-ing the same term 20,000 times cannot change which
144
+ // rows match and took 32.7 SECONDS here before it was deduplicated.
145
+ //
146
+ // Building the MATCH from TERMS, never from the raw question, is what keeps FTS5's
147
+ // expression syntax (`*`, `NEAR`, `AND`, a bare `-`) out of it: a user question passed
148
+ // through is parsed as operators, and either raises a syntax error or quietly means
149
+ // something else. `tokenize` already yields lowercase alphanumerics — which cannot be
150
+ // FTS5 keywords, those being uppercase — and the quoting below is the second belt, so
151
+ // that widening what counts as a term later cannot reopen this.
152
+ const match = terms.map((term) => `"${term}"`).join(" OR ");
153
+ const rows = this.searchStmt.all(match, key, clampSearchLimit(opts.limit)) as EntryRow[];
154
+ return rows.map((row) => JSON.parse(row.entry) as MemoryEntry);
155
+ }
156
+
157
+ async close(): Promise<void> {
158
+ this.db.close();
159
+ }
160
+ }
@@ -0,0 +1,378 @@
1
+ import type {
2
+ FoldedApply,
3
+ FoldedApplyResult,
4
+ FoldedRecord,
5
+ FoldedSelector,
6
+ ReadModelPage,
7
+ ReadModelQuery,
8
+ ReadModelRow,
9
+ ReadModelScalar,
10
+ ReadModelStore,
11
+ } from "@loomwork/plugin-read-model";
12
+ import {
13
+ assertFoldedSelector,
14
+ assertManyKeys,
15
+ assertPurgeFilter,
16
+ clampQueryLimit,
17
+ foldedSelectorColumn,
18
+ parseQueryCursor,
19
+ } from "@loomwork/plugin-read-model";
20
+ import Database from "better-sqlite3";
21
+
22
+ interface RowRecord {
23
+ row_id: string;
24
+ execution_id: string;
25
+ sequence: number;
26
+ timestamp: number;
27
+ kind: string;
28
+ fields: string;
29
+ value: string;
30
+ }
31
+
32
+ interface FoldedDbRow {
33
+ execution_id: string;
34
+ app_key: string | null;
35
+ scope: string | null;
36
+ sequence: number;
37
+ state: string | null;
38
+ erased: number;
39
+ }
40
+
41
+ function toFolded(r: FoldedDbRow): FoldedRecord {
42
+ return {
43
+ executionId: r.execution_id,
44
+ key: r.app_key,
45
+ scope: r.scope,
46
+ sequence: r.sequence,
47
+ state: r.state === null ? null : JSON.parse(r.state),
48
+ erased: r.erased === 1,
49
+ };
50
+ }
51
+
52
+ /**
53
+ * SQLite-backed {@link ReadModelStore} (RM, issue #183). Point it at the same
54
+ * DB file as the SqliteEventStore so one backup captures log + read models.
55
+ * Own idempotent migration, same pattern as SqliteDurableStateStore. Field
56
+ * equality queries use `json_extract` over the JSON `fields` column; keyset
57
+ * pagination uses row-value comparison over (timestamp, sequence, row_id).
58
+ */
59
+ export class SqliteReadModelStore implements ReadModelStore {
60
+ private readonly db: Database.Database;
61
+ // Audit fix (DS&A rank 2): statements are prepared ONCE — apply-heavy
62
+ // catch-ups no longer re-parse identical SQL per call.
63
+ private readonly insertRowStmt: Database.Statement;
64
+ private readonly upsertMarkStmt: Database.Statement;
65
+ private readonly watermarkStmt: Database.Statement;
66
+ private readonly watermarkedStmt: Database.Statement;
67
+ private readonly clearRowsStmt: Database.Statement;
68
+ private readonly clearMarksStmt: Database.Statement;
69
+ private readonly foldHeaderStmt: Database.Statement;
70
+ private readonly upsertFoldHeaderStmt: Database.Statement;
71
+ private readonly dropFoldStatesStmt: Database.Statement;
72
+ private readonly dropFoldHeaderStmt: Database.Statement;
73
+ private readonly applyFoldedStmt: Database.Statement;
74
+
75
+ constructor(dbPath: string) {
76
+ this.db = new Database(dbPath);
77
+ try {
78
+ this.db.pragma("journal_mode = WAL");
79
+ this.db.pragma("synchronous = NORMAL");
80
+ this.db.exec(`
81
+ CREATE TABLE IF NOT EXISTS read_model_rows (
82
+ model_id TEXT NOT NULL,
83
+ row_id TEXT NOT NULL,
84
+ execution_id TEXT NOT NULL,
85
+ sequence INTEGER NOT NULL,
86
+ timestamp INTEGER NOT NULL,
87
+ kind TEXT NOT NULL,
88
+ fields TEXT NOT NULL,
89
+ value TEXT NOT NULL,
90
+ PRIMARY KEY (model_id, row_id)
91
+ );
92
+ CREATE INDEX IF NOT EXISTS idx_rm_rows_order
93
+ ON read_model_rows (model_id, kind, timestamp, sequence);
94
+ CREATE TABLE IF NOT EXISTS read_model_watermarks (
95
+ model_id TEXT NOT NULL,
96
+ execution_id TEXT NOT NULL,
97
+ sequence INTEGER NOT NULL,
98
+ PRIMARY KEY (model_id, execution_id)
99
+ );
100
+ CREATE TABLE IF NOT EXISTS read_model_folds (
101
+ model_id TEXT PRIMARY KEY,
102
+ version TEXT NOT NULL
103
+ );
104
+ CREATE TABLE IF NOT EXISTS read_model_state (
105
+ model_id TEXT NOT NULL,
106
+ execution_id TEXT NOT NULL,
107
+ app_key TEXT,
108
+ scope TEXT,
109
+ sequence INTEGER NOT NULL,
110
+ state TEXT,
111
+ erased INTEGER NOT NULL DEFAULT 0,
112
+ PRIMARY KEY (model_id, execution_id)
113
+ );
114
+ CREATE INDEX IF NOT EXISTS idx_rm_state_key
115
+ ON read_model_state (model_id, app_key) WHERE app_key IS NOT NULL;
116
+ CREATE INDEX IF NOT EXISTS idx_rm_state_scope
117
+ ON read_model_state (model_id, scope) WHERE scope IS NOT NULL;
118
+ `);
119
+ this.insertRowStmt = this.db.prepare(
120
+ `INSERT OR IGNORE INTO read_model_rows
121
+ (model_id, row_id, execution_id, sequence, timestamp, kind, fields, value)
122
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
123
+ );
124
+ this.upsertMarkStmt = this.db.prepare(
125
+ `INSERT INTO read_model_watermarks (model_id, execution_id, sequence)
126
+ VALUES (?, ?, ?)
127
+ ON CONFLICT (model_id, execution_id)
128
+ DO UPDATE SET sequence = MAX(sequence, excluded.sequence)`
129
+ );
130
+ this.watermarkStmt = this.db.prepare(
131
+ "SELECT sequence FROM read_model_watermarks WHERE model_id = ? AND execution_id = ?"
132
+ );
133
+ this.watermarkedStmt = this.db.prepare(
134
+ "SELECT execution_id FROM read_model_watermarks WHERE model_id = ?"
135
+ );
136
+ this.clearRowsStmt = this.db.prepare("DELETE FROM read_model_rows WHERE model_id = ?");
137
+ this.clearMarksStmt = this.db.prepare("DELETE FROM read_model_watermarks WHERE model_id = ?");
138
+ this.foldHeaderStmt = this.db.prepare(
139
+ "SELECT version FROM read_model_folds WHERE model_id = ?"
140
+ );
141
+ this.upsertFoldHeaderStmt = this.db.prepare(
142
+ `INSERT INTO read_model_folds (model_id, version) VALUES (?, ?)
143
+ ON CONFLICT (model_id) DO UPDATE SET version = excluded.version`
144
+ );
145
+ this.dropFoldStatesStmt = this.db.prepare("DELETE FROM read_model_state WHERE model_id = ?");
146
+ this.dropFoldHeaderStmt = this.db.prepare("DELETE FROM read_model_folds WHERE model_id = ?");
147
+ // One statement: header check + monotonic CAS. RETURNING reports the (unchanged)
148
+ // erased flag so advancing a tombstone is "applied-erased" and keeps state NULL.
149
+ this.applyFoldedStmt = this.db.prepare(
150
+ `INSERT INTO read_model_state
151
+ (model_id, execution_id, app_key, scope, sequence, state, erased)
152
+ SELECT ?, ?, ?, ?, ?, ?, 0
153
+ WHERE EXISTS (SELECT 1 FROM read_model_folds WHERE model_id = ? AND version = ?)
154
+ ON CONFLICT (model_id, execution_id) DO UPDATE
155
+ SET sequence = excluded.sequence,
156
+ state = CASE WHEN read_model_state.erased = 1 THEN NULL ELSE excluded.state END,
157
+ app_key = COALESCE(excluded.app_key, read_model_state.app_key),
158
+ scope = COALESCE(excluded.scope, read_model_state.scope)
159
+ WHERE read_model_state.sequence < excluded.sequence
160
+ RETURNING erased`
161
+ );
162
+ } catch (err) {
163
+ this.db.close(); // don't leak the file handle if setup fails on a bad/locked file
164
+ throw err;
165
+ }
166
+ }
167
+
168
+ async apply(
169
+ modelId: string,
170
+ executionId: string,
171
+ upToSequence: number,
172
+ rows: readonly ReadModelRow[]
173
+ ): Promise<void> {
174
+ this.db.transaction(() => {
175
+ for (const row of rows) {
176
+ this.insertRowStmt.run(
177
+ modelId,
178
+ row.rowId,
179
+ row.executionId,
180
+ row.sequence,
181
+ row.timestamp,
182
+ row.kind,
183
+ JSON.stringify(row.fields),
184
+ JSON.stringify(row.value ?? null)
185
+ );
186
+ }
187
+ this.upsertMarkStmt.run(modelId, executionId, upToSequence);
188
+ })();
189
+ }
190
+
191
+ async watermark(modelId: string, executionId: string): Promise<number | null> {
192
+ const row = this.watermarkStmt.get(modelId, executionId) as { sequence: number } | undefined;
193
+ return row?.sequence ?? null;
194
+ }
195
+
196
+ async watermarkedExecutions(modelId: string): Promise<string[]> {
197
+ const rows = this.watermarkedStmt.all(modelId) as { execution_id: string }[];
198
+ return rows.map((r) => r.execution_id);
199
+ }
200
+
201
+ async query(modelId: string, q: ReadModelQuery = {}): Promise<ReadModelPage> {
202
+ const desc = q.order === "desc";
203
+ const limit = clampQueryLimit(q.limit);
204
+ const clauses = ["model_id = ?"];
205
+ const params: unknown[] = [modelId];
206
+ if (q.kind !== undefined) {
207
+ clauses.push("kind = ?");
208
+ params.push(q.kind);
209
+ }
210
+ for (const [field, value] of Object.entries(q.where ?? {})) {
211
+ const path = `$."${field.split('"').join('""')}"`;
212
+ if (value === null) {
213
+ clauses.push("json_extract(fields, ?) IS NULL");
214
+ params.push(path);
215
+ } else {
216
+ clauses.push("json_extract(fields, ?) = ?");
217
+ params.push(path, typeof value === "boolean" ? (value ? 1 : 0) : value);
218
+ }
219
+ }
220
+ if (q.cursor !== undefined) {
221
+ const [ts, seq, rowId] = parseQueryCursor(q.cursor);
222
+ clauses.push(`(timestamp, sequence, row_id) ${desc ? "<" : ">"} (?, ?, ?)`);
223
+ params.push(ts, seq, rowId);
224
+ }
225
+ const dir = desc ? "DESC" : "ASC";
226
+ const records = this.db
227
+ .prepare(
228
+ `SELECT row_id, execution_id, sequence, timestamp, kind, fields, value
229
+ FROM read_model_rows WHERE ${clauses.join(" AND ")}
230
+ ORDER BY timestamp ${dir}, sequence ${dir}, row_id ${dir}
231
+ LIMIT ?`
232
+ )
233
+ .all(...params, limit + 1) as RowRecord[];
234
+
235
+ const page = records.slice(0, limit).map(
236
+ (r): ReadModelRow => ({
237
+ rowId: r.row_id,
238
+ executionId: r.execution_id,
239
+ sequence: r.sequence,
240
+ timestamp: r.timestamp,
241
+ kind: r.kind,
242
+ fields: JSON.parse(r.fields),
243
+ value: JSON.parse(r.value),
244
+ })
245
+ );
246
+ const last = page[page.length - 1];
247
+ return {
248
+ rows: page,
249
+ ...(records.length > limit &&
250
+ last !== undefined && {
251
+ nextCursor: JSON.stringify([last.timestamp, last.sequence, last.rowId]),
252
+ }),
253
+ };
254
+ }
255
+
256
+ async purge(modelId: string, where: Record<string, ReadModelScalar>): Promise<number> {
257
+ assertPurgeFilter(where);
258
+ // Same `json_extract` equality the query path uses, so a purge deletes exactly the
259
+ // rows the equivalent query would have returned. Not prepared once: the clause shape
260
+ // varies with the filter's field set, and an erasure is a lifecycle call, not a hot path.
261
+ const clauses = ["model_id = ?"];
262
+ const params: unknown[] = [modelId];
263
+ for (const [field, value] of Object.entries(where)) {
264
+ const path = `$."${field.split('"').join('""')}"`;
265
+ if (value === null) {
266
+ clauses.push("json_extract(fields, ?) IS NULL");
267
+ params.push(path);
268
+ } else {
269
+ clauses.push("json_extract(fields, ?) = ?");
270
+ params.push(path, typeof value === "boolean" ? (value ? 1 : 0) : value);
271
+ }
272
+ }
273
+ // Watermarks are deliberately left alone (SPI contract) — deleting one would make the
274
+ // next catch-up re-derive the rows just erased.
275
+ const info = this.db
276
+ .prepare(`DELETE FROM read_model_rows WHERE ${clauses.join(" AND ")}`)
277
+ .run(...params);
278
+ return info.changes;
279
+ }
280
+
281
+ async clear(modelId: string): Promise<void> {
282
+ this.db.transaction(() => {
283
+ this.clearRowsStmt.run(modelId);
284
+ this.clearMarksStmt.run(modelId);
285
+ // Folded state (ADR 0132): this model only. Tables are created with the others in
286
+ // the constructor, so clear never has to create anything.
287
+ this.dropFoldStatesStmt.run(modelId);
288
+ this.dropFoldHeaderStmt.run(modelId);
289
+ })();
290
+ }
291
+
292
+ // ── Folded state (ADR 0132 D3, D5, D6, D12) ──────────────────────────────
293
+
294
+ async beginFold(modelId: string, foldVersion: string): Promise<{ rebuilt: boolean }> {
295
+ // IMMEDIATE: read-then-write must take the write lock up front, or a second process
296
+ // in WAL mode fails with SQLITE_BUSY_SNAPSHOT when upgrading a stale read snapshot.
297
+ return this.db
298
+ .transaction(() => {
299
+ const prior = this.foldHeaderStmt.get(modelId) as { version: string } | undefined;
300
+ this.upsertFoldHeaderStmt.run(modelId, foldVersion);
301
+ if (prior === undefined || prior.version === foldVersion) return { rebuilt: false };
302
+ this.dropFoldStatesStmt.run(modelId);
303
+ return { rebuilt: true };
304
+ })
305
+ .immediate();
306
+ }
307
+
308
+ async applyFolded(a: FoldedApply): Promise<FoldedApplyResult> {
309
+ // IMMEDIATE for the same reason: the miss path reads the header after the write attempt.
310
+ return this.db
311
+ .transaction((): FoldedApplyResult => {
312
+ const hit = this.applyFoldedStmt.get(
313
+ a.modelId,
314
+ a.executionId,
315
+ a.key ?? null,
316
+ a.scope ?? null,
317
+ a.upToSequence,
318
+ a.state === null || a.state === undefined ? null : JSON.stringify(a.state),
319
+ a.modelId,
320
+ a.foldVersion
321
+ ) as { erased: number } | undefined;
322
+ if (hit !== undefined) return hit.erased === 1 ? "applied-erased" : "applied";
323
+ const hdr = this.foldHeaderStmt.get(a.modelId) as { version: string } | undefined;
324
+ return hdr?.version === a.foldVersion ? "stale" : "version-mismatch";
325
+ })
326
+ .immediate();
327
+ }
328
+
329
+ async getFolded(modelId: string, executionIds: readonly string[]): Promise<FoldedRecord[]> {
330
+ assertManyKeys(executionIds, "getFolded");
331
+ const ids = [...new Set(executionIds)];
332
+ if (ids.length === 0) return [];
333
+ const rows = this.db
334
+ .prepare(
335
+ `SELECT execution_id, app_key, scope, sequence, state, erased FROM read_model_state
336
+ WHERE model_id = ? AND execution_id IN (${ids.map(() => "?").join(",")})`
337
+ )
338
+ .all(modelId, ...ids) as FoldedDbRow[];
339
+ const byId = new Map(rows.map((r) => [r.execution_id, toFolded(r)]));
340
+ return ids.flatMap((id) => {
341
+ const r = byId.get(id);
342
+ return r === undefined ? [] : [r];
343
+ });
344
+ }
345
+
346
+ async getFoldedByKey(modelId: string, keys: readonly string[]): Promise<FoldedRecord[]> {
347
+ assertManyKeys(keys, "getFoldedByKey");
348
+ const uniq = [...new Set(keys)];
349
+ if (uniq.length === 0) return [];
350
+ // BINARY collation = memcmp over UTF-8 = code-point order (ADR 0132), explicit so a
351
+ // future column collation can never change it.
352
+ const rows = this.db
353
+ .prepare(
354
+ `SELECT execution_id, app_key, scope, sequence, state, erased FROM read_model_state
355
+ WHERE model_id = ? AND app_key IN (${uniq.map(() => "?").join(",")})
356
+ ORDER BY execution_id COLLATE BINARY`
357
+ )
358
+ .all(modelId, ...uniq) as FoldedDbRow[];
359
+ return rows.map(toFolded);
360
+ }
361
+
362
+ async purgeFolded(modelId: string, selector: FoldedSelector): Promise<number> {
363
+ assertFoldedSelector(selector);
364
+ const [col, val] = foldedSelectorColumn(selector);
365
+ // Tombstone, never delete: the row IS the watermark (ADR 0132 D12). `col` is one of
366
+ // three literals from foldedSelectorColumn, never caller input.
367
+ return this.db
368
+ .prepare(
369
+ `UPDATE read_model_state SET state = NULL, erased = 1
370
+ WHERE model_id = ? AND ${col} = ? AND erased = 0`
371
+ )
372
+ .run(modelId, val).changes;
373
+ }
374
+
375
+ async close(): Promise<void> {
376
+ this.db.close();
377
+ }
378
+ }
@@ -0,0 +1,157 @@
1
+ import type { ExecutionState } from "@loomwork/core";
2
+ import {
3
+ type Snapshot,
4
+ type SnapshotCodecName,
5
+ type SnapshotStore,
6
+ assertSnapshotCodecAvailable,
7
+ decodeState,
8
+ encodeState,
9
+ parseSnapshotCodec,
10
+ } from "@loomwork/snapshot-runtime";
11
+ import Database from "better-sqlite3";
12
+ import { migrate } from "./snapshotSchema";
13
+
14
+ interface SnapshotRow {
15
+ id: string;
16
+ execution_id: string;
17
+ workflow_id: string;
18
+ sequence: number;
19
+ // TEXT for a pre-codec row (JSON text); a BLOB (Buffer) for an ADR 0141 codec frame. SQLite is dynamically
20
+ // typed, so the TEXT-declared column holds either; the storage class IS the format discriminator.
21
+ state: string | Buffer;
22
+ created_at: number;
23
+ }
24
+
25
+ export interface SqliteSnapshotStoreOptions {
26
+ /**
27
+ * ADR 0141 - how NEW snapshots are written. `"none"` (default): JSON text, byte-identical to before.
28
+ * `"zstd"`: a BLOB frame (format byte + zstd-compressed JSON) in the same `state` column. Reads accept
29
+ * BOTH whatever this is set to, and nothing is rewritten. Throws at construction on a Node without zstd
30
+ * (>= 22.15). No schema change: SQLite's column affinity keeps a BLOB a BLOB.
31
+ */
32
+ codec?: SnapshotCodecName;
33
+ }
34
+
35
+ function rowToSnapshot(row: SnapshotRow): Snapshot {
36
+ return {
37
+ id: row.id,
38
+ executionId: row.execution_id,
39
+ workflowId: row.workflow_id,
40
+ sequence: row.sequence,
41
+ state: (typeof row.state === "string"
42
+ ? JSON.parse(row.state)
43
+ : decodeState(row.state)) as ExecutionState,
44
+ createdAt: row.created_at,
45
+ };
46
+ }
47
+
48
+ export class SqliteSnapshotStore implements SnapshotStore {
49
+ private readonly db: Database.Database;
50
+ /** ADR 0141 - the codec new snapshots are written with (structural; `ProfilingSnapshotStore` reads it). */
51
+ readonly snapshotCodec: SnapshotCodecName;
52
+
53
+ constructor(dbPath: string, options: SqliteSnapshotStoreOptions = {}) {
54
+ this.snapshotCodec = parseSnapshotCodec(options.codec);
55
+ assertSnapshotCodecAvailable(this.snapshotCodec);
56
+ this.db = new Database(dbPath);
57
+ try {
58
+ this.db.pragma("journal_mode = WAL");
59
+ this.db.pragma("synchronous = NORMAL");
60
+ migrate(this.db);
61
+ } catch (err) {
62
+ this.db.close(); // don't leak the file handle if setup fails on a bad/locked file
63
+ throw err;
64
+ }
65
+ }
66
+
67
+ async save(snapshot: Snapshot): Promise<void> {
68
+ this.db
69
+ .prepare<[string, string, string, number, string | Buffer, number]>(`
70
+ INSERT OR REPLACE INTO snapshots (id, execution_id, workflow_id, sequence, state, created_at)
71
+ VALUES (?, ?, ?, ?, ?, ?)
72
+ `)
73
+ .run(
74
+ snapshot.id,
75
+ snapshot.executionId,
76
+ snapshot.workflowId,
77
+ snapshot.sequence,
78
+ this.snapshotCodec === "none"
79
+ ? JSON.stringify(snapshot.state)
80
+ : encodeState(snapshot.state, this.snapshotCodec),
81
+ snapshot.createdAt
82
+ );
83
+ }
84
+
85
+ async loadLatest(executionId: string): Promise<Snapshot | null> {
86
+ const row = this.db
87
+ .prepare<[string], SnapshotRow>(
88
+ "SELECT * FROM snapshots WHERE execution_id = ? ORDER BY sequence DESC LIMIT 1"
89
+ )
90
+ .get(executionId);
91
+ return row ? rowToSnapshot(row) : null;
92
+ }
93
+
94
+ async loadAt(executionId: string, sequence: number): Promise<Snapshot | null> {
95
+ const row = this.db
96
+ .prepare<[string, number], SnapshotRow>(
97
+ "SELECT * FROM snapshots WHERE execution_id = ? AND sequence = ?"
98
+ )
99
+ .get(executionId, sequence);
100
+ return row ? rowToSnapshot(row) : null;
101
+ }
102
+
103
+ async loadLatestBefore(executionId: string, sequence: number): Promise<Snapshot | null> {
104
+ const row = this.db
105
+ .prepare<[string, number], SnapshotRow>(
106
+ "SELECT * FROM snapshots WHERE execution_id = ? AND sequence <= ? ORDER BY sequence DESC LIMIT 1"
107
+ )
108
+ .get(executionId, sequence);
109
+ return row ? rowToSnapshot(row) : null;
110
+ }
111
+
112
+ async deleteExecution(executionId: string): Promise<void> {
113
+ this.db.prepare<[string]>("DELETE FROM snapshots WHERE execution_id = ?").run(executionId);
114
+ }
115
+
116
+ async listSequences(
117
+ executionId: string,
118
+ range: { readonly after?: number; readonly before?: number } = {}
119
+ ): Promise<number[]> {
120
+ // A bound the caller omitted is simply absent from the WHERE (a sentinel would drop a row at it).
121
+ const where = ["execution_id = ?"];
122
+ const args: Array<string | number> = [executionId];
123
+ if (range.after !== undefined) {
124
+ where.push("sequence > ?");
125
+ args.push(range.after);
126
+ }
127
+ if (range.before !== undefined) {
128
+ where.push("sequence < ?");
129
+ args.push(range.before);
130
+ }
131
+ const rows = this.db
132
+ .prepare<Array<string | number>, { sequence: number }>(
133
+ `SELECT sequence FROM snapshots WHERE ${where.join(" AND ")} ORDER BY sequence`
134
+ )
135
+ .all(...args);
136
+ return rows.map((r) => r.sequence);
137
+ }
138
+
139
+ async pruneSnapshots(
140
+ executionId: string,
141
+ range: { readonly after: number; readonly before: number },
142
+ keep: readonly number[] = []
143
+ ): Promise<number> {
144
+ // One statement. The keep-set is a bound JSON array, not an interpolated list.
145
+ const info = this.db
146
+ .prepare<[string, number, number, string]>(
147
+ `DELETE FROM snapshots WHERE execution_id = ? AND sequence > ? AND sequence < ?
148
+ AND sequence NOT IN (SELECT value FROM json_each(?))`
149
+ )
150
+ .run(executionId, range.after, range.before, JSON.stringify(keep));
151
+ return info.changes;
152
+ }
153
+
154
+ async close(): Promise<void> {
155
+ this.db.close();
156
+ }
157
+ }