@loomwork/plugin-sqlite 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +8 -0
- package/README.md +62 -2
- package/dist/SqliteArchiveStore.d.ts +22 -0
- package/dist/SqliteArchiveStore.d.ts.map +1 -0
- package/dist/SqliteArchiveStore.js +129 -0
- package/dist/SqliteArchiveStore.js.map +1 -0
- package/dist/SqliteBranchRegistry.d.ts +26 -0
- package/dist/SqliteBranchRegistry.d.ts.map +1 -0
- package/dist/SqliteBranchRegistry.js +154 -0
- package/dist/SqliteBranchRegistry.js.map +1 -0
- package/dist/SqliteContentStore.d.ts +55 -0
- package/dist/SqliteContentStore.d.ts.map +1 -0
- package/dist/SqliteContentStore.js +150 -0
- package/dist/SqliteContentStore.js.map +1 -0
- package/dist/SqliteDurableStateStore.d.ts +26 -0
- package/dist/SqliteDurableStateStore.d.ts.map +1 -0
- package/dist/SqliteDurableStateStore.js +81 -0
- package/dist/SqliteDurableStateStore.js.map +1 -0
- package/dist/SqliteEventStore.d.ts +55 -0
- package/dist/SqliteEventStore.d.ts.map +1 -0
- package/dist/SqliteEventStore.js +297 -0
- package/dist/SqliteEventStore.js.map +1 -0
- package/dist/SqliteLeaseStore.d.ts +25 -0
- package/dist/SqliteLeaseStore.d.ts.map +1 -0
- package/dist/SqliteLeaseStore.js +107 -0
- package/dist/SqliteLeaseStore.js.map +1 -0
- package/dist/SqliteMemoryStore.d.ts +52 -0
- package/dist/SqliteMemoryStore.d.ts.map +1 -0
- package/dist/SqliteMemoryStore.js +149 -0
- package/dist/SqliteMemoryStore.js.map +1 -0
- package/dist/SqliteReadModelStore.d.ts +38 -0
- package/dist/SqliteReadModelStore.d.ts.map +1 -0
- package/dist/SqliteReadModelStore.js +292 -0
- package/dist/SqliteReadModelStore.js.map +1 -0
- package/dist/SqliteSnapshotStore.d.ts +31 -0
- package/dist/SqliteSnapshotStore.d.ts.map +1 -0
- package/dist/SqliteSnapshotStore.js +101 -0
- package/dist/SqliteSnapshotStore.js.map +1 -0
- package/dist/SqliteTimerStore.d.ts +19 -0
- package/dist/SqliteTimerStore.d.ts.map +1 -0
- package/dist/SqliteTimerStore.js +106 -0
- package/dist/SqliteTimerStore.js.map +1 -0
- package/dist/index.d.ts +13 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +26 -0
- package/dist/index.js.map +1 -0
- package/dist/preset.d.ts +31 -0
- package/dist/preset.d.ts.map +1 -0
- package/dist/preset.js +51 -0
- package/dist/preset.js.map +1 -0
- package/dist/schema.d.ts +3 -0
- package/dist/schema.d.ts.map +1 -0
- package/dist/schema.js +103 -0
- package/dist/schema.js.map +1 -0
- package/dist/snapshotSchema.d.ts +3 -0
- package/dist/snapshotSchema.d.ts.map +1 -0
- package/dist/snapshotSchema.js +19 -0
- package/dist/snapshotSchema.js.map +1 -0
- package/package.json +62 -3
- package/src/SqliteArchiveStore.ts +189 -0
- package/src/SqliteBranchRegistry.ts +199 -0
- package/src/SqliteContentStore.ts +195 -0
- package/src/SqliteDurableStateStore.ts +101 -0
- package/src/SqliteEventStore.ts +414 -0
- package/src/SqliteLeaseStore.ts +165 -0
- package/src/SqliteMemoryStore.ts +160 -0
- package/src/SqliteReadModelStore.ts +378 -0
- package/src/SqliteSnapshotStore.ts +157 -0
- package/src/SqliteTimerStore.ts +124 -0
- package/src/index.ts +13 -0
- package/src/preset.ts +66 -0
- package/src/schema.ts +100 -0
- package/src/snapshotSchema.ts +17 -0
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
import {
|
|
2
|
+
type ArchiveExecutionPage,
|
|
3
|
+
type ArchiveSegment,
|
|
4
|
+
type ArchiveSegmentHeader,
|
|
5
|
+
type ArchiveStore,
|
|
6
|
+
assertReplaceable,
|
|
7
|
+
classifyAppend,
|
|
8
|
+
} from "@loomwork/event-runtime";
|
|
9
|
+
import Database from "better-sqlite3";
|
|
10
|
+
|
|
11
|
+
// ADR 0138 — the SQLite compaction archive. One row per segment, keyed (execution_id, from_seq).
|
|
12
|
+
// The envelopes (and a sealing wrapper's `sealed` form) are stored as one JSON text; the header
|
|
13
|
+
// fields are columns so `headers()` reads no bodies.
|
|
14
|
+
|
|
15
|
+
interface Row {
|
|
16
|
+
from_seq: number;
|
|
17
|
+
through_seq: number;
|
|
18
|
+
count: number;
|
|
19
|
+
digest: string;
|
|
20
|
+
incarnation: string;
|
|
21
|
+
kind: string | null;
|
|
22
|
+
sender_digest: string | null;
|
|
23
|
+
body: string;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
const kindOf = (k: string | null): Pick<ArchiveSegmentHeader, "kind"> =>
|
|
27
|
+
k === "legacy-checkpoint" ? { kind: "legacy-checkpoint" } : {};
|
|
28
|
+
|
|
29
|
+
const senderOf = (d: string | null): Pick<ArchiveSegmentHeader, "senderDigest"> =>
|
|
30
|
+
d === null ? {} : { senderDigest: d };
|
|
31
|
+
|
|
32
|
+
const toSegment = (r: Row): ArchiveSegment => {
|
|
33
|
+
const body = JSON.parse(r.body) as { envelopes: ArchiveSegment["envelopes"]; sealed?: unknown };
|
|
34
|
+
return {
|
|
35
|
+
from: r.from_seq,
|
|
36
|
+
through: r.through_seq,
|
|
37
|
+
count: r.count,
|
|
38
|
+
digest: r.digest,
|
|
39
|
+
incarnation: r.incarnation,
|
|
40
|
+
...kindOf(r.kind),
|
|
41
|
+
...senderOf(r.sender_digest),
|
|
42
|
+
envelopes: body.envelopes,
|
|
43
|
+
...(body.sealed !== undefined && { sealed: body.sealed }),
|
|
44
|
+
};
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
export class SqliteArchiveStore implements ArchiveStore {
|
|
48
|
+
private readonly db: Database.Database;
|
|
49
|
+
|
|
50
|
+
constructor(dbPath: string) {
|
|
51
|
+
this.db = new Database(dbPath);
|
|
52
|
+
try {
|
|
53
|
+
this.db.pragma("journal_mode = WAL");
|
|
54
|
+
this.db.pragma("synchronous = NORMAL");
|
|
55
|
+
this.db.exec(`
|
|
56
|
+
CREATE TABLE IF NOT EXISTS archive_segments (
|
|
57
|
+
execution_id TEXT NOT NULL,
|
|
58
|
+
from_seq INTEGER NOT NULL,
|
|
59
|
+
through_seq INTEGER NOT NULL,
|
|
60
|
+
count INTEGER NOT NULL,
|
|
61
|
+
digest TEXT NOT NULL,
|
|
62
|
+
incarnation TEXT NOT NULL,
|
|
63
|
+
kind TEXT,
|
|
64
|
+
sender_digest TEXT,
|
|
65
|
+
body TEXT NOT NULL,
|
|
66
|
+
PRIMARY KEY (execution_id, from_seq)
|
|
67
|
+
);
|
|
68
|
+
`);
|
|
69
|
+
// ADR 0139 N7: a table an earlier build created gains the column (CREATE IF NOT EXISTS never does).
|
|
70
|
+
const cols = this.db.prepare("PRAGMA table_info(archive_segments)").all() as Array<{
|
|
71
|
+
name: string;
|
|
72
|
+
}>;
|
|
73
|
+
if (!cols.some((c) => c.name === "sender_digest")) {
|
|
74
|
+
this.db.exec("ALTER TABLE archive_segments ADD COLUMN sender_digest TEXT");
|
|
75
|
+
}
|
|
76
|
+
} catch (err) {
|
|
77
|
+
this.db.close();
|
|
78
|
+
throw err;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
async append(executionId: string, segment: ArchiveSegment): Promise<void> {
|
|
83
|
+
// One transaction: the classify read and the insert see the same rows.
|
|
84
|
+
const tx = this.db.transaction(() => {
|
|
85
|
+
const stored = this.headersSync(executionId);
|
|
86
|
+
if (classifyAppend(executionId, stored, segment) === "noop") return;
|
|
87
|
+
this.insert(executionId, segment);
|
|
88
|
+
});
|
|
89
|
+
tx.immediate();
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
private insert(executionId: string, segment: ArchiveSegment): void {
|
|
93
|
+
this.db
|
|
94
|
+
.prepare(
|
|
95
|
+
"INSERT INTO archive_segments (execution_id, from_seq, through_seq, count, digest, incarnation, kind, sender_digest, body) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"
|
|
96
|
+
)
|
|
97
|
+
.run(
|
|
98
|
+
executionId,
|
|
99
|
+
segment.from,
|
|
100
|
+
segment.through,
|
|
101
|
+
segment.count,
|
|
102
|
+
segment.digest,
|
|
103
|
+
segment.incarnation,
|
|
104
|
+
segment.kind ?? null,
|
|
105
|
+
segment.senderDigest ?? null,
|
|
106
|
+
JSON.stringify({
|
|
107
|
+
envelopes: segment.envelopes,
|
|
108
|
+
...(segment.sealed !== undefined && { sealed: segment.sealed }),
|
|
109
|
+
})
|
|
110
|
+
);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** ADR 0139 N1: one transaction - compare the headers, delete the key's segments, insert the new ones. */
|
|
114
|
+
async replaceSegments(
|
|
115
|
+
executionId: string,
|
|
116
|
+
expected: readonly ArchiveSegmentHeader[],
|
|
117
|
+
next: readonly ArchiveSegment[]
|
|
118
|
+
): Promise<void> {
|
|
119
|
+
const tx = this.db.transaction(() => {
|
|
120
|
+
assertReplaceable(executionId, this.headersSync(executionId), expected, next);
|
|
121
|
+
this.db.prepare("DELETE FROM archive_segments WHERE execution_id = ?").run(executionId);
|
|
122
|
+
for (const seg of next) this.insert(executionId, seg);
|
|
123
|
+
});
|
|
124
|
+
tx.immediate();
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
private headersSync(executionId: string): ArchiveSegmentHeader[] {
|
|
128
|
+
return this.db
|
|
129
|
+
.prepare<[string], Omit<Row, "body">>(
|
|
130
|
+
"SELECT from_seq, through_seq, count, digest, incarnation, kind, sender_digest FROM archive_segments WHERE execution_id = ? ORDER BY from_seq"
|
|
131
|
+
)
|
|
132
|
+
.all(executionId)
|
|
133
|
+
.map((r) => ({
|
|
134
|
+
from: r.from_seq,
|
|
135
|
+
through: r.through_seq,
|
|
136
|
+
count: r.count,
|
|
137
|
+
digest: r.digest,
|
|
138
|
+
incarnation: r.incarnation,
|
|
139
|
+
...kindOf(r.kind),
|
|
140
|
+
...senderOf(r.sender_digest),
|
|
141
|
+
}));
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
async headers(executionId: string): Promise<ArchiveSegmentHeader[]> {
|
|
145
|
+
return this.headersSync(executionId);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
async listExecutions(
|
|
149
|
+
opts: { cursor?: string; limit?: number } = {}
|
|
150
|
+
): Promise<ArchiveExecutionPage> {
|
|
151
|
+
const limit = opts.limit ?? 500;
|
|
152
|
+
// BINARY collation = code point order for well-formed ids (the SPI's order).
|
|
153
|
+
const ids = this.db
|
|
154
|
+
.prepare<[string, number], { execution_id: string }>(
|
|
155
|
+
"SELECT DISTINCT execution_id FROM archive_segments WHERE execution_id > ? ORDER BY execution_id LIMIT ?"
|
|
156
|
+
)
|
|
157
|
+
.all(opts.cursor ?? "", limit + 1)
|
|
158
|
+
.map((r) => r.execution_id);
|
|
159
|
+
// An empty-string id sorts first and `> ''` would skip it; it names no execution, so that is moot.
|
|
160
|
+
const page = ids.slice(0, limit);
|
|
161
|
+
return ids.length > limit
|
|
162
|
+
? { executionIds: page, nextCursor: page[page.length - 1]! }
|
|
163
|
+
: { executionIds: page };
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
async read(
|
|
167
|
+
executionId: string,
|
|
168
|
+
range: { readonly from?: number; readonly through?: number } = {}
|
|
169
|
+
): Promise<ArchiveSegment[]> {
|
|
170
|
+
return this.db
|
|
171
|
+
.prepare<[string, number, number], Row>(
|
|
172
|
+
"SELECT * FROM archive_segments WHERE execution_id = ? AND through_seq >= ? AND from_seq <= ? ORDER BY from_seq"
|
|
173
|
+
)
|
|
174
|
+
.all(
|
|
175
|
+
executionId,
|
|
176
|
+
range.from ?? Number.MIN_SAFE_INTEGER,
|
|
177
|
+
range.through ?? Number.MAX_SAFE_INTEGER
|
|
178
|
+
)
|
|
179
|
+
.map(toSegment);
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
async deleteExecution(executionId: string): Promise<void> {
|
|
183
|
+
this.db.prepare("DELETE FROM archive_segments WHERE execution_id = ?").run(executionId);
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
async close(): Promise<void> {
|
|
187
|
+
this.db.close();
|
|
188
|
+
}
|
|
189
|
+
}
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
import {
|
|
2
|
+
BranchPromotionConflictError,
|
|
3
|
+
type BranchRef,
|
|
4
|
+
type BranchRegistry,
|
|
5
|
+
type LineageRecord,
|
|
6
|
+
type ListRefsOptions,
|
|
7
|
+
type PromoteOptions,
|
|
8
|
+
assertListRefsLimit,
|
|
9
|
+
} from "@loomwork/event-runtime";
|
|
10
|
+
import Database from "better-sqlite3";
|
|
11
|
+
import { migrate } from "./schema";
|
|
12
|
+
|
|
13
|
+
interface RefRow {
|
|
14
|
+
name: string;
|
|
15
|
+
execution_id: string;
|
|
16
|
+
tip: number;
|
|
17
|
+
parent_execution_id: string | null;
|
|
18
|
+
fork_sequence: number | null;
|
|
19
|
+
created_at: number;
|
|
20
|
+
updated_at: number;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
interface LineageRow {
|
|
24
|
+
execution_id: string;
|
|
25
|
+
parent_execution_id: string;
|
|
26
|
+
fork_sequence: number;
|
|
27
|
+
created_at: number;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function rowToRef(row: RefRow): BranchRef {
|
|
31
|
+
return {
|
|
32
|
+
name: row.name,
|
|
33
|
+
executionId: row.execution_id,
|
|
34
|
+
tip: row.tip,
|
|
35
|
+
...(row.parent_execution_id !== null && { parentExecutionId: row.parent_execution_id }),
|
|
36
|
+
...(row.fork_sequence !== null && { forkSequence: row.fork_sequence }),
|
|
37
|
+
createdAt: row.created_at,
|
|
38
|
+
updatedAt: row.updated_at,
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function rowToLineage(row: LineageRow): LineageRecord {
|
|
43
|
+
return {
|
|
44
|
+
executionId: row.execution_id,
|
|
45
|
+
parentExecutionId: row.parent_execution_id,
|
|
46
|
+
forkSequence: row.fork_sequence,
|
|
47
|
+
createdAt: row.created_at,
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* SQLite-backed {@link BranchRegistry}. Lives in @loomwork/event-runtime because
|
|
53
|
+
* refs describe event-log topology; point it at the same DB file as the
|
|
54
|
+
* SqliteEventStore (migrate() creates the `refs`/`lineage` tables idempotently).
|
|
55
|
+
*/
|
|
56
|
+
export class SqliteBranchRegistry implements BranchRegistry {
|
|
57
|
+
private readonly db: Database.Database;
|
|
58
|
+
|
|
59
|
+
constructor(dbPath: string) {
|
|
60
|
+
this.db = new Database(dbPath);
|
|
61
|
+
try {
|
|
62
|
+
this.db.pragma("journal_mode = WAL");
|
|
63
|
+
this.db.pragma("synchronous = NORMAL");
|
|
64
|
+
migrate(this.db);
|
|
65
|
+
} catch (err) {
|
|
66
|
+
this.db.close(); // don't leak the file handle if setup fails on a bad/locked file
|
|
67
|
+
throw err;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
async putRef(ref: BranchRef): Promise<void> {
|
|
72
|
+
this.writeRef(ref);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** Synchronous upsert — `putRef`'s body, callable from inside a transaction. */
|
|
76
|
+
private writeRef(ref: BranchRef): void {
|
|
77
|
+
this.db
|
|
78
|
+
.prepare<[string, string, number, string | null, number | null, number, number]>(`
|
|
79
|
+
INSERT INTO refs (name, execution_id, tip, parent_execution_id, fork_sequence, created_at, updated_at)
|
|
80
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
81
|
+
ON CONFLICT(name) DO UPDATE SET
|
|
82
|
+
execution_id = excluded.execution_id,
|
|
83
|
+
tip = excluded.tip,
|
|
84
|
+
parent_execution_id = excluded.parent_execution_id,
|
|
85
|
+
fork_sequence = excluded.fork_sequence,
|
|
86
|
+
updated_at = excluded.updated_at
|
|
87
|
+
`)
|
|
88
|
+
.run(
|
|
89
|
+
ref.name,
|
|
90
|
+
ref.executionId,
|
|
91
|
+
ref.tip,
|
|
92
|
+
ref.parentExecutionId ?? null,
|
|
93
|
+
ref.forkSequence ?? null,
|
|
94
|
+
ref.createdAt,
|
|
95
|
+
ref.updatedAt
|
|
96
|
+
);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
async getRef(name: string): Promise<BranchRef | null> {
|
|
100
|
+
const row = this.db.prepare<[string], RefRow>("SELECT * FROM refs WHERE name = ?").get(name);
|
|
101
|
+
return row ? rowToRef(row) : null;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
async listRefs(options?: ListRefsOptions): Promise<BranchRef[]> {
|
|
105
|
+
assertListRefsLimit(options?.limit);
|
|
106
|
+
const after = options?.after ?? null;
|
|
107
|
+
// `LIMIT -1` is SQLite for "no limit", so the guard above is load-bearing: an
|
|
108
|
+
// unvalidated negative would return the whole table to a caller asking for a page.
|
|
109
|
+
const rows = this.db
|
|
110
|
+
.prepare<[string | null, string | null, number], RefRow>(
|
|
111
|
+
`SELECT * FROM refs
|
|
112
|
+
WHERE (? IS NULL OR name > ?)
|
|
113
|
+
ORDER BY name ASC
|
|
114
|
+
LIMIT ?`
|
|
115
|
+
)
|
|
116
|
+
.all(after, after, options?.limit ?? -1);
|
|
117
|
+
return rows.map(rowToRef);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
async moveTip(name: string, tip: number): Promise<void> {
|
|
121
|
+
this.db
|
|
122
|
+
.prepare<[number, number, string]>("UPDATE refs SET tip = ?, updated_at = ? WHERE name = ?")
|
|
123
|
+
.run(tip, Date.now(), name);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
async promote(fromName: string, toName: string, options?: PromoteOptions): Promise<void> {
|
|
127
|
+
// better-sqlite3 is synchronous, so wrapping the read-modify-write in a transaction
|
|
128
|
+
// makes the compare-and-set atomic against other connections to the same file.
|
|
129
|
+
const cas = options !== undefined && "expect" in options;
|
|
130
|
+
const expected = options?.expect ?? null;
|
|
131
|
+
const run = this.db.transaction(() => {
|
|
132
|
+
const from = this.readRef(fromName);
|
|
133
|
+
if (!from) {
|
|
134
|
+
throw new Error(`Cannot promote: ref "${fromName}" does not exist`);
|
|
135
|
+
}
|
|
136
|
+
const existing = this.readRef(toName);
|
|
137
|
+
if (cas) {
|
|
138
|
+
const actual = existing?.executionId ?? null;
|
|
139
|
+
if (actual !== expected) {
|
|
140
|
+
throw new BranchPromotionConflictError(toName, expected, actual);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
const now = Date.now();
|
|
144
|
+
this.writeRef({
|
|
145
|
+
name: toName,
|
|
146
|
+
executionId: from.executionId,
|
|
147
|
+
tip: from.tip,
|
|
148
|
+
...(from.parentExecutionId !== undefined && { parentExecutionId: from.parentExecutionId }),
|
|
149
|
+
...(from.forkSequence !== undefined && { forkSequence: from.forkSequence }),
|
|
150
|
+
createdAt: existing?.createdAt ?? now,
|
|
151
|
+
updatedAt: now,
|
|
152
|
+
});
|
|
153
|
+
});
|
|
154
|
+
run();
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/** Synchronous ref read — the transaction body cannot await. */
|
|
158
|
+
private readRef(name: string): BranchRef | null {
|
|
159
|
+
const row = this.db.prepare<[string], RefRow>("SELECT * FROM refs WHERE name = ?").get(name);
|
|
160
|
+
return row ? rowToRef(row) : null;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
async deleteRef(name: string): Promise<void> {
|
|
164
|
+
this.db.prepare<[string]>("DELETE FROM refs WHERE name = ?").run(name);
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
async recordLineage(rec: Omit<LineageRecord, "createdAt">): Promise<void> {
|
|
168
|
+
this.db
|
|
169
|
+
.prepare<[string, string, number, number]>(`
|
|
170
|
+
INSERT OR REPLACE INTO lineage (execution_id, parent_execution_id, fork_sequence, created_at)
|
|
171
|
+
VALUES (?, ?, ?, ?)
|
|
172
|
+
`)
|
|
173
|
+
.run(rec.executionId, rec.parentExecutionId, rec.forkSequence, Date.now());
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
async removeLineage(executionId: string): Promise<void> {
|
|
177
|
+
this.db.prepare<[string]>("DELETE FROM lineage WHERE execution_id = ?").run(executionId);
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
async getLineage(executionId: string): Promise<LineageRecord | null> {
|
|
181
|
+
const row = this.db
|
|
182
|
+
.prepare<[string], LineageRow>("SELECT * FROM lineage WHERE execution_id = ?")
|
|
183
|
+
.get(executionId);
|
|
184
|
+
return row ? rowToLineage(row) : null;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
async children(executionId: string): Promise<LineageRecord[]> {
|
|
188
|
+
const rows = this.db
|
|
189
|
+
.prepare<[string], LineageRow>(
|
|
190
|
+
"SELECT * FROM lineage WHERE parent_execution_id = ? ORDER BY created_at ASC"
|
|
191
|
+
)
|
|
192
|
+
.all(executionId);
|
|
193
|
+
return rows.map(rowToLineage);
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
async close(): Promise<void> {
|
|
197
|
+
this.db.close();
|
|
198
|
+
}
|
|
199
|
+
}
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import {
|
|
3
|
+
type ContentListPage,
|
|
4
|
+
type ContentListPageOptions,
|
|
5
|
+
type ContentRef,
|
|
6
|
+
type ContentStore,
|
|
7
|
+
contentListPageLimit,
|
|
8
|
+
} from "@loomwork/plugin-content";
|
|
9
|
+
import Database from "better-sqlite3";
|
|
10
|
+
|
|
11
|
+
/** Bind variables per `touch` UPDATE (SQLite builds cap at 999 on old versions). */
|
|
12
|
+
const TOUCH_CHUNK = 500;
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* SQLite-backed {@link ContentStore} (the first durable content backend —
|
|
16
|
+
* ADR 0046 deferred one "until a real requirement lands"; the db-copilot's
|
|
17
|
+
* evidence retention is that requirement, and `loom sweep content` needs a
|
|
18
|
+
* file-backed store to operate on). Content-addressed (`sha256:<hex>`),
|
|
19
|
+
* idempotent by hash, `putStream` per the ADR 0059 outcome contract
|
|
20
|
+
* (hash-while-spooling). Point it at its own file — blob I/O must not contend
|
|
21
|
+
* with event-append latency (TS-7 discipline).
|
|
22
|
+
*
|
|
23
|
+
* SPI 1.4.0 adds `touch` (a batched `UPDATE … RETURNING`). SPI 1.3.0 (ADR 0055 D5/D6): every blob carries `touched_at`, bumped by EVERY put (a dedup'd
|
|
24
|
+
* put is a re-citation), so `deleteIfUntouchedSince` is one conditional `DELETE` and
|
|
25
|
+
* `listPage` is a keyset scan on the `address` primary key. A pre-1.3.0 file is migrated on
|
|
26
|
+
* open: the column is added and existing blobs are backfilled to the migration time
|
|
27
|
+
* (conservative — none ages out until a full TTL after the upgrade).
|
|
28
|
+
*/
|
|
29
|
+
export interface SqliteContentStoreOptions {
|
|
30
|
+
/** Clock for touch times (epoch ms). Default `Date.now`. */
|
|
31
|
+
now?: () => number;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export class SqliteContentStore implements ContentStore {
|
|
35
|
+
private readonly db: Database.Database;
|
|
36
|
+
private readonly now: () => number;
|
|
37
|
+
private readonly insertStmt: Database.Statement;
|
|
38
|
+
private readonly getStmt: Database.Statement;
|
|
39
|
+
private readonly deleteStmt: Database.Statement;
|
|
40
|
+
private readonly deleteIfUntouchedStmt: Database.Statement;
|
|
41
|
+
private readonly pageStmt: Database.Statement;
|
|
42
|
+
private readonly pageBeforeStmt: Database.Statement;
|
|
43
|
+
|
|
44
|
+
constructor(dbPath: string, options: SqliteContentStoreOptions = {}) {
|
|
45
|
+
this.now = options.now ?? Date.now;
|
|
46
|
+
this.db = new Database(dbPath);
|
|
47
|
+
try {
|
|
48
|
+
this.db.pragma("journal_mode = WAL");
|
|
49
|
+
this.db.pragma("synchronous = NORMAL");
|
|
50
|
+
this.db.exec(`
|
|
51
|
+
CREATE TABLE IF NOT EXISTS content_blobs (
|
|
52
|
+
address TEXT PRIMARY KEY,
|
|
53
|
+
bytes BLOB NOT NULL,
|
|
54
|
+
touched_at INTEGER NOT NULL DEFAULT 0
|
|
55
|
+
);
|
|
56
|
+
`);
|
|
57
|
+
this.migrateTouchedAt();
|
|
58
|
+
// Monotonic bump: a clock stepping back cannot make a re-cited blob look older.
|
|
59
|
+
this.insertStmt = this.db.prepare(
|
|
60
|
+
`INSERT INTO content_blobs (address, bytes, touched_at) VALUES (?, ?, ?)
|
|
61
|
+
ON CONFLICT(address) DO UPDATE SET touched_at = max(touched_at, excluded.touched_at)`
|
|
62
|
+
);
|
|
63
|
+
this.getStmt = this.db.prepare("SELECT bytes FROM content_blobs WHERE address = ?");
|
|
64
|
+
this.deleteStmt = this.db.prepare("DELETE FROM content_blobs WHERE address = ?");
|
|
65
|
+
this.deleteIfUntouchedStmt = this.db.prepare(
|
|
66
|
+
"DELETE FROM content_blobs WHERE address = ? AND touched_at < ?"
|
|
67
|
+
);
|
|
68
|
+
this.pageStmt = this.db.prepare(
|
|
69
|
+
`SELECT address, touched_at AS touchedAt FROM content_blobs
|
|
70
|
+
WHERE address > ? ORDER BY address LIMIT ?`
|
|
71
|
+
);
|
|
72
|
+
this.pageBeforeStmt = this.db.prepare(
|
|
73
|
+
`SELECT address, touched_at AS touchedAt FROM content_blobs
|
|
74
|
+
WHERE address > ? AND touched_at < ? ORDER BY address LIMIT ?`
|
|
75
|
+
);
|
|
76
|
+
} catch (err) {
|
|
77
|
+
this.db.close(); // don't leak the file handle if setup fails on a bad/locked file
|
|
78
|
+
throw err;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
async put(bytes: Uint8Array): Promise<ContentRef> {
|
|
83
|
+
const address = `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
|
|
84
|
+
this.insertStmt.run(address, Buffer.from(bytes), this.now());
|
|
85
|
+
return { __loomContent: address, size: bytes.byteLength };
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
async putStream(stream: AsyncIterable<Uint8Array>): Promise<ContentRef> {
|
|
89
|
+
// ADR 0059 outcome contract: address of the full sequence, idempotent —
|
|
90
|
+
// hash while spooling, then a single idempotent insert.
|
|
91
|
+
const hash = createHash("sha256");
|
|
92
|
+
const chunks: Buffer[] = [];
|
|
93
|
+
let size = 0;
|
|
94
|
+
for await (const chunk of stream) {
|
|
95
|
+
hash.update(chunk);
|
|
96
|
+
chunks.push(Buffer.from(chunk));
|
|
97
|
+
size += chunk.byteLength;
|
|
98
|
+
}
|
|
99
|
+
const address = `sha256:${hash.digest("hex")}`;
|
|
100
|
+
this.insertStmt.run(address, Buffer.concat(chunks, size), this.now());
|
|
101
|
+
return { __loomContent: address, size };
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
async get(ref: ContentRef): Promise<Uint8Array | null> {
|
|
105
|
+
const row = this.getStmt.get(ref.__loomContent) as { bytes: Buffer } | undefined;
|
|
106
|
+
return row === undefined ? null : new Uint8Array(row.bytes);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
async delete(ref: ContentRef): Promise<void> {
|
|
110
|
+
this.deleteStmt.run(ref.__loomContent);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
async deleteIfUntouchedSince(ref: ContentRef, cutoffMs: number): Promise<boolean> {
|
|
114
|
+
return this.deleteIfUntouchedStmt.run(ref.__loomContent, cutoffMs).changes > 0;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
async touchClock(): Promise<number> {
|
|
118
|
+
return this.now();
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* SPI 1.4.0: bump `touched_at` for existing blobs without a put. One `UPDATE … WHERE address
|
|
123
|
+
* IN (…) RETURNING address` per chunk (bounded bind-variable count), all inside one
|
|
124
|
+
* transaction so a batch is all-or-nothing. `max()` keeps the time monotonic; an unknown
|
|
125
|
+
* address matches no row (no insert), and is reported back as missing.
|
|
126
|
+
*/
|
|
127
|
+
/** One prepared `UPDATE` per distinct chunk size (at most {@link TOUCH_CHUNK} of them): a full chunk is prepared once, not per call. */
|
|
128
|
+
private readonly touchStatements = new Map<number, Database.Statement>();
|
|
129
|
+
|
|
130
|
+
private touchStatement(size: number): Database.Statement {
|
|
131
|
+
let statement = this.touchStatements.get(size);
|
|
132
|
+
if (statement === undefined) {
|
|
133
|
+
statement = this.db.prepare(
|
|
134
|
+
`UPDATE content_blobs SET touched_at = max(touched_at, ?)
|
|
135
|
+
WHERE address IN (${Array.from({ length: size }, () => "?").join(",")}) RETURNING address`
|
|
136
|
+
);
|
|
137
|
+
this.touchStatements.set(size, statement);
|
|
138
|
+
}
|
|
139
|
+
return statement;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
async touch(addresses: readonly string[]): Promise<string[]> {
|
|
143
|
+
const unique = [...new Set(addresses)];
|
|
144
|
+
if (unique.length === 0) return [];
|
|
145
|
+
const touchedAt = this.now();
|
|
146
|
+
const present = new Set<string>();
|
|
147
|
+
this.db.transaction(() => {
|
|
148
|
+
for (let i = 0; i < unique.length; i += TOUCH_CHUNK) {
|
|
149
|
+
const chunk = unique.slice(i, i + TOUCH_CHUNK);
|
|
150
|
+
const rows = this.touchStatement(chunk.length).all(touchedAt, ...chunk) as {
|
|
151
|
+
address: string;
|
|
152
|
+
}[];
|
|
153
|
+
for (const row of rows) present.add(row.address);
|
|
154
|
+
}
|
|
155
|
+
})();
|
|
156
|
+
return unique.filter((a) => !present.has(a));
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
async listPage(opts: ContentListPageOptions = {}): Promise<ContentListPage> {
|
|
160
|
+
const limit = contentListPageLimit(opts.limit);
|
|
161
|
+
const after = opts.cursor ?? "";
|
|
162
|
+
const entries = (
|
|
163
|
+
opts.touchedBefore === undefined
|
|
164
|
+
? this.pageStmt.all(after, limit)
|
|
165
|
+
: this.pageBeforeStmt.all(after, opts.touchedBefore, limit)
|
|
166
|
+
) as { address: string; touchedAt: number }[];
|
|
167
|
+
const last = entries[entries.length - 1];
|
|
168
|
+
return entries.length === limit && last !== undefined
|
|
169
|
+
? { entries, nextCursor: last.address }
|
|
170
|
+
: { entries };
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* One-time upgrade of a pre-1.3.0 file. IMMEDIATE so two processes opening the same file
|
|
175
|
+
* cannot both add the column; the re-check inside the transaction makes it idempotent.
|
|
176
|
+
*/
|
|
177
|
+
private migrateTouchedAt(): void {
|
|
178
|
+
const hasColumn = (): boolean =>
|
|
179
|
+
(this.db.prepare("PRAGMA table_info(content_blobs)").all() as { name: string }[]).some(
|
|
180
|
+
(c) => c.name === "touched_at"
|
|
181
|
+
);
|
|
182
|
+
if (hasColumn()) return;
|
|
183
|
+
this.db
|
|
184
|
+
.transaction(() => {
|
|
185
|
+
if (hasColumn()) return;
|
|
186
|
+
this.db.exec("ALTER TABLE content_blobs ADD COLUMN touched_at INTEGER NOT NULL DEFAULT 0");
|
|
187
|
+
this.db.prepare("UPDATE content_blobs SET touched_at = ?").run(this.now());
|
|
188
|
+
})
|
|
189
|
+
.immediate();
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
async close(): Promise<void> {
|
|
193
|
+
this.db.close();
|
|
194
|
+
}
|
|
195
|
+
}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import type { DurableStateRecord, DurableStateStore } from "@loomwork/plugin-durable-state";
|
|
2
|
+
import { RevisionConflictError } from "@loomwork/plugin-durable-state";
|
|
3
|
+
import Database from "better-sqlite3";
|
|
4
|
+
|
|
5
|
+
interface DurableStateRow {
|
|
6
|
+
contract_version: number;
|
|
7
|
+
revision: number;
|
|
8
|
+
payload: string;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* SQLite-backed {@link DurableStateStore} (ADR 0051 D2). Point it at the same
|
|
13
|
+
* DB file as the SqliteEventStore so one backup captures events + carry state.
|
|
14
|
+
* Own idempotent migration, same pattern as SqliteTimerStore. CAS rides
|
|
15
|
+
* single-statement atomicity: the guarded INSERT/UPDATE either lands the new
|
|
16
|
+
* revision or changes nothing, and a zero-change outcome reports the actual
|
|
17
|
+
* revision in a loud RevisionConflictError.
|
|
18
|
+
*/
|
|
19
|
+
export class SqliteDurableStateStore implements DurableStateStore {
|
|
20
|
+
private readonly db: Database.Database;
|
|
21
|
+
private readonly loadStmt: Database.Statement;
|
|
22
|
+
private readonly insertStmt: Database.Statement;
|
|
23
|
+
private readonly updateStmt: Database.Statement;
|
|
24
|
+
private readonly revisionStmt: Database.Statement;
|
|
25
|
+
private readonly clearStmt: Database.Statement;
|
|
26
|
+
|
|
27
|
+
constructor(dbPath: string) {
|
|
28
|
+
this.db = new Database(dbPath);
|
|
29
|
+
try {
|
|
30
|
+
this.db.pragma("journal_mode = WAL");
|
|
31
|
+
this.db.pragma("synchronous = NORMAL");
|
|
32
|
+
this.db.exec(`
|
|
33
|
+
CREATE TABLE IF NOT EXISTS durable_state (
|
|
34
|
+
key TEXT PRIMARY KEY,
|
|
35
|
+
contract_version INTEGER NOT NULL,
|
|
36
|
+
revision INTEGER NOT NULL,
|
|
37
|
+
payload TEXT NOT NULL
|
|
38
|
+
);
|
|
39
|
+
`);
|
|
40
|
+
this.loadStmt = this.db.prepare(
|
|
41
|
+
"SELECT contract_version, revision, payload FROM durable_state WHERE key = ?"
|
|
42
|
+
);
|
|
43
|
+
this.insertStmt = this.db.prepare(
|
|
44
|
+
"INSERT OR IGNORE INTO durable_state (key, contract_version, revision, payload) VALUES (?, ?, 1, ?)"
|
|
45
|
+
);
|
|
46
|
+
this.updateStmt = this.db.prepare(
|
|
47
|
+
"UPDATE durable_state SET contract_version = ?, revision = ?, payload = ? WHERE key = ? AND revision = ?"
|
|
48
|
+
);
|
|
49
|
+
this.revisionStmt = this.db.prepare("SELECT revision FROM durable_state WHERE key = ?");
|
|
50
|
+
this.clearStmt = this.db.prepare("DELETE FROM durable_state WHERE key = ?");
|
|
51
|
+
} catch (err) {
|
|
52
|
+
this.db.close(); // don't leak the file handle if setup fails on a bad/locked file
|
|
53
|
+
throw err;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
async load(key: string): Promise<DurableStateRecord | null> {
|
|
58
|
+
const row = this.loadStmt.get(key) as DurableStateRow | undefined;
|
|
59
|
+
if (row === undefined) return null;
|
|
60
|
+
return {
|
|
61
|
+
contractVersion: row.contract_version,
|
|
62
|
+
revision: row.revision,
|
|
63
|
+
payload: JSON.parse(row.payload),
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
async save(
|
|
68
|
+
key: string,
|
|
69
|
+
value: { contractVersion: number; payload: unknown },
|
|
70
|
+
expectedRevision: number
|
|
71
|
+
): Promise<DurableStateRecord> {
|
|
72
|
+
const payload = JSON.stringify(value.payload);
|
|
73
|
+
const changes =
|
|
74
|
+
expectedRevision === 0
|
|
75
|
+
? this.insertStmt.run(key, value.contractVersion, payload).changes
|
|
76
|
+
: this.updateStmt.run(
|
|
77
|
+
value.contractVersion,
|
|
78
|
+
expectedRevision + 1,
|
|
79
|
+
payload,
|
|
80
|
+
key,
|
|
81
|
+
expectedRevision
|
|
82
|
+
).changes;
|
|
83
|
+
if (changes === 1) {
|
|
84
|
+
return {
|
|
85
|
+
contractVersion: value.contractVersion,
|
|
86
|
+
revision: expectedRevision + 1,
|
|
87
|
+
payload: value.payload,
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
const row = this.revisionStmt.get(key) as { revision: number } | undefined;
|
|
91
|
+
throw new RevisionConflictError(key, expectedRevision, row?.revision ?? 0);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
async clear(key: string): Promise<void> {
|
|
95
|
+
this.clearStmt.run(key);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
async close(): Promise<void> {
|
|
99
|
+
this.db.close();
|
|
100
|
+
}
|
|
101
|
+
}
|