@vaur94/agz-memory 0.4.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +41 -25
- package/CHANGELOG.md +59 -0
- package/README.md +39 -16
- package/README.tr.md +40 -17
- package/dist/admin.js +3527 -622
- package/dist/core.js +4348 -924
- package/dist/server.js +4075 -874
- package/dist/types/capture/contract.d.ts +7 -5
- package/dist/types/capture/redact.d.ts +1 -0
- package/dist/types/context.d.ts +1 -1
- package/dist/types/core.d.ts +2 -2
- package/dist/types/db/backup.d.ts +2 -1
- package/dist/types/db/health.d.ts +2 -0
- package/dist/types/db/legacy-health.d.ts +2 -0
- package/dist/types/db/maintenance.d.ts +21 -0
- package/dist/types/db/migrations/v011.d.ts +3 -0
- package/dist/types/db/schema.d.ts +10 -1
- package/dist/types/db.d.ts +1 -0
- package/dist/types/hash.d.ts +4 -0
- package/dist/types/retrieval/contract.d.ts +15 -3
- package/dist/types/server.d.ts +1 -1
- package/dist/types/store/capture.d.ts +12 -4
- package/dist/types/store/outbox.d.ts +6 -3
- package/dist/types/store/retrieval.d.ts +2 -2
- package/dist/types/store.d.ts +12 -3
- package/dist/types/types.d.ts +1 -1
- package/dist/types/version.d.ts +1 -1
- package/docs/adr/hash-identity-v2.md +57 -0
- package/docs/adr/maintenance-lock.md +93 -0
- package/docs/backup-restore-runbook.md +36 -16
- package/docs/backup-restore-runbook.tr.md +36 -16
- package/docs/repository-hardening.md +125 -0
- package/docs/review-resolution.md +78 -0
- package/docs/schema-v11.md +75 -0
- package/package.json +19 -4
- package/skills/agz-memory/agz-memory.md +39 -0
- package/skills/index.json +9 -0
package/dist/admin.js
CHANGED
|
@@ -2,16 +2,16 @@
|
|
|
2
2
|
// @bun
|
|
3
3
|
|
|
4
4
|
// src/admin/index.ts
|
|
5
|
-
import { createHash as
|
|
6
|
-
import { Database as
|
|
5
|
+
import { createHash as createHash7 } from "crypto";
|
|
6
|
+
import { Database as Database4 } from "bun:sqlite";
|
|
7
7
|
import {
|
|
8
|
-
existsSync as
|
|
9
|
-
lstatSync as
|
|
10
|
-
readdirSync,
|
|
11
|
-
readFileSync as
|
|
8
|
+
existsSync as existsSync5,
|
|
9
|
+
lstatSync as lstatSync4,
|
|
10
|
+
readdirSync as readdirSync2,
|
|
11
|
+
readFileSync as readFileSync4,
|
|
12
12
|
rmSync as rmSync3
|
|
13
13
|
} from "fs";
|
|
14
|
-
import { basename as
|
|
14
|
+
import { basename as basename3, dirname as dirname3, resolve as resolve3 } from "path";
|
|
15
15
|
|
|
16
16
|
// src/config.ts
|
|
17
17
|
import { homedir } from "os";
|
|
@@ -22,114 +22,2736 @@ function resolveConfig(environment = process.env) {
|
|
|
22
22
|
}
|
|
23
23
|
|
|
24
24
|
// src/db.ts
|
|
25
|
-
import { randomUUID as
|
|
26
|
-
import { Database as
|
|
27
|
-
import { chmodSync as
|
|
25
|
+
import { randomUUID as randomUUID6 } from "crypto";
|
|
26
|
+
import { Database as Database3 } from "bun:sqlite";
|
|
27
|
+
import { chmodSync as chmodSync3, copyFileSync as copyFileSync2, existsSync as existsSync4, lstatSync as lstatSync3 } from "fs";
|
|
28
28
|
|
|
29
29
|
// src/identity.ts
|
|
30
30
|
import { createHash } from "crypto";
|
|
31
31
|
function hashRoot(directory) {
|
|
32
32
|
return createHash("sha256").update(directory).digest("hex");
|
|
33
33
|
}
|
|
34
|
-
|
|
35
|
-
// src/project.ts
|
|
36
|
-
function cleanProjectName(value) {
|
|
37
|
-
return value.trim().replace(/\s+/g, " ");
|
|
34
|
+
|
|
35
|
+
// src/project.ts
|
|
36
|
+
function cleanProjectName(value) {
|
|
37
|
+
return value.trim().replace(/\s+/g, " ");
|
|
38
|
+
}
|
|
39
|
+
function normalizeProjectName(value) {
|
|
40
|
+
return cleanProjectName(value).normalize("NFKC").toLowerCase();
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// src/types.ts
|
|
44
|
+
var SCHEMA_VERSION = 11;
|
|
45
|
+
var KINDS = ["decision", "fact", "procedure", "context", "research", "preference", "task"];
|
|
46
|
+
var PREDICATES = ["SUPPORTS", "DERIVED_FROM", "PART_OF", "ABOUT", "PRECEDES", "SUPERSEDES"];
|
|
47
|
+
|
|
48
|
+
// src/db/backup.ts
|
|
49
|
+
import { createHash as createHash4, randomUUID as randomUUID3 } from "crypto";
|
|
50
|
+
import { Database as Database2 } from "bun:sqlite";
|
|
51
|
+
import {
|
|
52
|
+
chmodSync as chmodSync2,
|
|
53
|
+
closeSync as closeSync2,
|
|
54
|
+
constants as constants2,
|
|
55
|
+
existsSync as existsSync2,
|
|
56
|
+
fstatSync,
|
|
57
|
+
fsyncSync as fsyncSync2,
|
|
58
|
+
lstatSync as lstatSync2,
|
|
59
|
+
mkdirSync as mkdirSync2,
|
|
60
|
+
openSync as openSync2,
|
|
61
|
+
readFileSync as readFileSync2,
|
|
62
|
+
readSync,
|
|
63
|
+
renameSync as renameSync2,
|
|
64
|
+
rmSync,
|
|
65
|
+
writeFileSync as writeFileSync2,
|
|
66
|
+
writeSync
|
|
67
|
+
} from "fs";
|
|
68
|
+
import { basename as basename2, dirname as dirname2, join as join3, resolve as resolve2 } from "path";
|
|
69
|
+
import { platform as platform2 } from "os";
|
|
70
|
+
|
|
71
|
+
// src/db/schema.ts
|
|
72
|
+
import { Database } from "bun:sqlite";
|
|
73
|
+
import { randomUUID } from "crypto";
|
|
74
|
+
|
|
75
|
+
// src/capture/contract.ts
|
|
76
|
+
import * as z from "zod/v4";
|
|
77
|
+
var CAPTURE_SCHEMA = "agz-memory.capture/2";
|
|
78
|
+
var SUPPORTED_OPENCODE_VERSION = "0.0.0-beta-18743";
|
|
79
|
+
var CAPTURE_EVENT_MAX_BYTES = 16 * 1024;
|
|
80
|
+
var CAPTURE_CONTENT_MAX_CHARACTERS = 4800;
|
|
81
|
+
var CAPTURE_SUMMARY_MAX_CHARACTERS = 1200;
|
|
82
|
+
var CAPTURE_SUBJECT_MAX_CHARACTERS = 240;
|
|
83
|
+
var candidateSchema = z.object({
|
|
84
|
+
kind: z.enum(KINDS),
|
|
85
|
+
title: z.string().min(1).max(240),
|
|
86
|
+
summary: z.string().min(1).max(CAPTURE_SUMMARY_MAX_CHARACTERS),
|
|
87
|
+
content: z.string().min(1).max(CAPTURE_CONTENT_MAX_CHARACTERS),
|
|
88
|
+
subjectKey: z.string().min(1).max(CAPTURE_SUBJECT_MAX_CHARACTERS).optional(),
|
|
89
|
+
intent: z.enum(["create", "supersede", "ignore", "review"]),
|
|
90
|
+
targetNoteID: z.string().min(1).max(240).optional(),
|
|
91
|
+
confidence: z.number().finite().min(0).max(1),
|
|
92
|
+
evidence: z.enum(["explicit-user", "verified-outcome", "session-summary"])
|
|
93
|
+
}).strict();
|
|
94
|
+
var signalSchema = z.object({
|
|
95
|
+
tool: z.string().min(1).max(160),
|
|
96
|
+
status: z.enum(["completed", "error"]),
|
|
97
|
+
errorType: z.string().min(1).max(160).optional()
|
|
98
|
+
}).strict();
|
|
99
|
+
var captureEventSchema = z.object({
|
|
100
|
+
schema: z.literal(CAPTURE_SCHEMA),
|
|
101
|
+
idempotencyKey: z.string().regex(/^[0-9a-f]{64}$/),
|
|
102
|
+
projectID: z.uuid(),
|
|
103
|
+
bindingKey: z.string().regex(/^[0-9a-f]{64}$/),
|
|
104
|
+
kind: z.enum(["user-candidate", "assistant-candidate", "session-summary", "tool-signal"]),
|
|
105
|
+
source: z.object({
|
|
106
|
+
system: z.literal("opencode-v2"),
|
|
107
|
+
opencodeVersion: z.literal(SUPPORTED_OPENCODE_VERSION),
|
|
108
|
+
pluginVersion: z.string().min(1).max(80),
|
|
109
|
+
sessionID: z.string().min(1).max(240),
|
|
110
|
+
messageID: z.string().min(1).max(240).optional(),
|
|
111
|
+
ordinal: z.number().finite().int().nonnegative().refine(Number.isSafeInteger, "must be a safe integer").optional(),
|
|
112
|
+
toolCallID: z.string().min(1).max(240).optional(),
|
|
113
|
+
observedAt: z.number().finite().int().nonnegative().refine(Number.isSafeInteger, "must be a safe integer")
|
|
114
|
+
}).strict(),
|
|
115
|
+
candidate: candidateSchema.optional(),
|
|
116
|
+
signal: signalSchema.optional(),
|
|
117
|
+
redaction: z.object({
|
|
118
|
+
policyVersion: z.string().min(1).max(80),
|
|
119
|
+
replacements: z.number().int().nonnegative(),
|
|
120
|
+
truncated: z.boolean()
|
|
121
|
+
}).strict()
|
|
122
|
+
}).strict().superRefine((event, context) => {
|
|
123
|
+
if (event.kind === "tool-signal" && !event.signal) {
|
|
124
|
+
context.addIssue({ code: "custom", message: "tool-signal requires signal" });
|
|
125
|
+
}
|
|
126
|
+
if (event.kind !== "tool-signal" && !event.candidate) {
|
|
127
|
+
context.addIssue({ code: "custom", message: `${event.kind} requires candidate` });
|
|
128
|
+
}
|
|
129
|
+
if (event.kind === "tool-signal" && event.candidate) {
|
|
130
|
+
context.addIssue({ code: "custom", message: "tool-signal cannot carry candidate" });
|
|
131
|
+
}
|
|
132
|
+
if (event.kind !== "tool-signal" && event.signal) {
|
|
133
|
+
context.addIssue({ code: "custom", message: `${event.kind} cannot carry signal` });
|
|
134
|
+
}
|
|
135
|
+
const hasSourceField = (field) => Object.prototype.hasOwnProperty.call(event.source, field);
|
|
136
|
+
const requiredFields = event.kind === "user-candidate" ? ["messageID"] : event.kind === "assistant-candidate" ? ["messageID", "ordinal"] : event.kind === "session-summary" ? ["messageID"] : ["messageID", "toolCallID"];
|
|
137
|
+
const forbiddenFields = event.kind === "assistant-candidate" ? ["toolCallID"] : event.kind === "tool-signal" ? ["ordinal"] : ["ordinal", "toolCallID"];
|
|
138
|
+
for (const field of requiredFields) {
|
|
139
|
+
if (event.source[field] === undefined) {
|
|
140
|
+
context.addIssue({ code: "custom", message: `${event.kind} requires source.${field}` });
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
for (const field of forbiddenFields) {
|
|
144
|
+
if (hasSourceField(field)) {
|
|
145
|
+
context.addIssue({ code: "custom", message: `${event.kind} forbids source.${field}` });
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
});
|
|
149
|
+
function parseCaptureEvent(value) {
|
|
150
|
+
const event = captureEventSchema.parse(value);
|
|
151
|
+
if (Buffer.byteLength(JSON.stringify(event), "utf8") > CAPTURE_EVENT_MAX_BYTES) {
|
|
152
|
+
throw new Error(`capture event exceeds ${CAPTURE_EVENT_MAX_BYTES} UTF-8 bytes`);
|
|
153
|
+
}
|
|
154
|
+
return event;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// src/hash.ts
|
|
158
|
+
import { createHash as createHash2 } from "crypto";
|
|
159
|
+
var HASH_TUPLE_MARKER = Buffer.from("agz-memory/hash-tuple", "utf8");
|
|
160
|
+
var MAX_UINT32 = 4294967295;
|
|
161
|
+
function hashTuple(domain, version, fields) {
|
|
162
|
+
if (typeof domain !== "string")
|
|
163
|
+
throw new TypeError("hash tuple domain must be a string");
|
|
164
|
+
if (!Number.isSafeInteger(version) || version < 0 || version > MAX_UINT32) {
|
|
165
|
+
throw new RangeError("hash tuple version must be an unsigned safe integer");
|
|
166
|
+
}
|
|
167
|
+
if (!Array.isArray(fields))
|
|
168
|
+
throw new TypeError("hash tuple fields must be an array");
|
|
169
|
+
if (fields.length > MAX_UINT32)
|
|
170
|
+
throw new RangeError("hash tuple has too many fields");
|
|
171
|
+
const domainBytes = utf8(domain);
|
|
172
|
+
const chunks = [
|
|
173
|
+
HASH_TUPLE_MARKER,
|
|
174
|
+
lengthPrefix(domainBytes),
|
|
175
|
+
uint32(version),
|
|
176
|
+
uint32(fields.length)
|
|
177
|
+
];
|
|
178
|
+
for (const field of fields) {
|
|
179
|
+
const encoded = encodeField(field);
|
|
180
|
+
chunks.push(Buffer.from([encoded.tag]), lengthPrefix(encoded.payload));
|
|
181
|
+
}
|
|
182
|
+
return createHash2("sha256").update(Buffer.concat(chunks)).digest("hex");
|
|
183
|
+
}
|
|
184
|
+
function noteContentHash(kind, title, summary, content) {
|
|
185
|
+
return hashTuple("canonical-note", 2, [kind, title, summary, content]);
|
|
186
|
+
}
|
|
187
|
+
function encodeField(value) {
|
|
188
|
+
if (value === null)
|
|
189
|
+
return { tag: 0, payload: Buffer.alloc(0) };
|
|
190
|
+
if (typeof value === "string")
|
|
191
|
+
return { tag: 1, payload: utf8(value) };
|
|
192
|
+
if (typeof value === "boolean")
|
|
193
|
+
return { tag: 2, payload: Buffer.from([value ? 1 : 0]) };
|
|
194
|
+
if (typeof value === "number") {
|
|
195
|
+
if (!Number.isFinite(value))
|
|
196
|
+
throw new RangeError("hash tuple numbers must be finite");
|
|
197
|
+
if (Number.isInteger(value) && !Number.isSafeInteger(value)) {
|
|
198
|
+
throw new RangeError("hash tuple integer numbers must be safe");
|
|
199
|
+
}
|
|
200
|
+
return { tag: 3, payload: utf8(String(value)) };
|
|
201
|
+
}
|
|
202
|
+
if (value instanceof Uint8Array)
|
|
203
|
+
return { tag: 4, payload: Buffer.from(value) };
|
|
204
|
+
throw new TypeError("hash tuple field has an unsupported type");
|
|
205
|
+
}
|
|
206
|
+
function utf8(value) {
|
|
207
|
+
if (!isWellFormedUnicode(value)) {
|
|
208
|
+
throw new TypeError("hash tuple strings must be well-formed Unicode");
|
|
209
|
+
}
|
|
210
|
+
const bytes = Buffer.from(value, "utf8");
|
|
211
|
+
if (bytes.length > MAX_UINT32)
|
|
212
|
+
throw new RangeError("hash tuple field is too large");
|
|
213
|
+
return bytes;
|
|
214
|
+
}
|
|
215
|
+
function isWellFormedUnicode(value) {
|
|
216
|
+
for (let index = 0;index < value.length; index++) {
|
|
217
|
+
const code = value.charCodeAt(index);
|
|
218
|
+
if (code >= 55296 && code <= 56319) {
|
|
219
|
+
if (index + 1 >= value.length)
|
|
220
|
+
return false;
|
|
221
|
+
const next = value.charCodeAt(index + 1);
|
|
222
|
+
if (next < 56320 || next > 57343)
|
|
223
|
+
return false;
|
|
224
|
+
index++;
|
|
225
|
+
} else if (code >= 56320 && code <= 57343) {
|
|
226
|
+
return false;
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
return true;
|
|
230
|
+
}
|
|
231
|
+
function lengthPrefix(bytes) {
|
|
232
|
+
return Buffer.concat([uint32(bytes.length), Buffer.from(bytes)]);
|
|
233
|
+
}
|
|
234
|
+
function uint32(value) {
|
|
235
|
+
const result = Buffer.alloc(4);
|
|
236
|
+
result.writeUInt32BE(value, 0);
|
|
237
|
+
return result;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
// src/db/schema.ts
|
|
241
|
+
var APPLICATION_ID = 1095195213;
|
|
242
|
+
var PRODUCT_ID = "agz-memory";
|
|
243
|
+
var HASH_POLICY = "hash-tuple/2";
|
|
244
|
+
var SQLITE_MANAGED_FTS_TABLES = new Set([
|
|
245
|
+
"notes_fts_config",
|
|
246
|
+
"notes_fts_data",
|
|
247
|
+
"notes_fts_docsize",
|
|
248
|
+
"notes_fts_idx"
|
|
249
|
+
]);
|
|
250
|
+
var SCHEMA_TABLES = `
|
|
251
|
+
CREATE TABLE IF NOT EXISTS projects (
|
|
252
|
+
id TEXT PRIMARY KEY,
|
|
253
|
+
name TEXT NOT NULL,
|
|
254
|
+
normalized_name TEXT NOT NULL UNIQUE,
|
|
255
|
+
created_at INTEGER NOT NULL,
|
|
256
|
+
updated_at INTEGER NOT NULL
|
|
257
|
+
);
|
|
258
|
+
CREATE TABLE IF NOT EXISTS notes (
|
|
259
|
+
id TEXT PRIMARY KEY,
|
|
260
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
261
|
+
kind TEXT NOT NULL CHECK (kind IN ('decision','fact','procedure','context','research','preference','task')),
|
|
262
|
+
title TEXT NOT NULL,
|
|
263
|
+
summary TEXT NOT NULL,
|
|
264
|
+
content TEXT NOT NULL,
|
|
265
|
+
size_class TEXT NOT NULL CHECK (size_class IN ('inline','indexed')),
|
|
266
|
+
pinned INTEGER NOT NULL DEFAULT 0 CHECK (pinned IN (0,1)),
|
|
267
|
+
status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('active','superseded','archived')),
|
|
268
|
+
supersedes_id TEXT,
|
|
269
|
+
current_revision INTEGER NOT NULL DEFAULT 1 CHECK (current_revision >= 1),
|
|
270
|
+
subject_key TEXT,
|
|
271
|
+
content_hash TEXT NOT NULL CHECK (length(content_hash) = 64),
|
|
272
|
+
created_at INTEGER NOT NULL,
|
|
273
|
+
updated_at INTEGER NOT NULL,
|
|
274
|
+
UNIQUE(project_id, id)
|
|
275
|
+
);
|
|
276
|
+
CREATE INDEX IF NOT EXISTS notes_project_idx ON notes(project_id, status);
|
|
277
|
+
CREATE UNIQUE INDEX IF NOT EXISTS notes_active_subject_idx
|
|
278
|
+
ON notes(project_id, kind, subject_key)
|
|
279
|
+
WHERE status = 'active' AND subject_key IS NOT NULL;
|
|
280
|
+
CREATE TABLE IF NOT EXISTS note_edges (
|
|
281
|
+
id TEXT PRIMARY KEY,
|
|
282
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
283
|
+
source_id TEXT NOT NULL,
|
|
284
|
+
target_id TEXT NOT NULL,
|
|
285
|
+
predicate TEXT NOT NULL CHECK (predicate IN ('SUPPORTS','DERIVED_FROM','PART_OF','ABOUT','PRECEDES','SUPERSEDES')),
|
|
286
|
+
created_at INTEGER NOT NULL,
|
|
287
|
+
UNIQUE(project_id, source_id, target_id, predicate),
|
|
288
|
+
FOREIGN KEY (project_id, source_id) REFERENCES notes(project_id, id) ON DELETE CASCADE,
|
|
289
|
+
FOREIGN KEY (project_id, target_id) REFERENCES notes(project_id, id) ON DELETE CASCADE
|
|
290
|
+
);
|
|
291
|
+
CREATE INDEX IF NOT EXISTS note_edges_source_idx ON note_edges(project_id, source_id);
|
|
292
|
+
CREATE INDEX IF NOT EXISTS note_edges_target_idx ON note_edges(project_id, target_id);
|
|
293
|
+
CREATE TABLE IF NOT EXISTS project_bindings (
|
|
294
|
+
binding_key TEXT PRIMARY KEY,
|
|
295
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
296
|
+
source TEXT NOT NULL CHECK (source = 'opencode-v2'),
|
|
297
|
+
source_project_id TEXT NOT NULL,
|
|
298
|
+
workspace_id TEXT NOT NULL,
|
|
299
|
+
canonical_path_hash TEXT NOT NULL CHECK (length(canonical_path_hash) = 64),
|
|
300
|
+
created_at INTEGER NOT NULL,
|
|
301
|
+
updated_at INTEGER NOT NULL,
|
|
302
|
+
UNIQUE(source, source_project_id, workspace_id)
|
|
303
|
+
);
|
|
304
|
+
CREATE TABLE IF NOT EXISTS capture_checkpoints (
|
|
305
|
+
session_id TEXT PRIMARY KEY,
|
|
306
|
+
binding_key TEXT NOT NULL REFERENCES project_bindings(binding_key) ON DELETE CASCADE,
|
|
307
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
308
|
+
state TEXT NOT NULL CHECK (state IN ('active','idle','unavailable','closed')),
|
|
309
|
+
last_message_id TEXT,
|
|
310
|
+
last_reconciled_at INTEGER,
|
|
311
|
+
next_reconcile_at INTEGER NOT NULL,
|
|
312
|
+
failure_count INTEGER NOT NULL DEFAULT 0 CHECK (failure_count >= 0),
|
|
313
|
+
lease_owner TEXT,
|
|
314
|
+
lease_expires_at INTEGER,
|
|
315
|
+
created_at INTEGER NOT NULL,
|
|
316
|
+
updated_at INTEGER NOT NULL
|
|
317
|
+
);
|
|
318
|
+
CREATE INDEX IF NOT EXISTS capture_checkpoints_due_idx
|
|
319
|
+
ON capture_checkpoints(state, next_reconcile_at);
|
|
320
|
+
${captureEventsTable()}
|
|
321
|
+
CREATE INDEX IF NOT EXISTS capture_events_state_idx ON capture_events(state, updated_at);
|
|
322
|
+
CREATE INDEX IF NOT EXISTS capture_events_session_idx ON capture_events(project_id, source_session_id);
|
|
323
|
+
CREATE INDEX IF NOT EXISTS capture_events_note_idx ON capture_events(project_id, note_id);
|
|
324
|
+
CREATE TABLE IF NOT EXISTS note_provenance (
|
|
325
|
+
id TEXT PRIMARY KEY,
|
|
326
|
+
project_id TEXT NOT NULL,
|
|
327
|
+
note_id TEXT NOT NULL,
|
|
328
|
+
source_type TEXT NOT NULL CHECK (source_type IN ('mcp-manual','opencode-capture','migration','legacy-import','admin')),
|
|
329
|
+
capture_event_id TEXT,
|
|
330
|
+
source_session_id TEXT,
|
|
331
|
+
source_message_id TEXT,
|
|
332
|
+
source_ordinal INTEGER,
|
|
333
|
+
source_tool_call_id TEXT,
|
|
334
|
+
redaction_version TEXT,
|
|
335
|
+
extractor_version TEXT,
|
|
336
|
+
confidence REAL CHECK (confidence IS NULL OR (confidence >= 0 AND confidence <= 1)),
|
|
337
|
+
created_at INTEGER NOT NULL,
|
|
338
|
+
UNIQUE(project_id, id),
|
|
339
|
+
FOREIGN KEY (project_id, note_id) REFERENCES notes(project_id, id) ON DELETE CASCADE
|
|
340
|
+
);
|
|
341
|
+
CREATE TABLE IF NOT EXISTS note_revisions (
|
|
342
|
+
project_id TEXT NOT NULL,
|
|
343
|
+
note_id TEXT NOT NULL,
|
|
344
|
+
revision INTEGER NOT NULL CHECK (revision >= 1),
|
|
345
|
+
kind TEXT NOT NULL,
|
|
346
|
+
title TEXT NOT NULL,
|
|
347
|
+
summary TEXT NOT NULL,
|
|
348
|
+
content TEXT NOT NULL,
|
|
349
|
+
size_class TEXT NOT NULL,
|
|
350
|
+
pinned INTEGER NOT NULL CHECK (pinned IN (0,1)),
|
|
351
|
+
status TEXT NOT NULL CHECK (status IN ('active','superseded','archived')),
|
|
352
|
+
supersedes_id TEXT,
|
|
353
|
+
subject_key TEXT,
|
|
354
|
+
content_hash TEXT NOT NULL,
|
|
355
|
+
provenance_id TEXT NOT NULL,
|
|
356
|
+
created_at INTEGER NOT NULL,
|
|
357
|
+
PRIMARY KEY(project_id, note_id, revision),
|
|
358
|
+
FOREIGN KEY (project_id, note_id) REFERENCES notes(project_id, id) ON DELETE CASCADE,
|
|
359
|
+
FOREIGN KEY (project_id, provenance_id) REFERENCES note_provenance(project_id, id)
|
|
360
|
+
);
|
|
361
|
+
CREATE TABLE IF NOT EXISTS index_outbox (
|
|
362
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
363
|
+
backend TEXT NOT NULL,
|
|
364
|
+
operation TEXT NOT NULL CHECK (operation IN ('upsert-note','delete-note','purge-project')),
|
|
365
|
+
project_id TEXT NOT NULL,
|
|
366
|
+
note_id TEXT,
|
|
367
|
+
revision INTEGER,
|
|
368
|
+
content_hash TEXT,
|
|
369
|
+
state TEXT NOT NULL DEFAULT 'pending' CHECK (state IN ('pending','leased','succeeded','dead')),
|
|
370
|
+
attempt_count INTEGER NOT NULL DEFAULT 0 CHECK (attempt_count >= 0),
|
|
371
|
+
available_at INTEGER NOT NULL,
|
|
372
|
+
lease_owner TEXT,
|
|
373
|
+
lease_expires_at INTEGER,
|
|
374
|
+
last_error_code TEXT,
|
|
375
|
+
created_at INTEGER NOT NULL,
|
|
376
|
+
completed_at INTEGER,
|
|
377
|
+
UNIQUE(backend, operation, project_id, note_id, revision)
|
|
378
|
+
);
|
|
379
|
+
CREATE INDEX IF NOT EXISTS index_outbox_due_idx
|
|
380
|
+
ON index_outbox(backend, project_id, state, available_at, id);
|
|
381
|
+
CREATE TABLE IF NOT EXISTS schema_state (version INTEGER PRIMARY KEY);
|
|
382
|
+
`;
|
|
383
|
+
var FTS_V9 = `
|
|
384
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS notes_fts USING fts5(
|
|
385
|
+
title, summary, content,
|
|
386
|
+
content='notes', content_rowid='rowid',
|
|
387
|
+
tokenize='unicode61'
|
|
388
|
+
);
|
|
389
|
+
CREATE TRIGGER IF NOT EXISTS notes_fts_ai AFTER INSERT ON notes BEGIN
|
|
390
|
+
INSERT INTO notes_fts(rowid, title, summary, content)
|
|
391
|
+
VALUES (new.rowid, new.title, new.summary, new.content);
|
|
392
|
+
END;
|
|
393
|
+
CREATE TRIGGER IF NOT EXISTS notes_fts_ad AFTER DELETE ON notes BEGIN
|
|
394
|
+
INSERT INTO notes_fts(notes_fts, rowid, title, summary, content)
|
|
395
|
+
VALUES ('delete', old.rowid, old.title, old.summary, old.content);
|
|
396
|
+
END;
|
|
397
|
+
CREATE TRIGGER IF NOT EXISTS notes_fts_au AFTER UPDATE OF title, summary, content ON notes BEGIN
|
|
398
|
+
INSERT INTO notes_fts(notes_fts, rowid, title, summary, content)
|
|
399
|
+
VALUES ('delete', old.rowid, old.title, old.summary, old.content);
|
|
400
|
+
INSERT INTO notes_fts(rowid, title, summary, content)
|
|
401
|
+
VALUES (new.rowid, new.title, new.summary, new.content);
|
|
402
|
+
END;
|
|
403
|
+
`;
|
|
404
|
+
var SCHEMA_V11_TABLES = `
|
|
405
|
+
CREATE TABLE projects (
|
|
406
|
+
id TEXT PRIMARY KEY,
|
|
407
|
+
name TEXT NOT NULL,
|
|
408
|
+
normalized_name TEXT NOT NULL UNIQUE,
|
|
409
|
+
created_at INTEGER NOT NULL,
|
|
410
|
+
updated_at INTEGER NOT NULL
|
|
411
|
+
);
|
|
412
|
+
CREATE TABLE notes (
|
|
413
|
+
id TEXT PRIMARY KEY,
|
|
414
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
415
|
+
kind TEXT NOT NULL CHECK (kind IN ('decision','fact','procedure','context','research','preference','task')),
|
|
416
|
+
title TEXT NOT NULL,
|
|
417
|
+
summary TEXT NOT NULL,
|
|
418
|
+
content TEXT NOT NULL,
|
|
419
|
+
size_class TEXT NOT NULL CHECK (size_class IN ('inline','indexed')),
|
|
420
|
+
pinned INTEGER NOT NULL DEFAULT 0 CHECK (pinned IN (0,1)),
|
|
421
|
+
status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('active','superseded','archived')),
|
|
422
|
+
supersedes_id TEXT,
|
|
423
|
+
current_revision INTEGER NOT NULL DEFAULT 1 CHECK (current_revision >= 1),
|
|
424
|
+
subject_key TEXT,
|
|
425
|
+
content_hash TEXT NOT NULL CHECK (length(content_hash) = 64),
|
|
426
|
+
created_at INTEGER NOT NULL,
|
|
427
|
+
updated_at INTEGER NOT NULL,
|
|
428
|
+
UNIQUE(project_id, id),
|
|
429
|
+
FOREIGN KEY (project_id, supersedes_id) REFERENCES notes(project_id, id) ON DELETE NO ACTION
|
|
430
|
+
);
|
|
431
|
+
CREATE INDEX notes_project_idx ON notes(project_id, status);
|
|
432
|
+
CREATE UNIQUE INDEX notes_active_subject_idx
|
|
433
|
+
ON notes(project_id, kind, subject_key)
|
|
434
|
+
WHERE status = 'active' AND subject_key IS NOT NULL;
|
|
435
|
+
CREATE TABLE note_edges (
|
|
436
|
+
id TEXT PRIMARY KEY,
|
|
437
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
438
|
+
source_id TEXT NOT NULL,
|
|
439
|
+
target_id TEXT NOT NULL,
|
|
440
|
+
predicate TEXT NOT NULL CHECK (predicate IN ('SUPPORTS','DERIVED_FROM','PART_OF','ABOUT','PRECEDES','SUPERSEDES')),
|
|
441
|
+
created_at INTEGER NOT NULL,
|
|
442
|
+
UNIQUE(project_id, source_id, target_id, predicate),
|
|
443
|
+
FOREIGN KEY (project_id, source_id) REFERENCES notes(project_id, id) ON DELETE CASCADE,
|
|
444
|
+
FOREIGN KEY (project_id, target_id) REFERENCES notes(project_id, id) ON DELETE CASCADE
|
|
445
|
+
);
|
|
446
|
+
CREATE INDEX note_edges_source_idx ON note_edges(project_id, source_id);
|
|
447
|
+
CREATE INDEX note_edges_target_idx ON note_edges(project_id, target_id);
|
|
448
|
+
CREATE TABLE project_bindings (
|
|
449
|
+
binding_key TEXT NOT NULL CHECK (length(binding_key) = 64),
|
|
450
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
451
|
+
source TEXT NOT NULL CHECK (source = 'opencode-v2'),
|
|
452
|
+
source_project_id TEXT NOT NULL,
|
|
453
|
+
workspace_id TEXT NOT NULL,
|
|
454
|
+
canonical_path_hash TEXT NOT NULL CHECK (length(canonical_path_hash) = 64),
|
|
455
|
+
created_at INTEGER NOT NULL,
|
|
456
|
+
updated_at INTEGER NOT NULL,
|
|
457
|
+
PRIMARY KEY(binding_key, project_id),
|
|
458
|
+
UNIQUE(source, source_project_id, workspace_id)
|
|
459
|
+
);
|
|
460
|
+
CREATE TABLE capture_checkpoints (
|
|
461
|
+
session_id TEXT NOT NULL,
|
|
462
|
+
binding_key TEXT NOT NULL,
|
|
463
|
+
project_id TEXT NOT NULL,
|
|
464
|
+
state TEXT NOT NULL CHECK (state IN ('active','idle','unavailable','closed')),
|
|
465
|
+
last_message_id TEXT,
|
|
466
|
+
last_reconciled_at INTEGER,
|
|
467
|
+
next_reconcile_at INTEGER NOT NULL,
|
|
468
|
+
failure_count INTEGER NOT NULL DEFAULT 0 CHECK (failure_count >= 0),
|
|
469
|
+
lease_owner TEXT,
|
|
470
|
+
lease_expires_at INTEGER,
|
|
471
|
+
created_at INTEGER NOT NULL,
|
|
472
|
+
updated_at INTEGER NOT NULL,
|
|
473
|
+
PRIMARY KEY(binding_key, session_id),
|
|
474
|
+
FOREIGN KEY (binding_key, project_id) REFERENCES project_bindings(binding_key, project_id) ON DELETE CASCADE
|
|
475
|
+
);
|
|
476
|
+
CREATE INDEX capture_checkpoints_due_idx
|
|
477
|
+
ON capture_checkpoints(state, next_reconcile_at);
|
|
478
|
+
${captureEventsTableV11()}
|
|
479
|
+
CREATE INDEX capture_events_state_idx ON capture_events(state, updated_at);
|
|
480
|
+
CREATE INDEX capture_events_session_idx ON capture_events(project_id, source_session_id);
|
|
481
|
+
CREATE INDEX capture_events_note_idx ON capture_events(project_id, note_id);
|
|
482
|
+
CREATE TABLE note_provenance (
|
|
483
|
+
id TEXT PRIMARY KEY,
|
|
484
|
+
project_id TEXT NOT NULL,
|
|
485
|
+
note_id TEXT NOT NULL,
|
|
486
|
+
source_type TEXT NOT NULL CHECK (source_type IN ('mcp-manual','opencode-capture','migration','legacy-import','admin')),
|
|
487
|
+
capture_event_id TEXT,
|
|
488
|
+
source_session_id TEXT,
|
|
489
|
+
source_message_id TEXT,
|
|
490
|
+
source_ordinal INTEGER,
|
|
491
|
+
source_tool_call_id TEXT,
|
|
492
|
+
redaction_version TEXT,
|
|
493
|
+
extractor_version TEXT,
|
|
494
|
+
confidence REAL CHECK (confidence IS NULL OR (confidence >= 0 AND confidence <= 1)),
|
|
495
|
+
created_at INTEGER NOT NULL,
|
|
496
|
+
UNIQUE(project_id, id),
|
|
497
|
+
FOREIGN KEY (project_id, note_id) REFERENCES notes(project_id, id) ON DELETE CASCADE
|
|
498
|
+
);
|
|
499
|
+
CREATE TABLE note_revisions (
|
|
500
|
+
project_id TEXT NOT NULL,
|
|
501
|
+
note_id TEXT NOT NULL,
|
|
502
|
+
revision INTEGER NOT NULL CHECK (revision >= 1),
|
|
503
|
+
kind TEXT NOT NULL CHECK (kind IN ('decision','fact','procedure','context','research','preference','task')),
|
|
504
|
+
title TEXT NOT NULL,
|
|
505
|
+
summary TEXT NOT NULL,
|
|
506
|
+
content TEXT NOT NULL,
|
|
507
|
+
size_class TEXT NOT NULL CHECK (size_class IN ('inline','indexed')),
|
|
508
|
+
pinned INTEGER NOT NULL CHECK (pinned IN (0,1)),
|
|
509
|
+
status TEXT NOT NULL CHECK (status IN ('active','superseded','archived')),
|
|
510
|
+
supersedes_id TEXT,
|
|
511
|
+
subject_key TEXT,
|
|
512
|
+
content_hash TEXT NOT NULL CHECK (length(content_hash) = 64),
|
|
513
|
+
provenance_id TEXT NOT NULL,
|
|
514
|
+
created_at INTEGER NOT NULL,
|
|
515
|
+
PRIMARY KEY(project_id, note_id, revision),
|
|
516
|
+
FOREIGN KEY (project_id, note_id) REFERENCES notes(project_id, id) ON DELETE CASCADE,
|
|
517
|
+
FOREIGN KEY (project_id, provenance_id) REFERENCES note_provenance(project_id, id),
|
|
518
|
+
FOREIGN KEY (project_id, supersedes_id) REFERENCES notes(project_id, id) ON DELETE NO ACTION
|
|
519
|
+
);
|
|
520
|
+
CREATE TABLE index_outbox (
|
|
521
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
522
|
+
backend TEXT NOT NULL,
|
|
523
|
+
operation_key TEXT NOT NULL CHECK (length(operation_key) = 64),
|
|
524
|
+
operation TEXT NOT NULL CHECK (operation IN ('upsert-note','delete-note','purge-project')),
|
|
525
|
+
project_id TEXT NOT NULL,
|
|
526
|
+
note_id TEXT,
|
|
527
|
+
revision INTEGER,
|
|
528
|
+
content_hash TEXT,
|
|
529
|
+
generation INTEGER NOT NULL DEFAULT 0 CHECK (generation >= 0),
|
|
530
|
+
lease_generation INTEGER NOT NULL DEFAULT 0 CHECK (lease_generation >= 0),
|
|
531
|
+
fence INTEGER NOT NULL DEFAULT 0 CHECK (fence >= 0),
|
|
532
|
+
state TEXT NOT NULL DEFAULT 'pending' CHECK (state IN ('pending','leased','succeeded','dead')),
|
|
533
|
+
attempt_count INTEGER NOT NULL DEFAULT 0 CHECK (attempt_count >= 0),
|
|
534
|
+
available_at INTEGER NOT NULL,
|
|
535
|
+
lease_owner TEXT,
|
|
536
|
+
lease_expires_at INTEGER,
|
|
537
|
+
heartbeat_at INTEGER CHECK (heartbeat_at IS NULL OR heartbeat_at >= 0),
|
|
538
|
+
last_error_code TEXT,
|
|
539
|
+
created_at INTEGER NOT NULL,
|
|
540
|
+
completed_at INTEGER,
|
|
541
|
+
CHECK (
|
|
542
|
+
(operation = 'upsert-note' AND note_id IS NOT NULL AND revision IS NOT NULL AND revision >= 1
|
|
543
|
+
AND content_hash IS NOT NULL AND length(content_hash) = 64)
|
|
544
|
+
OR (operation = 'delete-note' AND note_id IS NOT NULL AND revision IS NOT NULL AND revision >= 1
|
|
545
|
+
AND content_hash IS NULL)
|
|
546
|
+
OR (operation = 'purge-project' AND note_id IS NULL AND revision IS NULL AND content_hash IS NULL)
|
|
547
|
+
)
|
|
548
|
+
);
|
|
549
|
+
CREATE TRIGGER index_outbox_note_project_insert
|
|
550
|
+
BEFORE INSERT ON index_outbox
|
|
551
|
+
WHEN NEW.note_id IS NOT NULL
|
|
552
|
+
AND NEW.state IN ('pending','leased')
|
|
553
|
+
AND (NEW.operation = 'upsert-note'
|
|
554
|
+
OR EXISTS (SELECT 1 FROM notes WHERE id = NEW.note_id))
|
|
555
|
+
AND NOT EXISTS (
|
|
556
|
+
SELECT 1 FROM notes WHERE project_id = NEW.project_id AND id = NEW.note_id
|
|
557
|
+
)
|
|
558
|
+
BEGIN
|
|
559
|
+
SELECT RAISE(ABORT, 'foreign key index_outbox note project mismatch');
|
|
560
|
+
END;
|
|
561
|
+
CREATE TRIGGER index_outbox_note_project_update
|
|
562
|
+
BEFORE UPDATE OF project_id, note_id, operation ON index_outbox
|
|
563
|
+
WHEN NEW.note_id IS NOT NULL
|
|
564
|
+
AND NEW.state IN ('pending','leased')
|
|
565
|
+
AND (NEW.operation = 'upsert-note'
|
|
566
|
+
OR EXISTS (SELECT 1 FROM notes WHERE id = NEW.note_id))
|
|
567
|
+
AND NOT EXISTS (
|
|
568
|
+
SELECT 1 FROM notes WHERE project_id = NEW.project_id AND id = NEW.note_id
|
|
569
|
+
)
|
|
570
|
+
BEGIN
|
|
571
|
+
SELECT RAISE(ABORT, 'foreign key index_outbox note project mismatch');
|
|
572
|
+
END;
|
|
573
|
+
CREATE INDEX index_outbox_due_idx
|
|
574
|
+
ON index_outbox(backend, project_id, state, available_at, id);
|
|
575
|
+
CREATE UNIQUE INDEX index_outbox_active_operation_idx
|
|
576
|
+
ON index_outbox(operation_key)
|
|
577
|
+
WHERE state IN ('pending','leased');
|
|
578
|
+
CREATE UNIQUE INDEX index_outbox_active_upsert_idx
|
|
579
|
+
ON index_outbox(backend, project_id, note_id, revision, generation)
|
|
580
|
+
WHERE operation = 'upsert-note' AND state IN ('pending','leased');
|
|
581
|
+
CREATE UNIQUE INDEX index_outbox_active_delete_idx
|
|
582
|
+
ON index_outbox(backend, project_id, note_id, revision, generation)
|
|
583
|
+
WHERE operation = 'delete-note' AND state IN ('pending','leased');
|
|
584
|
+
CREATE UNIQUE INDEX index_outbox_active_purge_idx
|
|
585
|
+
ON index_outbox(backend, project_id, generation)
|
|
586
|
+
WHERE operation = 'purge-project' AND state IN ('pending','leased');
|
|
587
|
+
CREATE TABLE schema_state (version INTEGER PRIMARY KEY);
|
|
588
|
+
CREATE TABLE agz_meta (
|
|
589
|
+
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
590
|
+
database_id TEXT NOT NULL UNIQUE,
|
|
591
|
+
product_id TEXT NOT NULL CHECK (product_id = '${PRODUCT_ID}'),
|
|
592
|
+
schema_version INTEGER NOT NULL CHECK (schema_version = 11),
|
|
593
|
+
schema_fingerprint TEXT NOT NULL CHECK (length(schema_fingerprint) = 64),
|
|
594
|
+
hash_policy TEXT NOT NULL CHECK (hash_policy = '${HASH_POLICY}'),
|
|
595
|
+
created_at INTEGER NOT NULL
|
|
596
|
+
);
|
|
597
|
+
`;
|
|
598
|
+
function captureEventsTableV11(table = "capture_events") {
|
|
599
|
+
return `CREATE TABLE ${table} (
|
|
600
|
+
idempotency_key TEXT PRIMARY KEY CHECK (length(idempotency_key) = 64),
|
|
601
|
+
contract TEXT NOT NULL CHECK (contract = '${CAPTURE_SCHEMA}'),
|
|
602
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
603
|
+
binding_key TEXT NOT NULL,
|
|
604
|
+
event_kind TEXT NOT NULL CHECK (event_kind IN ('user-candidate','assistant-candidate','session-summary','tool-signal')),
|
|
605
|
+
source_session_id TEXT NOT NULL,
|
|
606
|
+
source_message_id TEXT,
|
|
607
|
+
source_ordinal INTEGER CHECK (source_ordinal IS NULL OR source_ordinal >= 0),
|
|
608
|
+
source_tool_call_id TEXT,
|
|
609
|
+
payload_json TEXT CHECK (payload_json IS NULL OR json_valid(payload_json)),
|
|
610
|
+
payload_hash TEXT CHECK (payload_hash IS NULL OR length(payload_hash) = 64),
|
|
611
|
+
redaction_version TEXT NOT NULL,
|
|
612
|
+
state TEXT NOT NULL CHECK (state IN ('pending','shadowed','review','materialized','duplicate','ignored','rejected','quarantined','failed','dead')),
|
|
613
|
+
attempt_count INTEGER NOT NULL DEFAULT 0 CHECK (attempt_count >= 0),
|
|
614
|
+
note_id TEXT,
|
|
615
|
+
last_error_code TEXT,
|
|
616
|
+
generation INTEGER NOT NULL DEFAULT 0 CHECK (generation >= 0),
|
|
617
|
+
created_at INTEGER NOT NULL,
|
|
618
|
+
updated_at INTEGER NOT NULL,
|
|
619
|
+
processed_at INTEGER,
|
|
620
|
+
UNIQUE(project_id, idempotency_key),
|
|
621
|
+
CHECK (
|
|
622
|
+
(event_kind = 'user-candidate' AND source_message_id IS NOT NULL
|
|
623
|
+
AND source_ordinal IS NULL AND source_tool_call_id IS NULL)
|
|
624
|
+
OR (event_kind = 'assistant-candidate' AND source_message_id IS NOT NULL
|
|
625
|
+
AND source_ordinal IS NOT NULL AND source_tool_call_id IS NULL)
|
|
626
|
+
OR (event_kind = 'session-summary' AND source_message_id IS NOT NULL
|
|
627
|
+
AND source_ordinal IS NULL AND source_tool_call_id IS NULL)
|
|
628
|
+
OR (event_kind = 'tool-signal' AND source_message_id IS NOT NULL
|
|
629
|
+
AND source_ordinal IS NULL AND source_tool_call_id IS NOT NULL)
|
|
630
|
+
),
|
|
631
|
+
FOREIGN KEY (binding_key, project_id) REFERENCES project_bindings(binding_key, project_id) ON DELETE CASCADE
|
|
632
|
+
);
|
|
633
|
+
CREATE TRIGGER ${table}_note_project_insert
|
|
634
|
+
BEFORE INSERT ON ${table}
|
|
635
|
+
WHEN NEW.note_id IS NOT NULL
|
|
636
|
+
AND EXISTS (SELECT 1 FROM notes WHERE id = NEW.note_id)
|
|
637
|
+
AND NOT EXISTS (SELECT 1 FROM notes WHERE project_id = NEW.project_id AND id = NEW.note_id)
|
|
638
|
+
BEGIN
|
|
639
|
+
SELECT RAISE(ABORT, 'foreign key capture event note project mismatch');
|
|
640
|
+
END;
|
|
641
|
+
CREATE TRIGGER ${table}_note_project_update
|
|
642
|
+
BEFORE UPDATE OF project_id, note_id ON ${table}
|
|
643
|
+
WHEN NEW.note_id IS NOT NULL
|
|
644
|
+
AND EXISTS (SELECT 1 FROM notes WHERE id = NEW.note_id)
|
|
645
|
+
AND NOT EXISTS (SELECT 1 FROM notes WHERE project_id = NEW.project_id AND id = NEW.note_id)
|
|
646
|
+
BEGIN
|
|
647
|
+
SELECT RAISE(ABORT, 'foreign key capture event note project mismatch');
|
|
648
|
+
END;`;
|
|
649
|
+
}
|
|
650
|
+
function schemaFingerprint(db) {
|
|
651
|
+
const rows = db.query(`SELECT type, name, tbl_name, sql
|
|
652
|
+
FROM sqlite_master
|
|
653
|
+
WHERE name NOT LIKE 'sqlite_%'
|
|
654
|
+
ORDER BY type, name`).all();
|
|
655
|
+
const fields = [];
|
|
656
|
+
for (const row of rows) {
|
|
657
|
+
fields.push(row.type, row.name, row.tbl_name, SQLITE_MANAGED_FTS_TABLES.has(row.name) ? null : row.sql);
|
|
658
|
+
}
|
|
659
|
+
return hashTuple("schema-fingerprint", 2, fields);
|
|
660
|
+
}
|
|
661
|
+
var cachedExpectedFingerprint;
|
|
662
|
+
function expectedSchemaFingerprint() {
|
|
663
|
+
if (cachedExpectedFingerprint)
|
|
664
|
+
return cachedExpectedFingerprint;
|
|
665
|
+
const db = new Database(":memory:");
|
|
666
|
+
try {
|
|
667
|
+
db.exec(SCHEMA_V11_TABLES);
|
|
668
|
+
db.exec(FTS_V9);
|
|
669
|
+
cachedExpectedFingerprint = schemaFingerprint(db);
|
|
670
|
+
return cachedExpectedFingerprint;
|
|
671
|
+
} finally {
|
|
672
|
+
db.close();
|
|
673
|
+
}
|
|
674
|
+
}
|
|
675
|
+
function createSchemaV11(db) {
|
|
676
|
+
db.exec(`PRAGMA application_id = ${APPLICATION_ID}`);
|
|
677
|
+
db.exec(SCHEMA_V11_TABLES);
|
|
678
|
+
db.exec(FTS_V9);
|
|
679
|
+
db.query("INSERT INTO schema_state(version) VALUES (11)").run();
|
|
680
|
+
const fingerprint = schemaFingerprint(db);
|
|
681
|
+
if (fingerprint !== expectedSchemaFingerprint()) {
|
|
682
|
+
throw new Error("schema_fingerprint_mismatch");
|
|
683
|
+
}
|
|
684
|
+
db.query(`INSERT INTO agz_meta
|
|
685
|
+
(id, database_id, product_id, schema_version, schema_fingerprint, hash_policy, created_at)
|
|
686
|
+
VALUES (1, ?, ?, 11, ?, ?, ?)`).run(randomUUID(), PRODUCT_ID, fingerprint, HASH_POLICY, Date.now());
|
|
687
|
+
}
|
|
688
|
+
function insertV11Identity(db, databaseID = randomUUID(), createdAt = Date.now()) {
|
|
689
|
+
const fingerprint = schemaFingerprint(db);
|
|
690
|
+
if (fingerprint !== expectedSchemaFingerprint()) {
|
|
691
|
+
throw new Error("schema_fingerprint_mismatch");
|
|
692
|
+
}
|
|
693
|
+
db.query(`INSERT INTO agz_meta
|
|
694
|
+
(id, database_id, product_id, schema_version, schema_fingerprint, hash_policy, created_at)
|
|
695
|
+
VALUES (1, ?, ?, 11, ?, ?, ?)`).run(databaseID, PRODUCT_ID, fingerprint, HASH_POLICY, createdAt);
|
|
696
|
+
}
|
|
697
|
+
function rebuildFts(db) {
|
|
698
|
+
db.query("INSERT INTO notes_fts(notes_fts) VALUES ('rebuild')").run();
|
|
699
|
+
}
|
|
700
|
+
function captureEventsTable(table = "capture_events") {
|
|
701
|
+
return `CREATE TABLE IF NOT EXISTS ${table} (
|
|
702
|
+
idempotency_key TEXT PRIMARY KEY,
|
|
703
|
+
contract TEXT NOT NULL CHECK (contract = '${CAPTURE_SCHEMA}'),
|
|
704
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
705
|
+
binding_key TEXT NOT NULL REFERENCES project_bindings(binding_key) ON DELETE CASCADE,
|
|
706
|
+
event_kind TEXT NOT NULL CHECK (event_kind IN ('user-candidate','assistant-candidate','session-summary','tool-signal')),
|
|
707
|
+
source_session_id TEXT NOT NULL,
|
|
708
|
+
source_message_id TEXT,
|
|
709
|
+
source_ordinal INTEGER CHECK (source_ordinal IS NULL OR source_ordinal >= 0),
|
|
710
|
+
source_tool_call_id TEXT,
|
|
711
|
+
payload_json TEXT CHECK (payload_json IS NULL OR json_valid(payload_json)),
|
|
712
|
+
payload_hash TEXT,
|
|
713
|
+
redaction_version TEXT NOT NULL,
|
|
714
|
+
state TEXT NOT NULL CHECK (state IN ('pending','shadowed','review','materialized','duplicate','ignored','rejected','quarantined','failed','dead')),
|
|
715
|
+
attempt_count INTEGER NOT NULL DEFAULT 0 CHECK (attempt_count >= 0),
|
|
716
|
+
note_id TEXT,
|
|
717
|
+
last_error_code TEXT,
|
|
718
|
+
generation INTEGER NOT NULL DEFAULT 0 CHECK (generation >= 0),
|
|
719
|
+
created_at INTEGER NOT NULL,
|
|
720
|
+
updated_at INTEGER NOT NULL,
|
|
721
|
+
processed_at INTEGER
|
|
722
|
+
);`;
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
// src/db/health.ts
|
|
726
|
+
function inspectDatabase(db) {
|
|
727
|
+
const integrity = db.query("PRAGMA integrity_check").get().integrity_check;
|
|
728
|
+
const foreignKeyViolations = db.query("PRAGMA foreign_key_check").all();
|
|
729
|
+
const schemaVersion = hasTable(db, "schema_state") ? db.query("SELECT MAX(version) AS version FROM schema_state").get().version ?? undefined : undefined;
|
|
730
|
+
const counts = {};
|
|
731
|
+
for (const table of [
|
|
732
|
+
"projects",
|
|
733
|
+
"notes",
|
|
734
|
+
"note_edges",
|
|
735
|
+
"notes_fts",
|
|
736
|
+
"project_bindings",
|
|
737
|
+
"capture_events",
|
|
738
|
+
"capture_checkpoints",
|
|
739
|
+
"note_provenance",
|
|
740
|
+
"note_revisions",
|
|
741
|
+
"index_outbox"
|
|
742
|
+
]) {
|
|
743
|
+
if (!hasTable(db, table))
|
|
744
|
+
continue;
|
|
745
|
+
counts[table] = db.query(`SELECT COUNT(*) AS count FROM ${table}`).get().count;
|
|
746
|
+
}
|
|
747
|
+
return { integrity, foreignKeyViolations, schemaVersion, counts };
|
|
748
|
+
}
|
|
749
|
+
function assertHealthyDatabase(db) {
|
|
750
|
+
const health = inspectDatabase(db);
|
|
751
|
+
if (health.integrity !== "ok") {
|
|
752
|
+
throw new Error(`database integrity check failed: ${health.integrity}`);
|
|
753
|
+
}
|
|
754
|
+
if (health.foreignKeyViolations.length > 0) {
|
|
755
|
+
throw new Error(`database foreign key check failed: ${health.foreignKeyViolations.length} violation(s)`);
|
|
756
|
+
}
|
|
757
|
+
if (health.schemaVersion === 11)
|
|
758
|
+
assertSchemaV11(db);
|
|
759
|
+
return health;
|
|
760
|
+
}
|
|
761
|
+
function assertSchemaV11(db) {
|
|
762
|
+
try {
|
|
763
|
+
if (!hasTable(db, "agz_meta"))
|
|
764
|
+
throw new Error("missing metadata table");
|
|
765
|
+
const applicationID = db.query("PRAGMA application_id").get().application_id;
|
|
766
|
+
if (applicationID !== APPLICATION_ID)
|
|
767
|
+
throw new Error("application id mismatch");
|
|
768
|
+
const states = db.query("SELECT version FROM schema_state").all();
|
|
769
|
+
if (states.length !== 1 || states[0]?.version !== 11)
|
|
770
|
+
throw new Error("schema state mismatch");
|
|
771
|
+
const metadata = db.query(`SELECT id, database_id, product_id, schema_version, schema_fingerprint, hash_policy, created_at
|
|
772
|
+
FROM agz_meta`).all();
|
|
773
|
+
if (metadata.length !== 1)
|
|
774
|
+
throw new Error("metadata cardinality mismatch");
|
|
775
|
+
const [meta] = metadata;
|
|
776
|
+
if (!meta || meta.id !== 1 || !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(meta.database_id) || meta.product_id !== PRODUCT_ID || meta.schema_version !== 11 || meta.hash_policy !== HASH_POLICY || !/^[0-9a-f]{64}$/.test(meta.schema_fingerprint) || !Number.isSafeInteger(meta.created_at) || meta.created_at < 0) {
|
|
777
|
+
throw new Error("metadata value mismatch");
|
|
778
|
+
}
|
|
779
|
+
const expected = expectedSchemaFingerprint();
|
|
780
|
+
if (meta.schema_fingerprint !== expected || schemaFingerprint(db) !== expected) {
|
|
781
|
+
throw new Error("schema fingerprint mismatch");
|
|
782
|
+
}
|
|
783
|
+
} catch (error) {
|
|
784
|
+
if (isSQLiteBusyError(error))
|
|
785
|
+
throw error;
|
|
786
|
+
if (error instanceof Error && error.message === "schema_fingerprint_mismatch")
|
|
787
|
+
throw error;
|
|
788
|
+
throw new Error("schema_fingerprint_mismatch");
|
|
789
|
+
}
|
|
790
|
+
}
|
|
791
|
+
function isSQLiteBusyError(error) {
|
|
792
|
+
if (error && typeof error === "object" && "code" in error) {
|
|
793
|
+
const code = String(error.code);
|
|
794
|
+
if (code === "SQLITE_BUSY")
|
|
795
|
+
return true;
|
|
796
|
+
}
|
|
797
|
+
return error instanceof Error && /\bdatabase is (?:busy|locked)\b/i.test(error.message);
|
|
798
|
+
}
|
|
799
|
+
function hasTable(db, table) {
|
|
800
|
+
const row = db.query("SELECT COUNT(*) AS count FROM sqlite_master WHERE type IN ('table','view') AND name = ?").get(table);
|
|
801
|
+
return row.count > 0;
|
|
802
|
+
}
|
|
803
|
+
|
|
804
|
+
// src/db/migrations/v011.ts
|
|
805
|
+
import { createHash as createHash3 } from "crypto";
|
|
806
|
+
|
|
807
|
+
// src/capture/identity.ts
|
|
808
|
+
function captureIdempotencyKey(input) {
|
|
809
|
+
let fields;
|
|
810
|
+
if (input.kind === "user") {
|
|
811
|
+
fields = ["user", input.bindingKey, input.sessionID, input.messageID];
|
|
812
|
+
} else if (input.kind === "assistant") {
|
|
813
|
+
if (!Number.isSafeInteger(input.ordinal) || input.ordinal < 0) {
|
|
814
|
+
throw new RangeError("assistant capture ordinal must be a non-negative safe integer");
|
|
815
|
+
}
|
|
816
|
+
fields = [
|
|
817
|
+
"assistant",
|
|
818
|
+
input.bindingKey,
|
|
819
|
+
input.sessionID,
|
|
820
|
+
input.assistantMessageID,
|
|
821
|
+
input.ordinal
|
|
822
|
+
];
|
|
823
|
+
} else if (input.kind === "tool") {
|
|
824
|
+
fields = [
|
|
825
|
+
"tool",
|
|
826
|
+
input.bindingKey,
|
|
827
|
+
input.sessionID,
|
|
828
|
+
input.assistantMessageID,
|
|
829
|
+
input.toolCallID,
|
|
830
|
+
input.terminalStatus
|
|
831
|
+
];
|
|
832
|
+
} else {
|
|
833
|
+
fields = ["summary", input.bindingKey, input.sessionID, input.checkpointMessageID];
|
|
834
|
+
}
|
|
835
|
+
return hashTuple("capture-identity", 2, fields);
|
|
836
|
+
}
|
|
837
|
+
|
|
838
|
+
// src/capture/redact.ts
|
|
839
|
+
var SECRET_ASSIGNMENT_KEY = String.raw`(?:[A-Za-z][A-Za-z0-9_-]*(?:password|passwd|secret|token|api[_-]?key|private[_-]?key)[A-Za-z0-9_-]*|password|passwd|secret|token|api[_-]?key|private[_-]?key)`;
|
|
840
|
+
var SECRET_ASSIGNMENT_VALUE = String.raw`(?:"(?![<$\[])[^"\r\n]{8,}"|'(?![<$\[])[^'\r\n]{8,}'|(?![<$\[])[^\s,;]{8,})`;
|
|
841
|
+
var RULES = [
|
|
842
|
+
{
|
|
843
|
+
name: "private-key",
|
|
844
|
+
pattern: /-----BEGIN (?:RSA |EC |OPENSSH |DSA |ENCRYPTED |PGP )?PRIVATE KEY(?: BLOCK)?-----[\s\S]*?-----END (?:RSA |EC |OPENSSH |DSA |ENCRYPTED |PGP )?PRIVATE KEY(?: BLOCK)?-----/gi
|
|
845
|
+
},
|
|
846
|
+
{
|
|
847
|
+
name: "credential-uri",
|
|
848
|
+
pattern: /[a-z][a-z0-9+.-]*:\/\/[^\s/:@]+:[^\s/@]+@[^\s]+/gi
|
|
849
|
+
},
|
|
850
|
+
{
|
|
851
|
+
name: "bearer",
|
|
852
|
+
pattern: /Bearer\s+[A-Za-z0-9._~+/=-]{12,}(?=$|[^A-Za-z0-9._~+/=-])/gi
|
|
853
|
+
},
|
|
854
|
+
{
|
|
855
|
+
name: "basic-auth",
|
|
856
|
+
pattern: /Basic\s+[A-Za-z0-9+/=]{12,}(?=$|[^A-Za-z0-9+/=])/gi
|
|
857
|
+
},
|
|
858
|
+
{
|
|
859
|
+
name: "github-token",
|
|
860
|
+
pattern: /(?:gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,})(?=$|[^A-Za-z0-9_])/g
|
|
861
|
+
},
|
|
862
|
+
{
|
|
863
|
+
name: "gitlab-token",
|
|
864
|
+
pattern: /glpat-[A-Za-z0-9_-]{16,}(?=$|[^A-Za-z0-9_-])/g
|
|
865
|
+
},
|
|
866
|
+
{
|
|
867
|
+
name: "aws-access-key",
|
|
868
|
+
pattern: /(?:AKIA|ASIA)[A-Z0-9]{16}(?=$|[^A-Z0-9])/g
|
|
869
|
+
},
|
|
870
|
+
{
|
|
871
|
+
name: "jwt",
|
|
872
|
+
pattern: /eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}(?=$|[^A-Za-z0-9_-])/g
|
|
873
|
+
},
|
|
874
|
+
{
|
|
875
|
+
name: "anthropic-token",
|
|
876
|
+
pattern: /sk-ant-[A-Za-z0-9_-]{20,}(?=$|[^A-Za-z0-9_-])/g
|
|
877
|
+
},
|
|
878
|
+
{
|
|
879
|
+
name: "openai-token",
|
|
880
|
+
pattern: /sk-(?!ant-)(?:proj-)?[A-Za-z0-9_-]{20,}(?=$|[^A-Za-z0-9_-])/g
|
|
881
|
+
},
|
|
882
|
+
{
|
|
883
|
+
name: "google-api-key",
|
|
884
|
+
pattern: /AIza[A-Za-z0-9_-]{30,}(?=$|[^A-Za-z0-9_-])/g
|
|
885
|
+
},
|
|
886
|
+
{
|
|
887
|
+
name: "slack-token",
|
|
888
|
+
pattern: /xox[baprs]-[A-Za-z0-9-]{32,}(?=$|[^A-Za-z0-9-])/g
|
|
889
|
+
},
|
|
890
|
+
{
|
|
891
|
+
name: "stripe-token",
|
|
892
|
+
pattern: /[sr]k_(?:live|test)_[A-Za-z0-9]{16,}(?=$|[^A-Za-z0-9])/g
|
|
893
|
+
},
|
|
894
|
+
{
|
|
895
|
+
name: "npm-token",
|
|
896
|
+
pattern: /npm_[A-Za-z0-9]{20,}(?=$|[^A-Za-z0-9])/g
|
|
897
|
+
},
|
|
898
|
+
{
|
|
899
|
+
name: "api-key-header",
|
|
900
|
+
pattern: new RegExp(String.raw`(?:x-api-key|api-key|api_key)\s*[:=]\s*${SECRET_ASSIGNMENT_VALUE}`, "gi")
|
|
901
|
+
},
|
|
902
|
+
{
|
|
903
|
+
name: "secret-assignment",
|
|
904
|
+
pattern: new RegExp(String.raw`${SECRET_ASSIGNMENT_KEY}\s*[:=]\s*${SECRET_ASSIGNMENT_VALUE}`, "gi")
|
|
905
|
+
}
|
|
906
|
+
];
|
|
907
|
+
function redactText(value, options = {}) {
|
|
908
|
+
const maxCharacters = normalizeLimit(options.maxCharacters);
|
|
909
|
+
const inputLength = value.length;
|
|
910
|
+
let text = value;
|
|
911
|
+
let replacements = 0;
|
|
912
|
+
let detectedSecret = false;
|
|
913
|
+
const classes = {};
|
|
914
|
+
for (const literal2 of options.denylist ?? []) {
|
|
915
|
+
if (typeof literal2 !== "string" || !literal2)
|
|
916
|
+
continue;
|
|
917
|
+
let count = 0;
|
|
918
|
+
let offset = 0;
|
|
919
|
+
while (offset <= text.length) {
|
|
920
|
+
const index = text.indexOf(literal2, offset);
|
|
921
|
+
if (index < 0)
|
|
922
|
+
break;
|
|
923
|
+
count++;
|
|
924
|
+
offset = index + literal2.length;
|
|
925
|
+
}
|
|
926
|
+
if (count === 0)
|
|
927
|
+
continue;
|
|
928
|
+
replacements += count;
|
|
929
|
+
detectedSecret = true;
|
|
930
|
+
classes.denylist = (classes.denylist ?? 0) + count;
|
|
931
|
+
text = text.replaceAll(literal2, "[REDACTED:denylist]");
|
|
932
|
+
}
|
|
933
|
+
for (const rule of RULES) {
|
|
934
|
+
rule.pattern.lastIndex = 0;
|
|
935
|
+
text = text.replace(rule.pattern, () => {
|
|
936
|
+
replacements++;
|
|
937
|
+
detectedSecret = true;
|
|
938
|
+
classes[rule.name] = (classes[rule.name] ?? 0) + 1;
|
|
939
|
+
return `[REDACTED:${rule.name}]`;
|
|
940
|
+
});
|
|
941
|
+
}
|
|
942
|
+
text = text.replace(/[A-Za-z0-9+/=_-]{32,}/g, (candidate) => {
|
|
943
|
+
if (!looksHighEntropy(candidate))
|
|
944
|
+
return candidate;
|
|
945
|
+
replacements++;
|
|
946
|
+
detectedSecret = true;
|
|
947
|
+
classes.entropy = (classes.entropy ?? 0) + 1;
|
|
948
|
+
return "[REDACTED:entropy]";
|
|
949
|
+
});
|
|
950
|
+
const truncated = Boolean(options.sourceTruncated) || inputLength > maxCharacters || text.length > maxCharacters;
|
|
951
|
+
if (text.length > maxCharacters)
|
|
952
|
+
text = truncateHead(text, maxCharacters);
|
|
953
|
+
return {
|
|
954
|
+
text,
|
|
955
|
+
replacements,
|
|
956
|
+
classes,
|
|
957
|
+
truncated,
|
|
958
|
+
quarantined: detectedSecret
|
|
959
|
+
};
|
|
960
|
+
}
|
|
961
|
+
function normalizeLimit(value) {
|
|
962
|
+
if (value === undefined || value === Number.POSITIVE_INFINITY)
|
|
963
|
+
return Number.MAX_SAFE_INTEGER;
|
|
964
|
+
if (!Number.isFinite(value))
|
|
965
|
+
return 0;
|
|
966
|
+
return Math.max(0, Math.floor(value));
|
|
967
|
+
}
|
|
968
|
+
function truncateHead(value, maxCharacters) {
|
|
969
|
+
if (maxCharacters <= 0)
|
|
970
|
+
return "";
|
|
971
|
+
if (value.length <= maxCharacters)
|
|
972
|
+
return value;
|
|
973
|
+
let offset = 0;
|
|
974
|
+
while (offset < value.length) {
|
|
975
|
+
const codePoint = value.codePointAt(offset);
|
|
976
|
+
const width = codePoint > 65535 ? 2 : 1;
|
|
977
|
+
if (offset + width > maxCharacters)
|
|
978
|
+
break;
|
|
979
|
+
offset += width;
|
|
980
|
+
}
|
|
981
|
+
return value.slice(0, offset);
|
|
982
|
+
}
|
|
983
|
+
function looksHighEntropy(value) {
|
|
984
|
+
if (!/[A-Za-z]/.test(value) || !/\d/.test(value))
|
|
985
|
+
return false;
|
|
986
|
+
const counts = new Map;
|
|
987
|
+
for (const character of value)
|
|
988
|
+
counts.set(character, (counts.get(character) ?? 0) + 1);
|
|
989
|
+
let entropy = 0;
|
|
990
|
+
for (const count of counts.values()) {
|
|
991
|
+
const probability = count / value.length;
|
|
992
|
+
entropy -= probability * Math.log2(probability);
|
|
993
|
+
}
|
|
994
|
+
return entropy >= 4.1;
|
|
995
|
+
}
|
|
996
|
+
|
|
997
|
+
// src/retrieval/derived.ts
|
|
998
|
+
function deriveDocument(source) {
|
|
999
|
+
const title = redactText(source.title);
|
|
1000
|
+
const summary = redactText(source.summary);
|
|
1001
|
+
const content = redactText(source.content);
|
|
1002
|
+
const contentHash = hashTuple("derived-note", 2, [source.kind, title.text, summary.text, content.text]);
|
|
1003
|
+
return {
|
|
1004
|
+
projectID: source.projectID,
|
|
1005
|
+
noteID: source.noteID,
|
|
1006
|
+
revision: source.revision,
|
|
1007
|
+
kind: source.kind,
|
|
1008
|
+
title: title.text,
|
|
1009
|
+
summary: summary.text,
|
|
1010
|
+
content: content.text,
|
|
1011
|
+
contentHash
|
|
1012
|
+
};
|
|
1013
|
+
}
|
|
1014
|
+
|
|
1015
|
+
// src/db/migrations/v011.ts
|
|
1016
|
+
var LEGACY_CAPTURE_SCHEMA = "agz-memory.capture/1";
|
|
1017
|
+
var CAPTURE_SCHEMAS = new Set([LEGACY_CAPTURE_SCHEMA, CAPTURE_SCHEMA]);
|
|
1018
|
+
var KINDS_SET = new Set(KINDS);
|
|
1019
|
+
var PREDICATES_SET = new Set(PREDICATES);
|
|
1020
|
+
function migrateV10ToV11(db) {
|
|
1021
|
+
const source = readSourceRows(db);
|
|
1022
|
+
const migrated = transformRows(source);
|
|
1023
|
+
replaceWithV11(db, migrated);
|
|
1024
|
+
console.warn(`[agz-memory] v10\u2192v11 migration complete: projects=${migrated.projects.length}, notes=${migrated.notes.length}, edges=${migrated.edges.length}, bindings=${migrated.bindings.length}, checkpoints=${migrated.checkpoints.length}, captures=${migrated.captures.length}, revisions=${migrated.revisions.length}, outbox=${migrated.outbox.length}`);
|
|
1025
|
+
}
|
|
1026
|
+
function assertV10SourceDatabase(db) {
|
|
1027
|
+
transformRows(readSourceRows(db));
|
|
1028
|
+
}
|
|
1029
|
+
function readSourceRows(db) {
|
|
1030
|
+
for (const table of [
|
|
1031
|
+
"projects",
|
|
1032
|
+
"notes",
|
|
1033
|
+
"note_edges",
|
|
1034
|
+
"project_bindings",
|
|
1035
|
+
"capture_checkpoints",
|
|
1036
|
+
"capture_events",
|
|
1037
|
+
"note_provenance",
|
|
1038
|
+
"note_revisions",
|
|
1039
|
+
"index_outbox",
|
|
1040
|
+
"schema_state"
|
|
1041
|
+
]) {
|
|
1042
|
+
requireTable(db, table);
|
|
1043
|
+
}
|
|
1044
|
+
if (!hasTable2(db, "notes_fts"))
|
|
1045
|
+
fail("notes_fts", "schema", "source_schema");
|
|
1046
|
+
validateSourceObjects(db);
|
|
1047
|
+
const states = selectRows(db, "schema_state", "SELECT version FROM schema_state");
|
|
1048
|
+
if (states.length !== 1 || states[0]?.version !== 10)
|
|
1049
|
+
fail("schema_state", "schema", "source_schema");
|
|
1050
|
+
const projects = selectRows(db, "projects", "SELECT id, name, normalized_name, created_at, updated_at FROM projects ORDER BY rowid");
|
|
1051
|
+
const notes = selectRows(db, "notes", `SELECT id, project_id, kind, title, summary, content, size_class, pinned, status,
|
|
1052
|
+
supersedes_id, current_revision, subject_key, content_hash, created_at, updated_at
|
|
1053
|
+
FROM notes ORDER BY rowid`);
|
|
1054
|
+
const edges = selectRows(db, "note_edges", "SELECT id, project_id, source_id, target_id, predicate, created_at FROM note_edges ORDER BY rowid");
|
|
1055
|
+
const bindings = selectRows(db, "project_bindings", `SELECT binding_key, project_id, source, source_project_id, workspace_id,
|
|
1056
|
+
canonical_path_hash, created_at, updated_at
|
|
1057
|
+
FROM project_bindings ORDER BY rowid`);
|
|
1058
|
+
const checkpoints = selectRows(db, "capture_checkpoints", `SELECT session_id, binding_key, project_id, state, last_message_id,
|
|
1059
|
+
last_reconciled_at, next_reconcile_at, failure_count, lease_owner,
|
|
1060
|
+
lease_expires_at, created_at, updated_at
|
|
1061
|
+
FROM capture_checkpoints ORDER BY rowid`);
|
|
1062
|
+
const captures = selectRows(db, "capture_events", `SELECT idempotency_key, contract, project_id, binding_key, event_kind,
|
|
1063
|
+
source_session_id, source_message_id, source_ordinal, source_tool_call_id,
|
|
1064
|
+
payload_json, payload_hash, redaction_version, state, attempt_count,
|
|
1065
|
+
note_id, last_error_code, generation, created_at, updated_at, processed_at
|
|
1066
|
+
FROM capture_events ORDER BY rowid`);
|
|
1067
|
+
const provenance = selectRows(db, "note_provenance", `SELECT id, project_id, note_id, source_type, capture_event_id,
|
|
1068
|
+
source_session_id, source_message_id, source_ordinal, source_tool_call_id,
|
|
1069
|
+
redaction_version, extractor_version, confidence, created_at
|
|
1070
|
+
FROM note_provenance ORDER BY rowid`);
|
|
1071
|
+
const revisions = selectRows(db, "note_revisions", `SELECT project_id, note_id, revision, kind, title, summary, content, size_class,
|
|
1072
|
+
pinned, status, supersedes_id, subject_key, content_hash, provenance_id, created_at
|
|
1073
|
+
FROM note_revisions ORDER BY rowid`);
|
|
1074
|
+
const outbox = selectRows(db, "index_outbox", `SELECT id, backend, operation, project_id, note_id, revision, content_hash,
|
|
1075
|
+
state, attempt_count, available_at, lease_owner, lease_expires_at,
|
|
1076
|
+
last_error_code, created_at, completed_at
|
|
1077
|
+
FROM index_outbox ORDER BY id`);
|
|
1078
|
+
return { projects, notes, edges, bindings, checkpoints, captures, provenance, revisions, outbox };
|
|
1079
|
+
}
|
|
1080
|
+
function validateSourceObjects(db) {
|
|
1081
|
+
const expected = new Set([
|
|
1082
|
+
"index\x00capture_checkpoints_due_idx",
|
|
1083
|
+
"index\x00capture_events_note_idx",
|
|
1084
|
+
"index\x00capture_events_session_idx",
|
|
1085
|
+
"index\x00capture_events_state_idx",
|
|
1086
|
+
"index\x00index_outbox_due_idx",
|
|
1087
|
+
"index\x00note_edges_source_idx",
|
|
1088
|
+
"index\x00note_edges_target_idx",
|
|
1089
|
+
"index\x00notes_active_subject_idx",
|
|
1090
|
+
"index\x00notes_project_idx",
|
|
1091
|
+
"table\x00capture_checkpoints",
|
|
1092
|
+
"table\x00capture_events",
|
|
1093
|
+
"table\x00index_outbox",
|
|
1094
|
+
"table\x00note_edges",
|
|
1095
|
+
"table\x00note_provenance",
|
|
1096
|
+
"table\x00note_revisions",
|
|
1097
|
+
"table\x00notes",
|
|
1098
|
+
"table\x00notes_fts",
|
|
1099
|
+
"table\x00notes_fts_config",
|
|
1100
|
+
"table\x00notes_fts_data",
|
|
1101
|
+
"table\x00notes_fts_docsize",
|
|
1102
|
+
"table\x00notes_fts_idx",
|
|
1103
|
+
"table\x00project_bindings",
|
|
1104
|
+
"table\x00projects",
|
|
1105
|
+
"table\x00schema_state",
|
|
1106
|
+
"trigger\x00notes_fts_ad",
|
|
1107
|
+
"trigger\x00notes_fts_ai",
|
|
1108
|
+
"trigger\x00notes_fts_au"
|
|
1109
|
+
]);
|
|
1110
|
+
const actual = db.query("SELECT type, name FROM sqlite_master WHERE name NOT LIKE 'sqlite_%'").all();
|
|
1111
|
+
if (actual.length !== expected.size || actual.some((object2) => !expected.has(`${object2.type}\x00${object2.name}`))) {
|
|
1112
|
+
fail("database", "schema", "source_schema");
|
|
1113
|
+
}
|
|
1114
|
+
}
|
|
1115
|
+
function transformRows(source) {
|
|
1116
|
+
const projectIDs = new Set;
|
|
1117
|
+
const normalizedProjectNames = new Set;
|
|
1118
|
+
for (const project of source.projects) {
|
|
1119
|
+
requireString(project.id, "projects", project.id, "project_id_invalid");
|
|
1120
|
+
requireString(project.name, "projects", project.id, "project_name_invalid");
|
|
1121
|
+
requireString(project.normalized_name, "projects", project.id, "project_name_invalid");
|
|
1122
|
+
requireInteger(project.created_at, "projects", project.id, "timestamp_invalid");
|
|
1123
|
+
requireInteger(project.updated_at, "projects", project.id, "timestamp_invalid");
|
|
1124
|
+
if (projectIDs.has(project.id))
|
|
1125
|
+
fail("projects", project.id, "duplicate_row");
|
|
1126
|
+
if (normalizedProjectNames.has(project.normalized_name)) {
|
|
1127
|
+
fail("projects", project.id, "duplicate_normalized_name");
|
|
1128
|
+
}
|
|
1129
|
+
projectIDs.add(project.id);
|
|
1130
|
+
normalizedProjectNames.add(project.normalized_name);
|
|
1131
|
+
}
|
|
1132
|
+
const noteMap = new Map;
|
|
1133
|
+
const noteIDs = new Set;
|
|
1134
|
+
for (const note of source.notes) {
|
|
1135
|
+
requireString(note.id, "notes", note.id, "note_id_invalid");
|
|
1136
|
+
requireString(note.project_id, "notes", note.id, "project_id_invalid");
|
|
1137
|
+
if (!projectIDs.has(note.project_id))
|
|
1138
|
+
fail("notes", note.id, "cross_project_reference");
|
|
1139
|
+
requireKind(note.kind, "notes", note.id);
|
|
1140
|
+
requireString(note.title, "notes", note.id, "note_value_invalid");
|
|
1141
|
+
requireString(note.summary, "notes", note.id, "note_value_invalid");
|
|
1142
|
+
requireString(note.content, "notes", note.id, "note_value_invalid");
|
|
1143
|
+
if (note.size_class !== "inline" && note.size_class !== "indexed") {
|
|
1144
|
+
fail("notes", note.id, "note_value_invalid");
|
|
1145
|
+
}
|
|
1146
|
+
if (note.pinned !== 0 && note.pinned !== 1)
|
|
1147
|
+
fail("notes", note.id, "note_value_invalid");
|
|
1148
|
+
if (!new Set(["active", "superseded", "archived"]).has(note.status)) {
|
|
1149
|
+
fail("notes", note.id, "note_value_invalid");
|
|
1150
|
+
}
|
|
1151
|
+
requireNullableString(note.supersedes_id, "notes", note.id, "note_value_invalid");
|
|
1152
|
+
requireIntegerAtLeast(note.current_revision, 1, "notes", note.id, "revision_invalid");
|
|
1153
|
+
requireNullableString(note.subject_key, "notes", note.id, "note_value_invalid");
|
|
1154
|
+
requireString(note.content_hash, "notes", note.id, "hash_invalid");
|
|
1155
|
+
const legacyHash = legacyNoteContentHash(note.kind, note.title, note.summary, note.content);
|
|
1156
|
+
const v2Hash = noteContentHash(note.kind, note.title, note.summary, note.content);
|
|
1157
|
+
if (note.content_hash !== legacyHash && note.content_hash !== v2Hash) {
|
|
1158
|
+
fail("notes", note.id, "hash_invalid");
|
|
1159
|
+
}
|
|
1160
|
+
requireInteger(note.created_at, "notes", note.id, "timestamp_invalid");
|
|
1161
|
+
requireInteger(note.updated_at, "notes", note.id, "timestamp_invalid");
|
|
1162
|
+
if (noteMap.has(noteKey(note.project_id, note.id)))
|
|
1163
|
+
fail("notes", note.id, "duplicate_row");
|
|
1164
|
+
noteMap.set(noteKey(note.project_id, note.id), note);
|
|
1165
|
+
noteIDs.add(note.id);
|
|
1166
|
+
}
|
|
1167
|
+
const activeSubjects = new Set;
|
|
1168
|
+
for (const note of source.notes) {
|
|
1169
|
+
if (note.supersedes_id !== null) {
|
|
1170
|
+
const superseded = noteMap.get(noteKey(note.project_id, note.supersedes_id));
|
|
1171
|
+
if (!superseded || note.supersedes_id === note.id) {
|
|
1172
|
+
fail("notes", note.id, "cross_project_reference");
|
|
1173
|
+
}
|
|
1174
|
+
}
|
|
1175
|
+
if (note.status === "active" && note.subject_key !== null) {
|
|
1176
|
+
const subject = hashTuple("active-subject", 2, [note.project_id, note.kind, note.subject_key]);
|
|
1177
|
+
if (activeSubjects.has(subject))
|
|
1178
|
+
fail("notes", note.id, "duplicate_active_subject");
|
|
1179
|
+
activeSubjects.add(subject);
|
|
1180
|
+
}
|
|
1181
|
+
}
|
|
1182
|
+
const edges = source.edges.map((edge) => {
|
|
1183
|
+
requireString(edge.id, "note_edges", edge.id, "edge_id_invalid");
|
|
1184
|
+
requireString(edge.project_id, "note_edges", edge.id, "project_id_invalid");
|
|
1185
|
+
requireString(edge.source_id, "note_edges", edge.id, "edge_value_invalid");
|
|
1186
|
+
requireString(edge.target_id, "note_edges", edge.id, "edge_value_invalid");
|
|
1187
|
+
const sourceNote = noteMap.get(noteKey(edge.project_id, edge.source_id));
|
|
1188
|
+
const targetNote = noteMap.get(noteKey(edge.project_id, edge.target_id));
|
|
1189
|
+
if (!projectIDs.has(edge.project_id) || !sourceNote || !targetNote) {
|
|
1190
|
+
fail("note_edges", edge.id, "cross_project_reference");
|
|
1191
|
+
}
|
|
1192
|
+
if (!PREDICATES_SET.has(edge.predicate))
|
|
1193
|
+
fail("note_edges", edge.id, "edge_value_invalid");
|
|
1194
|
+
requireInteger(edge.created_at, "note_edges", edge.id, "timestamp_invalid");
|
|
1195
|
+
return edge;
|
|
1196
|
+
});
|
|
1197
|
+
const bindingMap = new Map;
|
|
1198
|
+
const bindingProjects = new Map;
|
|
1199
|
+
const bindingKeys = new Set;
|
|
1200
|
+
const bindings = source.bindings.map((binding) => {
|
|
1201
|
+
requireHash(binding.binding_key, "project_bindings", binding.binding_key, "binding_id_invalid");
|
|
1202
|
+
requireString(binding.project_id, "project_bindings", binding.binding_key, "project_id_invalid");
|
|
1203
|
+
if (!projectIDs.has(binding.project_id)) {
|
|
1204
|
+
fail("project_bindings", binding.binding_key, "cross_project_reference");
|
|
1205
|
+
}
|
|
1206
|
+
if (binding.source !== "opencode-v2")
|
|
1207
|
+
fail("project_bindings", binding.binding_key, "binding_value_invalid");
|
|
1208
|
+
requireString(binding.source_project_id, "project_bindings", binding.binding_key, "binding_value_invalid");
|
|
1209
|
+
requireString(binding.workspace_id, "project_bindings", binding.binding_key, "binding_value_invalid");
|
|
1210
|
+
requireHash(binding.canonical_path_hash, "project_bindings", binding.binding_key, "hash_invalid");
|
|
1211
|
+
requireInteger(binding.created_at, "project_bindings", binding.binding_key, "timestamp_invalid");
|
|
1212
|
+
requireInteger(binding.updated_at, "project_bindings", binding.binding_key, "timestamp_invalid");
|
|
1213
|
+
const nextKey = hashTuple("project-binding", 2, [
|
|
1214
|
+
binding.source,
|
|
1215
|
+
binding.source_project_id,
|
|
1216
|
+
binding.workspace_id,
|
|
1217
|
+
binding.canonical_path_hash
|
|
1218
|
+
]);
|
|
1219
|
+
if (bindingKeys.has(nextKey))
|
|
1220
|
+
fail("project_bindings", binding.binding_key, "identity_collision");
|
|
1221
|
+
bindingKeys.add(nextKey);
|
|
1222
|
+
bindingMap.set(binding.binding_key, nextKey);
|
|
1223
|
+
bindingProjects.set(binding.binding_key, binding.project_id);
|
|
1224
|
+
return { ...binding, binding_key: nextKey };
|
|
1225
|
+
});
|
|
1226
|
+
const checkpointIdentities = new Set;
|
|
1227
|
+
const checkpoints = source.checkpoints.map((checkpoint) => {
|
|
1228
|
+
requireNonEmptyString(checkpoint.session_id, "capture_checkpoints", checkpoint.session_id, "checkpoint_id_invalid");
|
|
1229
|
+
requireHash(checkpoint.binding_key, "capture_checkpoints", checkpoint.session_id, "checkpoint_value_invalid");
|
|
1230
|
+
requireString(checkpoint.project_id, "capture_checkpoints", checkpoint.session_id, "project_id_invalid");
|
|
1231
|
+
const nextBinding = bindingMap.get(checkpoint.binding_key);
|
|
1232
|
+
if (!nextBinding || checkpoint.project_id !== bindingProjects.get(checkpoint.binding_key)) {
|
|
1233
|
+
fail("capture_checkpoints", checkpoint.session_id, "cross_project_reference");
|
|
1234
|
+
}
|
|
1235
|
+
if (checkpoint.state !== "active" && checkpoint.state !== "idle" && checkpoint.state !== "unavailable" && checkpoint.state !== "closed") {
|
|
1236
|
+
fail("capture_checkpoints", checkpoint.session_id, "checkpoint_value_invalid");
|
|
1237
|
+
}
|
|
1238
|
+
requireNullableString(checkpoint.last_message_id, "capture_checkpoints", checkpoint.session_id, "checkpoint_value_invalid");
|
|
1239
|
+
requireNullableInteger(checkpoint.last_reconciled_at, "capture_checkpoints", checkpoint.session_id, "timestamp_invalid");
|
|
1240
|
+
requireInteger(checkpoint.next_reconcile_at, "capture_checkpoints", checkpoint.session_id, "timestamp_invalid");
|
|
1241
|
+
requireIntegerAtLeast(checkpoint.failure_count, 0, "capture_checkpoints", checkpoint.session_id, "checkpoint_value_invalid");
|
|
1242
|
+
requireNullableString(checkpoint.lease_owner, "capture_checkpoints", checkpoint.session_id, "checkpoint_value_invalid");
|
|
1243
|
+
requireNullableInteger(checkpoint.lease_expires_at, "capture_checkpoints", checkpoint.session_id, "timestamp_invalid");
|
|
1244
|
+
requireInteger(checkpoint.created_at, "capture_checkpoints", checkpoint.session_id, "timestamp_invalid");
|
|
1245
|
+
requireInteger(checkpoint.updated_at, "capture_checkpoints", checkpoint.session_id, "timestamp_invalid");
|
|
1246
|
+
const identity = hashTuple("checkpoint-identity", 2, [nextBinding, checkpoint.session_id]);
|
|
1247
|
+
if (checkpointIdentities.has(identity)) {
|
|
1248
|
+
fail("capture_checkpoints", checkpoint.session_id, "identity_collision");
|
|
1249
|
+
}
|
|
1250
|
+
checkpointIdentities.add(identity);
|
|
1251
|
+
return { ...checkpoint, binding_key: nextBinding, last_message_id: null };
|
|
1252
|
+
});
|
|
1253
|
+
const captureMap = new Map;
|
|
1254
|
+
const captureProjects = new Map;
|
|
1255
|
+
const captureKeys = new Set;
|
|
1256
|
+
const captures = source.captures.map((capture) => {
|
|
1257
|
+
requireHash(capture.idempotency_key, "capture_events", capture.idempotency_key, "capture_id_invalid");
|
|
1258
|
+
requireString(capture.project_id, "capture_events", capture.idempotency_key, "project_id_invalid");
|
|
1259
|
+
requireHash(capture.binding_key, "capture_events", capture.idempotency_key, "binding_id_invalid");
|
|
1260
|
+
const nextBinding = bindingMap.get(capture.binding_key);
|
|
1261
|
+
const bindingProjectID = bindingProjects.get(capture.binding_key);
|
|
1262
|
+
if (!nextBinding || !bindingProjectID || capture.project_id !== bindingProjectID) {
|
|
1263
|
+
fail("capture_events", capture.idempotency_key, "cross_project_reference");
|
|
1264
|
+
}
|
|
1265
|
+
const migrated = migrateCapture(capture, nextBinding, noteMap, noteIDs);
|
|
1266
|
+
if (captureKeys.has(migrated.idempotency_key)) {
|
|
1267
|
+
fail("capture_events", capture.idempotency_key, "identity_collision");
|
|
1268
|
+
}
|
|
1269
|
+
captureKeys.add(migrated.idempotency_key);
|
|
1270
|
+
captureMap.set(capture.idempotency_key, migrated.idempotency_key);
|
|
1271
|
+
captureProjects.set(capture.idempotency_key, capture.project_id);
|
|
1272
|
+
return migrated;
|
|
1273
|
+
});
|
|
1274
|
+
const provenanceKeys = new Set;
|
|
1275
|
+
const provenance = source.provenance.map((row) => {
|
|
1276
|
+
requireString(row.id, "note_provenance", row.id, "provenance_id_invalid");
|
|
1277
|
+
requireString(row.project_id, "note_provenance", row.id, "project_id_invalid");
|
|
1278
|
+
requireString(row.note_id, "note_provenance", row.id, "note_id_invalid");
|
|
1279
|
+
if (!noteMap.has(noteKey(row.project_id, row.note_id))) {
|
|
1280
|
+
fail("note_provenance", row.id, "cross_project_reference");
|
|
1281
|
+
}
|
|
1282
|
+
if (!new Set(["mcp-manual", "opencode-capture", "migration", "legacy-import", "admin"]).has(row.source_type)) {
|
|
1283
|
+
fail("note_provenance", row.id, "provenance_value_invalid");
|
|
1284
|
+
}
|
|
1285
|
+
requireNullableString(row.capture_event_id, "note_provenance", row.id, "provenance_value_invalid");
|
|
1286
|
+
if (row.capture_event_id !== null) {
|
|
1287
|
+
if (!captureMap.has(row.capture_event_id))
|
|
1288
|
+
fail("note_provenance", row.id, "capture_reference_invalid");
|
|
1289
|
+
if (captureProjects.get(row.capture_event_id) !== row.project_id) {
|
|
1290
|
+
fail("note_provenance", row.id, "cross_project_reference");
|
|
1291
|
+
}
|
|
1292
|
+
}
|
|
1293
|
+
requireNullableString(row.source_session_id, "note_provenance", row.id, "provenance_value_invalid");
|
|
1294
|
+
requireNullableString(row.source_message_id, "note_provenance", row.id, "provenance_value_invalid");
|
|
1295
|
+
requireNullableInteger(row.source_ordinal, "note_provenance", row.id, "provenance_value_invalid");
|
|
1296
|
+
requireNullableString(row.source_tool_call_id, "note_provenance", row.id, "provenance_value_invalid");
|
|
1297
|
+
requireNullableString(row.redaction_version, "note_provenance", row.id, "provenance_value_invalid");
|
|
1298
|
+
requireNullableString(row.extractor_version, "note_provenance", row.id, "provenance_value_invalid");
|
|
1299
|
+
if (row.confidence !== null && (!Number.isFinite(row.confidence) || row.confidence < 0 || row.confidence > 1)) {
|
|
1300
|
+
fail("note_provenance", row.id, "provenance_value_invalid");
|
|
1301
|
+
}
|
|
1302
|
+
requireInteger(row.created_at, "note_provenance", row.id, "timestamp_invalid");
|
|
1303
|
+
const key = noteKey(row.project_id, row.id);
|
|
1304
|
+
if (provenanceKeys.has(key))
|
|
1305
|
+
fail("note_provenance", row.id, "duplicate_row");
|
|
1306
|
+
provenanceKeys.add(key);
|
|
1307
|
+
return {
|
|
1308
|
+
...row,
|
|
1309
|
+
capture_event_id: row.capture_event_id === null ? null : captureMap.get(row.capture_event_id)
|
|
1310
|
+
};
|
|
1311
|
+
});
|
|
1312
|
+
const provenanceMap = new Map(provenance.map((row) => [noteKey(row.project_id, row.id), row]));
|
|
1313
|
+
const revisionMap = new Map;
|
|
1314
|
+
const revisions = source.revisions.map((revision) => {
|
|
1315
|
+
requireString(revision.project_id, "note_revisions", revision.note_id, "project_id_invalid");
|
|
1316
|
+
requireString(revision.note_id, "note_revisions", revision.note_id, "note_id_invalid");
|
|
1317
|
+
const note = noteMap.get(noteKey(revision.project_id, revision.note_id));
|
|
1318
|
+
const provenanceRow = provenanceMap.get(noteKey(revision.project_id, revision.provenance_id));
|
|
1319
|
+
if (!note || !provenanceRow)
|
|
1320
|
+
fail("note_revisions", revision.note_id, "cross_project_reference");
|
|
1321
|
+
requireIntegerAtLeast(revision.revision, 1, "note_revisions", revision.note_id, "revision_invalid");
|
|
1322
|
+
requireKind(revision.kind, "note_revisions", revision.note_id);
|
|
1323
|
+
requireString(revision.title, "note_revisions", revision.note_id, "revision_value_invalid");
|
|
1324
|
+
requireString(revision.summary, "note_revisions", revision.note_id, "revision_value_invalid");
|
|
1325
|
+
requireString(revision.content, "note_revisions", revision.note_id, "revision_value_invalid");
|
|
1326
|
+
if (revision.size_class !== "inline" && revision.size_class !== "indexed") {
|
|
1327
|
+
fail("note_revisions", revision.note_id, "revision_value_invalid");
|
|
1328
|
+
}
|
|
1329
|
+
if (revision.pinned !== 0 && revision.pinned !== 1)
|
|
1330
|
+
fail("note_revisions", revision.note_id, "revision_value_invalid");
|
|
1331
|
+
if (!new Set(["active", "superseded", "archived"]).has(revision.status)) {
|
|
1332
|
+
fail("note_revisions", revision.note_id, "revision_value_invalid");
|
|
1333
|
+
}
|
|
1334
|
+
requireNullableString(revision.supersedes_id, "note_revisions", revision.note_id, "revision_value_invalid");
|
|
1335
|
+
if (revision.supersedes_id !== null) {
|
|
1336
|
+
if (revision.supersedes_id === revision.note_id) {
|
|
1337
|
+
fail("note_revisions", revision.note_id, "revision_value_invalid");
|
|
1338
|
+
}
|
|
1339
|
+
if (!noteMap.has(noteKey(revision.project_id, revision.supersedes_id))) {
|
|
1340
|
+
fail("note_revisions", revision.note_id, "cross_project_reference");
|
|
1341
|
+
}
|
|
1342
|
+
}
|
|
1343
|
+
requireNullableString(revision.subject_key, "note_revisions", revision.note_id, "revision_value_invalid");
|
|
1344
|
+
requireString(revision.content_hash, "note_revisions", revision.note_id, "hash_invalid");
|
|
1345
|
+
const legacyHash = legacyNoteContentHash(revision.kind, revision.title, revision.summary, revision.content);
|
|
1346
|
+
const v2Hash = noteContentHash(revision.kind, revision.title, revision.summary, revision.content);
|
|
1347
|
+
if (revision.content_hash !== legacyHash && revision.content_hash !== v2Hash) {
|
|
1348
|
+
fail("note_revisions", revision.note_id, "hash_invalid");
|
|
1349
|
+
}
|
|
1350
|
+
requireInteger(revision.created_at, "note_revisions", revision.note_id, "timestamp_invalid");
|
|
1351
|
+
const key = revisionKey(revision.project_id, revision.note_id, revision.revision);
|
|
1352
|
+
if (revisionMap.has(key))
|
|
1353
|
+
fail("note_revisions", revision.note_id, "duplicate_row");
|
|
1354
|
+
const migrated = {
|
|
1355
|
+
...revision,
|
|
1356
|
+
content_hash: noteContentHash(revision.kind, revision.title, revision.summary, revision.content)
|
|
1357
|
+
};
|
|
1358
|
+
revisionMap.set(key, migrated);
|
|
1359
|
+
return migrated;
|
|
1360
|
+
});
|
|
1361
|
+
for (const note of source.notes) {
|
|
1362
|
+
const current = revisionMap.get(revisionKey(note.project_id, note.id, note.current_revision));
|
|
1363
|
+
if (!current)
|
|
1364
|
+
fail("note_revisions", note.id, "revision_invalid");
|
|
1365
|
+
for (let revision = 1;revision <= note.current_revision; revision++) {
|
|
1366
|
+
if (!revisionMap.has(revisionKey(note.project_id, note.id, revision))) {
|
|
1367
|
+
fail("note_revisions", note.id, "revision_invalid");
|
|
1368
|
+
}
|
|
1369
|
+
}
|
|
1370
|
+
if (current.kind !== note.kind || current.title !== note.title || current.summary !== note.summary || current.content !== note.content || current.size_class !== note.size_class || current.pinned !== note.pinned || current.status !== note.status || current.supersedes_id !== note.supersedes_id || current.subject_key !== note.subject_key || current.content_hash !== noteContentHash(note.kind, note.title, note.summary, note.content)) {
|
|
1371
|
+
fail("note_revisions", note.id, "revision_invalid");
|
|
1372
|
+
}
|
|
1373
|
+
}
|
|
1374
|
+
const activeOperations = new Set;
|
|
1375
|
+
const outbox = source.outbox.map((row) => {
|
|
1376
|
+
requireIntegerAtLeast(row.id, 1, "index_outbox", String(row.id), "outbox_value_invalid");
|
|
1377
|
+
requireNonEmptyString(row.backend, "index_outbox", String(row.id), "outbox_value_invalid");
|
|
1378
|
+
requireString(row.operation, "index_outbox", String(row.id), "outbox_operation_invalid");
|
|
1379
|
+
requireString(row.project_id, "index_outbox", String(row.id), "project_id_invalid");
|
|
1380
|
+
requireNullableString(row.note_id, "index_outbox", String(row.id), "outbox_operation_invalid");
|
|
1381
|
+
requireNullableInteger(row.revision, "index_outbox", String(row.id), "outbox_operation_invalid");
|
|
1382
|
+
requireNullableString(row.content_hash, "index_outbox", String(row.id), "outbox_operation_invalid");
|
|
1383
|
+
if (row.state !== "pending" && row.state !== "leased" && row.state !== "succeeded" && row.state !== "dead") {
|
|
1384
|
+
fail("index_outbox", String(row.id), "outbox_value_invalid");
|
|
1385
|
+
}
|
|
1386
|
+
requireIntegerAtLeast(row.attempt_count, 0, "index_outbox", String(row.id), "outbox_value_invalid");
|
|
1387
|
+
requireInteger(row.available_at, "index_outbox", String(row.id), "timestamp_invalid");
|
|
1388
|
+
requireNullableString(row.lease_owner, "index_outbox", String(row.id), "outbox_value_invalid");
|
|
1389
|
+
requireNullableInteger(row.lease_expires_at, "index_outbox", String(row.id), "timestamp_invalid");
|
|
1390
|
+
requireNullableString(row.last_error_code, "index_outbox", String(row.id), "outbox_value_invalid");
|
|
1391
|
+
requireInteger(row.created_at, "index_outbox", String(row.id), "timestamp_invalid");
|
|
1392
|
+
requireNullableInteger(row.completed_at, "index_outbox", String(row.id), "timestamp_invalid");
|
|
1393
|
+
let contentHash = null;
|
|
1394
|
+
let state = row.state;
|
|
1395
|
+
let leaseOwner = row.lease_owner;
|
|
1396
|
+
let leaseExpiresAt = row.lease_expires_at;
|
|
1397
|
+
let completedAt = row.completed_at;
|
|
1398
|
+
if (row.operation === "upsert-note") {
|
|
1399
|
+
if (row.note_id === null || row.revision === null || row.content_hash === null) {
|
|
1400
|
+
fail("index_outbox", String(row.id), "outbox_operation_invalid");
|
|
1401
|
+
}
|
|
1402
|
+
requireNonEmptyString(row.note_id, "index_outbox", String(row.id), "outbox_operation_invalid");
|
|
1403
|
+
requireIntegerAtLeast(row.revision, 1, "index_outbox", String(row.id), "outbox_operation_invalid");
|
|
1404
|
+
requireHash(row.content_hash, "index_outbox", String(row.id), "hash_invalid");
|
|
1405
|
+
const revision = revisionMap.get(revisionKey(row.project_id, row.note_id, row.revision));
|
|
1406
|
+
if (!revision) {
|
|
1407
|
+
contentHash = row.content_hash;
|
|
1408
|
+
if (state === "pending" || state === "leased") {
|
|
1409
|
+
state = "succeeded";
|
|
1410
|
+
leaseOwner = null;
|
|
1411
|
+
leaseExpiresAt = null;
|
|
1412
|
+
completedAt = row.completed_at ?? row.created_at;
|
|
1413
|
+
}
|
|
1414
|
+
} else {
|
|
1415
|
+
const document = deriveDocument({
|
|
1416
|
+
projectID: revision.project_id,
|
|
1417
|
+
noteID: revision.note_id,
|
|
1418
|
+
revision: revision.revision,
|
|
1419
|
+
kind: revision.kind,
|
|
1420
|
+
title: revision.title,
|
|
1421
|
+
summary: revision.summary,
|
|
1422
|
+
content: revision.content
|
|
1423
|
+
});
|
|
1424
|
+
if (!document)
|
|
1425
|
+
fail("index_outbox", String(row.id), "derived_identity_unavailable");
|
|
1426
|
+
const legacyContentHash = legacyNoteContentHash(document.kind, document.title, document.summary, document.content);
|
|
1427
|
+
if (row.content_hash !== document.contentHash && row.content_hash !== legacyContentHash) {
|
|
1428
|
+
fail("index_outbox", String(row.id), "hash_invalid");
|
|
1429
|
+
}
|
|
1430
|
+
contentHash = document.contentHash;
|
|
1431
|
+
}
|
|
1432
|
+
} else if (row.operation === "delete-note") {
|
|
1433
|
+
if (row.note_id === null || row.revision === null) {
|
|
1434
|
+
fail("index_outbox", String(row.id), "outbox_operation_invalid");
|
|
1435
|
+
}
|
|
1436
|
+
requireNonEmptyString(row.note_id, "index_outbox", String(row.id), "outbox_operation_invalid");
|
|
1437
|
+
requireIntegerAtLeast(row.revision, 1, "index_outbox", String(row.id), "outbox_operation_invalid");
|
|
1438
|
+
if (row.content_hash !== null) {
|
|
1439
|
+
requireHash(row.content_hash, "index_outbox", String(row.id), "hash_invalid");
|
|
1440
|
+
}
|
|
1441
|
+
} else if (row.operation === "purge-project") {
|
|
1442
|
+
if (row.note_id !== null || row.revision !== null || row.content_hash !== null) {
|
|
1443
|
+
fail("index_outbox", String(row.id), "outbox_operation_invalid");
|
|
1444
|
+
}
|
|
1445
|
+
} else {
|
|
1446
|
+
fail("index_outbox", String(row.id), "outbox_operation_invalid");
|
|
1447
|
+
}
|
|
1448
|
+
if (state === "leased")
|
|
1449
|
+
state = "pending";
|
|
1450
|
+
leaseOwner = null;
|
|
1451
|
+
leaseExpiresAt = null;
|
|
1452
|
+
let generation = 0;
|
|
1453
|
+
if (state === "pending" || state === "leased") {
|
|
1454
|
+
let activeKey = hashTuple("outbox-active", 2, [
|
|
1455
|
+
row.backend,
|
|
1456
|
+
row.operation,
|
|
1457
|
+
row.project_id,
|
|
1458
|
+
row.note_id,
|
|
1459
|
+
row.revision,
|
|
1460
|
+
generation
|
|
1461
|
+
]);
|
|
1462
|
+
while (activeOperations.has(activeKey)) {
|
|
1463
|
+
generation++;
|
|
1464
|
+
activeKey = hashTuple("outbox-active", 2, [
|
|
1465
|
+
row.backend,
|
|
1466
|
+
row.operation,
|
|
1467
|
+
row.project_id,
|
|
1468
|
+
row.note_id,
|
|
1469
|
+
row.revision,
|
|
1470
|
+
generation
|
|
1471
|
+
]);
|
|
1472
|
+
}
|
|
1473
|
+
activeOperations.add(activeKey);
|
|
1474
|
+
}
|
|
1475
|
+
const operationKey = hashTuple("outbox-operation", 2, [
|
|
1476
|
+
row.backend,
|
|
1477
|
+
row.operation,
|
|
1478
|
+
row.project_id,
|
|
1479
|
+
row.note_id,
|
|
1480
|
+
row.revision,
|
|
1481
|
+
contentHash,
|
|
1482
|
+
generation
|
|
1483
|
+
]);
|
|
1484
|
+
return {
|
|
1485
|
+
...row,
|
|
1486
|
+
operation_key: operationKey,
|
|
1487
|
+
content_hash: contentHash,
|
|
1488
|
+
state,
|
|
1489
|
+
lease_owner: leaseOwner,
|
|
1490
|
+
lease_expires_at: leaseExpiresAt,
|
|
1491
|
+
completed_at: completedAt,
|
|
1492
|
+
generation,
|
|
1493
|
+
lease_generation: 0,
|
|
1494
|
+
fence: 0,
|
|
1495
|
+
heartbeat_at: null
|
|
1496
|
+
};
|
|
1497
|
+
});
|
|
1498
|
+
const migratedNotes = source.notes.map((note) => ({
|
|
1499
|
+
...note,
|
|
1500
|
+
content_hash: noteContentHash(note.kind, note.title, note.summary, note.content)
|
|
1501
|
+
}));
|
|
1502
|
+
return {
|
|
1503
|
+
projects: source.projects,
|
|
1504
|
+
notes: migratedNotes,
|
|
1505
|
+
edges,
|
|
1506
|
+
bindings,
|
|
1507
|
+
checkpoints,
|
|
1508
|
+
captures,
|
|
1509
|
+
provenance,
|
|
1510
|
+
revisions,
|
|
1511
|
+
outbox,
|
|
1512
|
+
bindingMap,
|
|
1513
|
+
captureMap
|
|
1514
|
+
};
|
|
1515
|
+
}
|
|
1516
|
+
function migrateCapture(row, bindingKey, notes, noteIDs) {
|
|
1517
|
+
requireHash(row.idempotency_key, "capture_events", row.idempotency_key, "capture_id_invalid");
|
|
1518
|
+
if (!CAPTURE_SCHEMAS.has(row.contract))
|
|
1519
|
+
fail("capture_events", row.idempotency_key, "capture_contract_invalid");
|
|
1520
|
+
requireString(row.project_id, "capture_events", row.idempotency_key, "project_id_invalid");
|
|
1521
|
+
requireString(row.event_kind, "capture_events", row.idempotency_key, "capture_identity_invalid");
|
|
1522
|
+
requireNonEmptyString(row.source_session_id, "capture_events", row.idempotency_key, "capture_identity_invalid");
|
|
1523
|
+
requireNullableString(row.source_message_id, "capture_events", row.idempotency_key, "capture_identity_invalid");
|
|
1524
|
+
requireNullableInteger(row.source_ordinal, "capture_events", row.idempotency_key, "capture_identity_invalid");
|
|
1525
|
+
requireNullableString(row.source_tool_call_id, "capture_events", row.idempotency_key, "capture_identity_invalid");
|
|
1526
|
+
requireString(row.redaction_version, "capture_events", row.idempotency_key, "capture_value_invalid");
|
|
1527
|
+
if (!new Set(["pending", "shadowed", "review", "materialized", "duplicate", "ignored", "rejected", "quarantined", "failed", "dead"]).has(row.state)) {
|
|
1528
|
+
fail("capture_events", row.idempotency_key, "capture_value_invalid");
|
|
1529
|
+
}
|
|
1530
|
+
requireIntegerAtLeast(row.attempt_count, 0, "capture_events", row.idempotency_key, "capture_value_invalid");
|
|
1531
|
+
if (row.note_id !== null && notes.get(noteKey(row.project_id, row.note_id)) === undefined) {
|
|
1532
|
+
if (noteIDs.has(row.note_id)) {
|
|
1533
|
+
fail("capture_events", row.idempotency_key, "cross_project_reference");
|
|
1534
|
+
}
|
|
1535
|
+
}
|
|
1536
|
+
requireNullableString(row.note_id, "capture_events", row.idempotency_key, "capture_value_invalid");
|
|
1537
|
+
requireNullableString(row.last_error_code, "capture_events", row.idempotency_key, "capture_value_invalid");
|
|
1538
|
+
requireIntegerAtLeast(row.generation, 0, "capture_events", row.idempotency_key, "capture_value_invalid");
|
|
1539
|
+
requireInteger(row.created_at, "capture_events", row.idempotency_key, "timestamp_invalid");
|
|
1540
|
+
requireInteger(row.updated_at, "capture_events", row.idempotency_key, "timestamp_invalid");
|
|
1541
|
+
requireNullableInteger(row.processed_at, "capture_events", row.idempotency_key, "timestamp_invalid");
|
|
1542
|
+
requireNullableString(row.payload_json, "capture_events", row.idempotency_key, "payload_value_invalid");
|
|
1543
|
+
requireNullableString(row.payload_hash, "capture_events", row.idempotency_key, "payload_value_invalid");
|
|
1544
|
+
if (row.payload_json === null) {
|
|
1545
|
+
if (row.payload_hash !== null)
|
|
1546
|
+
fail("capture_events", row.idempotency_key, "payload_identity_unavailable");
|
|
1547
|
+
let key;
|
|
1548
|
+
try {
|
|
1549
|
+
assertLegacyCaptureKey(row);
|
|
1550
|
+
key = captureKeyFromColumns(row, bindingKey);
|
|
1551
|
+
} catch {
|
|
1552
|
+
fail("capture_events", row.idempotency_key, "capture_identity_invalid");
|
|
1553
|
+
}
|
|
1554
|
+
return { ...row, idempotency_key: key, contract: CAPTURE_SCHEMA, binding_key: bindingKey };
|
|
1555
|
+
}
|
|
1556
|
+
if (row.payload_hash === null)
|
|
1557
|
+
fail("capture_events", row.idempotency_key, "payload_identity_unavailable");
|
|
1558
|
+
const legacyPayloadHash = sha256(row.payload_json);
|
|
1559
|
+
const v2PayloadHash = hashTuple("capture-payload", 2, [row.payload_json]);
|
|
1560
|
+
if (row.payload_hash !== legacyPayloadHash && row.payload_hash !== v2PayloadHash) {
|
|
1561
|
+
fail("capture_events", row.idempotency_key, "payload_identity_mismatch");
|
|
1562
|
+
}
|
|
1563
|
+
let raw;
|
|
1564
|
+
try {
|
|
1565
|
+
const value = JSON.parse(row.payload_json);
|
|
1566
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
1567
|
+
throw new Error("not an object");
|
|
1568
|
+
raw = value;
|
|
1569
|
+
} catch {
|
|
1570
|
+
fail("capture_events", row.idempotency_key, "payload_invalid");
|
|
1571
|
+
}
|
|
1572
|
+
if (raw.schema !== row.contract || raw.idempotencyKey !== row.idempotency_key) {
|
|
1573
|
+
fail("capture_events", row.idempotency_key, "payload_identity_mismatch");
|
|
1574
|
+
}
|
|
1575
|
+
if (raw.projectID !== row.project_id || raw.bindingKey !== row.binding_key || raw.kind !== row.event_kind) {
|
|
1576
|
+
fail("capture_events", row.idempotency_key, "cross_project_reference");
|
|
1577
|
+
}
|
|
1578
|
+
const source = raw.source;
|
|
1579
|
+
if (!source || typeof source !== "object" || Array.isArray(source)) {
|
|
1580
|
+
fail("capture_events", row.idempotency_key, "capture_identity_invalid");
|
|
1581
|
+
}
|
|
1582
|
+
let nextKey;
|
|
1583
|
+
try {
|
|
1584
|
+
nextKey = captureKeyFromPayload(row.event_kind, bindingKey, source, raw.signal);
|
|
1585
|
+
} catch {
|
|
1586
|
+
fail("capture_events", row.idempotency_key, "capture_identity_invalid");
|
|
1587
|
+
}
|
|
1588
|
+
const candidate = {
|
|
1589
|
+
...raw,
|
|
1590
|
+
schema: CAPTURE_SCHEMA,
|
|
1591
|
+
idempotencyKey: nextKey,
|
|
1592
|
+
projectID: row.project_id,
|
|
1593
|
+
bindingKey
|
|
1594
|
+
};
|
|
1595
|
+
let parsed;
|
|
1596
|
+
try {
|
|
1597
|
+
parsed = parseCaptureEvent(candidate);
|
|
1598
|
+
} catch {
|
|
1599
|
+
fail("capture_events", row.idempotency_key, "capture_identity_invalid");
|
|
1600
|
+
}
|
|
1601
|
+
if (parsed.kind !== row.event_kind || parsed.source.sessionID !== row.source_session_id || (parsed.source.messageID ?? null) !== row.source_message_id || (parsed.source.ordinal ?? null) !== row.source_ordinal || (parsed.source.toolCallID ?? null) !== row.source_tool_call_id) {
|
|
1602
|
+
fail("capture_events", row.idempotency_key, "capture_identity_mismatch");
|
|
1603
|
+
}
|
|
1604
|
+
const payload = JSON.stringify(parsed);
|
|
1605
|
+
return {
|
|
1606
|
+
...row,
|
|
1607
|
+
idempotency_key: nextKey,
|
|
1608
|
+
contract: CAPTURE_SCHEMA,
|
|
1609
|
+
binding_key: bindingKey,
|
|
1610
|
+
payload_json: payload,
|
|
1611
|
+
payload_hash: capturePayloadHash(parsed)
|
|
1612
|
+
};
|
|
1613
|
+
}
|
|
1614
|
+
function captureKeyFromColumns(row, bindingKey) {
|
|
1615
|
+
if (row.event_kind === "user-candidate") {
|
|
1616
|
+
requireSourceMessage(row.source_message_id, "capture_events", row.idempotency_key);
|
|
1617
|
+
if (row.source_ordinal !== null || row.source_tool_call_id !== null) {
|
|
1618
|
+
fail("capture_events", row.idempotency_key, "capture_identity_invalid");
|
|
1619
|
+
}
|
|
1620
|
+
return captureIdempotencyKey({
|
|
1621
|
+
kind: "user",
|
|
1622
|
+
bindingKey,
|
|
1623
|
+
sessionID: row.source_session_id,
|
|
1624
|
+
messageID: row.source_message_id
|
|
1625
|
+
});
|
|
1626
|
+
}
|
|
1627
|
+
if (row.event_kind === "assistant-candidate") {
|
|
1628
|
+
requireSourceMessage(row.source_message_id, "capture_events", row.idempotency_key);
|
|
1629
|
+
if (row.source_ordinal === null || row.source_tool_call_id !== null) {
|
|
1630
|
+
fail("capture_events", row.idempotency_key, "capture_identity_invalid");
|
|
1631
|
+
}
|
|
1632
|
+
return captureIdempotencyKey({
|
|
1633
|
+
kind: "assistant",
|
|
1634
|
+
bindingKey,
|
|
1635
|
+
sessionID: row.source_session_id,
|
|
1636
|
+
assistantMessageID: row.source_message_id,
|
|
1637
|
+
ordinal: row.source_ordinal
|
|
1638
|
+
});
|
|
1639
|
+
}
|
|
1640
|
+
if (row.event_kind === "session-summary") {
|
|
1641
|
+
requireSourceMessage(row.source_message_id, "capture_events", row.idempotency_key);
|
|
1642
|
+
if (row.source_ordinal !== null || row.source_tool_call_id !== null) {
|
|
1643
|
+
fail("capture_events", row.idempotency_key, "capture_identity_invalid");
|
|
1644
|
+
}
|
|
1645
|
+
return captureIdempotencyKey({
|
|
1646
|
+
kind: "summary",
|
|
1647
|
+
bindingKey,
|
|
1648
|
+
sessionID: row.source_session_id,
|
|
1649
|
+
checkpointMessageID: row.source_message_id
|
|
1650
|
+
});
|
|
1651
|
+
}
|
|
1652
|
+
if (row.event_kind === "tool-signal") {
|
|
1653
|
+
requireSourceMessage(row.source_message_id, "capture_events", row.idempotency_key);
|
|
1654
|
+
if (row.source_ordinal !== null || !row.source_tool_call_id) {
|
|
1655
|
+
fail("capture_events", row.idempotency_key, "capture_identity_invalid");
|
|
1656
|
+
}
|
|
1657
|
+
const status = legacyToolStatus(row);
|
|
1658
|
+
return captureIdempotencyKey({
|
|
1659
|
+
kind: "tool",
|
|
1660
|
+
bindingKey,
|
|
1661
|
+
sessionID: row.source_session_id,
|
|
1662
|
+
assistantMessageID: row.source_message_id,
|
|
1663
|
+
toolCallID: row.source_tool_call_id,
|
|
1664
|
+
terminalStatus: status
|
|
1665
|
+
});
|
|
1666
|
+
}
|
|
1667
|
+
fail("capture_events", row.idempotency_key, "capture_identity_invalid");
|
|
1668
|
+
}
|
|
1669
|
+
function assertLegacyCaptureKey(row) {
|
|
1670
|
+
let expected;
|
|
1671
|
+
if (row.event_kind === "user-candidate") {
|
|
1672
|
+
requireSourceMessage(row.source_message_id, "capture_events", row.idempotency_key);
|
|
1673
|
+
expected = legacyCaptureKey([
|
|
1674
|
+
"user",
|
|
1675
|
+
row.binding_key,
|
|
1676
|
+
row.source_session_id,
|
|
1677
|
+
row.source_message_id
|
|
1678
|
+
]);
|
|
1679
|
+
} else if (row.event_kind === "assistant-candidate") {
|
|
1680
|
+
requireSourceMessage(row.source_message_id, "capture_events", row.idempotency_key);
|
|
1681
|
+
if (row.source_ordinal === null)
|
|
1682
|
+
fail("capture_events", row.idempotency_key, "capture_identity_invalid");
|
|
1683
|
+
expected = legacyCaptureKey([
|
|
1684
|
+
"assistant",
|
|
1685
|
+
row.binding_key,
|
|
1686
|
+
row.source_session_id,
|
|
1687
|
+
row.source_message_id,
|
|
1688
|
+
String(row.source_ordinal)
|
|
1689
|
+
]);
|
|
1690
|
+
} else if (row.event_kind === "session-summary") {
|
|
1691
|
+
requireSourceMessage(row.source_message_id, "capture_events", row.idempotency_key);
|
|
1692
|
+
expected = legacyCaptureKey([
|
|
1693
|
+
"summary",
|
|
1694
|
+
row.binding_key,
|
|
1695
|
+
row.source_session_id,
|
|
1696
|
+
row.source_message_id
|
|
1697
|
+
]);
|
|
1698
|
+
} else if (row.event_kind === "tool-signal") {
|
|
1699
|
+
legacyToolStatus(row);
|
|
1700
|
+
return;
|
|
1701
|
+
} else {
|
|
1702
|
+
fail("capture_events", row.idempotency_key, "capture_identity_invalid");
|
|
1703
|
+
}
|
|
1704
|
+
if (row.idempotency_key !== expected) {
|
|
1705
|
+
fail("capture_events", row.idempotency_key, "capture_identity_mismatch");
|
|
1706
|
+
}
|
|
1707
|
+
}
|
|
1708
|
+
function legacyToolStatus(row) {
|
|
1709
|
+
requireSourceMessage(row.source_message_id, "capture_events", row.idempotency_key);
|
|
1710
|
+
if (!row.source_tool_call_id)
|
|
1711
|
+
fail("capture_events", row.idempotency_key, "capture_identity_invalid");
|
|
1712
|
+
const completed = legacyCaptureKey([
|
|
1713
|
+
"tool",
|
|
1714
|
+
row.binding_key,
|
|
1715
|
+
row.source_session_id,
|
|
1716
|
+
row.source_message_id,
|
|
1717
|
+
row.source_tool_call_id,
|
|
1718
|
+
"completed"
|
|
1719
|
+
]);
|
|
1720
|
+
const errored = legacyCaptureKey([
|
|
1721
|
+
"tool",
|
|
1722
|
+
row.binding_key,
|
|
1723
|
+
row.source_session_id,
|
|
1724
|
+
row.source_message_id,
|
|
1725
|
+
row.source_tool_call_id,
|
|
1726
|
+
"error"
|
|
1727
|
+
]);
|
|
1728
|
+
if (row.idempotency_key === completed)
|
|
1729
|
+
return "completed";
|
|
1730
|
+
if (row.idempotency_key === errored)
|
|
1731
|
+
return "error";
|
|
1732
|
+
fail("capture_events", row.idempotency_key, "capture_identity_mismatch");
|
|
1733
|
+
}
|
|
1734
|
+
function legacyCaptureKey(fields) {
|
|
1735
|
+
return sha256(["capture/1", ...fields].join("\x00"));
|
|
1736
|
+
}
|
|
1737
|
+
function captureKeyFromPayload(eventKind, bindingKey, source, signal) {
|
|
1738
|
+
const sessionID = source.sessionID;
|
|
1739
|
+
const messageID = source.messageID;
|
|
1740
|
+
if (typeof sessionID !== "string" || !sessionID)
|
|
1741
|
+
throw new Error("invalid source");
|
|
1742
|
+
if (eventKind === "user-candidate") {
|
|
1743
|
+
if (typeof messageID !== "string" || !messageID || hasOwn(source, "ordinal") || hasOwn(source, "toolCallID")) {
|
|
1744
|
+
throw new Error("invalid source");
|
|
1745
|
+
}
|
|
1746
|
+
return captureIdempotencyKey({ kind: "user", bindingKey, sessionID, messageID });
|
|
1747
|
+
}
|
|
1748
|
+
if (eventKind === "assistant-candidate") {
|
|
1749
|
+
if (typeof messageID !== "string" || !messageID || typeof source.ordinal !== "number" || hasOwn(source, "toolCallID")) {
|
|
1750
|
+
throw new Error("invalid source");
|
|
1751
|
+
}
|
|
1752
|
+
return captureIdempotencyKey({
|
|
1753
|
+
kind: "assistant",
|
|
1754
|
+
bindingKey,
|
|
1755
|
+
sessionID,
|
|
1756
|
+
assistantMessageID: messageID,
|
|
1757
|
+
ordinal: source.ordinal
|
|
1758
|
+
});
|
|
1759
|
+
}
|
|
1760
|
+
if (eventKind === "session-summary") {
|
|
1761
|
+
if (typeof messageID !== "string" || !messageID || hasOwn(source, "ordinal") || hasOwn(source, "toolCallID")) {
|
|
1762
|
+
throw new Error("invalid source");
|
|
1763
|
+
}
|
|
1764
|
+
return captureIdempotencyKey({ kind: "summary", bindingKey, sessionID, checkpointMessageID: messageID });
|
|
1765
|
+
}
|
|
1766
|
+
if (eventKind === "tool-signal") {
|
|
1767
|
+
const status = signal && typeof signal === "object" ? signal.status : undefined;
|
|
1768
|
+
const toolCallID = source.toolCallID;
|
|
1769
|
+
if (typeof messageID !== "string" || !messageID || typeof toolCallID !== "string" || !toolCallID || hasOwn(source, "ordinal") || status !== "completed" && status !== "error") {
|
|
1770
|
+
throw new Error("invalid source");
|
|
1771
|
+
}
|
|
1772
|
+
return captureIdempotencyKey({
|
|
1773
|
+
kind: "tool",
|
|
1774
|
+
bindingKey,
|
|
1775
|
+
sessionID,
|
|
1776
|
+
assistantMessageID: messageID,
|
|
1777
|
+
toolCallID,
|
|
1778
|
+
terminalStatus: status
|
|
1779
|
+
});
|
|
1780
|
+
}
|
|
1781
|
+
throw new Error("invalid event kind");
|
|
1782
|
+
}
|
|
1783
|
+
function replaceWithV11(db, rows) {
|
|
1784
|
+
db.exec(`
|
|
1785
|
+
DROP TRIGGER IF EXISTS notes_fts_ai;
|
|
1786
|
+
DROP TRIGGER IF EXISTS notes_fts_ad;
|
|
1787
|
+
DROP TRIGGER IF EXISTS notes_fts_au;
|
|
1788
|
+
DROP TABLE IF EXISTS notes_fts;
|
|
1789
|
+
DROP TABLE IF EXISTS index_outbox;
|
|
1790
|
+
DROP TABLE IF EXISTS note_revisions;
|
|
1791
|
+
DROP TABLE IF EXISTS note_provenance;
|
|
1792
|
+
DROP TABLE IF EXISTS capture_events;
|
|
1793
|
+
DROP TABLE IF EXISTS capture_checkpoints;
|
|
1794
|
+
DROP TABLE IF EXISTS project_bindings;
|
|
1795
|
+
DROP TABLE IF EXISTS note_edges;
|
|
1796
|
+
DROP TABLE IF EXISTS notes;
|
|
1797
|
+
DROP TABLE IF EXISTS projects;
|
|
1798
|
+
DROP TABLE IF EXISTS schema_state;
|
|
1799
|
+
DROP TABLE IF EXISTS agz_meta;
|
|
1800
|
+
`);
|
|
1801
|
+
db.exec(SCHEMA_V11_TABLES);
|
|
1802
|
+
db.exec(FTS_V9);
|
|
1803
|
+
for (const row of rows.projects) {
|
|
1804
|
+
insertRow("projects", row.id, () => {
|
|
1805
|
+
db.query("INSERT INTO projects (id, name, normalized_name, created_at, updated_at) VALUES (?, ?, ?, ?, ?)").run(row.id, row.name, row.normalized_name, row.created_at, row.updated_at);
|
|
1806
|
+
});
|
|
1807
|
+
}
|
|
1808
|
+
for (const row of rows.notes) {
|
|
1809
|
+
insertRow("notes", row.id, () => {
|
|
1810
|
+
db.query(`INSERT INTO notes
|
|
1811
|
+
(id, project_id, kind, title, summary, content, size_class, pinned, status,
|
|
1812
|
+
supersedes_id, current_revision, subject_key, content_hash, created_at, updated_at)
|
|
1813
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(row.id, row.project_id, row.kind, row.title, row.summary, row.content, row.size_class, row.pinned, row.status, row.supersedes_id, row.current_revision, row.subject_key, row.content_hash, row.created_at, row.updated_at);
|
|
1814
|
+
});
|
|
1815
|
+
}
|
|
1816
|
+
for (const row of rows.bindings) {
|
|
1817
|
+
insertRow("project_bindings", row.binding_key, () => {
|
|
1818
|
+
db.query(`INSERT INTO project_bindings
|
|
1819
|
+
(binding_key, project_id, source, source_project_id, workspace_id,
|
|
1820
|
+
canonical_path_hash, created_at, updated_at)
|
|
1821
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`).run(row.binding_key, row.project_id, row.source, row.source_project_id, row.workspace_id, row.canonical_path_hash, row.created_at, row.updated_at);
|
|
1822
|
+
});
|
|
1823
|
+
}
|
|
1824
|
+
for (const row of rows.checkpoints) {
|
|
1825
|
+
insertRow("capture_checkpoints", row.session_id, () => {
|
|
1826
|
+
db.query(`INSERT INTO capture_checkpoints
|
|
1827
|
+
(session_id, binding_key, project_id, state, last_message_id,
|
|
1828
|
+
last_reconciled_at, next_reconcile_at, failure_count, lease_owner,
|
|
1829
|
+
lease_expires_at, created_at, updated_at)
|
|
1830
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(row.session_id, row.binding_key, row.project_id, row.state, row.last_message_id, row.last_reconciled_at, row.next_reconcile_at, row.failure_count, row.lease_owner, row.lease_expires_at, row.created_at, row.updated_at);
|
|
1831
|
+
});
|
|
1832
|
+
}
|
|
1833
|
+
for (const row of rows.captures) {
|
|
1834
|
+
insertRow("capture_events", row.idempotency_key, () => {
|
|
1835
|
+
db.query(`INSERT INTO capture_events
|
|
1836
|
+
(idempotency_key, contract, project_id, binding_key, event_kind,
|
|
1837
|
+
source_session_id, source_message_id, source_ordinal, source_tool_call_id,
|
|
1838
|
+
payload_json, payload_hash, redaction_version, state, attempt_count,
|
|
1839
|
+
note_id, last_error_code, generation, created_at, updated_at, processed_at)
|
|
1840
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(row.idempotency_key, row.contract, row.project_id, row.binding_key, row.event_kind, row.source_session_id, row.source_message_id, row.source_ordinal, row.source_tool_call_id, row.payload_json, row.payload_hash, row.redaction_version, row.state, row.attempt_count, row.note_id, row.last_error_code, row.generation, row.created_at, row.updated_at, row.processed_at);
|
|
1841
|
+
});
|
|
1842
|
+
}
|
|
1843
|
+
for (const row of rows.provenance) {
|
|
1844
|
+
insertRow("note_provenance", row.id, () => {
|
|
1845
|
+
db.query(`INSERT INTO note_provenance
|
|
1846
|
+
(id, project_id, note_id, source_type, capture_event_id, source_session_id,
|
|
1847
|
+
source_message_id, source_ordinal, source_tool_call_id, redaction_version,
|
|
1848
|
+
extractor_version, confidence, created_at)
|
|
1849
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(row.id, row.project_id, row.note_id, row.source_type, row.capture_event_id, row.source_session_id, row.source_message_id, row.source_ordinal, row.source_tool_call_id, row.redaction_version, row.extractor_version, row.confidence, row.created_at);
|
|
1850
|
+
});
|
|
1851
|
+
}
|
|
1852
|
+
for (const row of rows.revisions) {
|
|
1853
|
+
insertRow("note_revisions", row.note_id, () => {
|
|
1854
|
+
db.query(`INSERT INTO note_revisions
|
|
1855
|
+
(project_id, note_id, revision, kind, title, summary, content, size_class,
|
|
1856
|
+
pinned, status, supersedes_id, subject_key, content_hash, provenance_id, created_at)
|
|
1857
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(row.project_id, row.note_id, row.revision, row.kind, row.title, row.summary, row.content, row.size_class, row.pinned, row.status, row.supersedes_id, row.subject_key, row.content_hash, row.provenance_id, row.created_at);
|
|
1858
|
+
});
|
|
1859
|
+
}
|
|
1860
|
+
for (const row of rows.edges) {
|
|
1861
|
+
insertRow("note_edges", row.id, () => {
|
|
1862
|
+
db.query("INSERT INTO note_edges (id, project_id, source_id, target_id, predicate, created_at) VALUES (?, ?, ?, ?, ?, ?)").run(row.id, row.project_id, row.source_id, row.target_id, row.predicate, row.created_at);
|
|
1863
|
+
});
|
|
1864
|
+
}
|
|
1865
|
+
for (const row of rows.outbox) {
|
|
1866
|
+
insertRow("index_outbox", String(row.id), () => {
|
|
1867
|
+
db.query(`INSERT INTO index_outbox
|
|
1868
|
+
(id, backend, operation_key, operation, project_id, note_id, revision,
|
|
1869
|
+
content_hash, generation, lease_generation, fence, state, attempt_count,
|
|
1870
|
+
available_at, lease_owner, lease_expires_at, heartbeat_at, last_error_code,
|
|
1871
|
+
created_at, completed_at)
|
|
1872
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(row.id, row.backend, row.operation_key, row.operation, row.project_id, row.note_id, row.revision, row.content_hash, row.generation, row.lease_generation, row.fence, row.state, row.attempt_count, row.available_at, row.lease_owner, row.lease_expires_at, row.heartbeat_at, row.last_error_code, row.created_at, row.completed_at);
|
|
1873
|
+
});
|
|
1874
|
+
}
|
|
1875
|
+
rebuildFts(db);
|
|
1876
|
+
db.query("INSERT INTO schema_state(version) VALUES (11)").run();
|
|
1877
|
+
insertV11Identity(db);
|
|
1878
|
+
}
|
|
1879
|
+
function revisionKey(projectID, noteID, revision) {
|
|
1880
|
+
return `${projectID.length}:${projectID}:${noteID.length}:${noteID}:${revision}`;
|
|
1881
|
+
}
|
|
1882
|
+
function noteKey(projectID, noteID) {
|
|
1883
|
+
return `${projectID.length}:${projectID}:${noteID.length}:${noteID}`;
|
|
1884
|
+
}
|
|
1885
|
+
function requireTable(db, table) {
|
|
1886
|
+
if (!hasTable2(db, table))
|
|
1887
|
+
fail(table, "schema", "source_schema");
|
|
1888
|
+
}
|
|
1889
|
+
function hasTable2(db, table) {
|
|
1890
|
+
const row = db.query("SELECT COUNT(*) AS count FROM sqlite_master WHERE type IN ('table','view') AND name = ?").get(table);
|
|
1891
|
+
return row.count > 0;
|
|
1892
|
+
}
|
|
1893
|
+
function selectRows(db, table, sql) {
|
|
1894
|
+
try {
|
|
1895
|
+
return db.query(sql).all();
|
|
1896
|
+
} catch {
|
|
1897
|
+
fail(table, "schema", "source_schema");
|
|
1898
|
+
}
|
|
1899
|
+
}
|
|
1900
|
+
function insertRow(table, identity, insert) {
|
|
1901
|
+
try {
|
|
1902
|
+
insert();
|
|
1903
|
+
} catch {
|
|
1904
|
+
fail(table, identity, "row_constraint");
|
|
1905
|
+
}
|
|
1906
|
+
}
|
|
1907
|
+
function requireString(value, table, identity, code) {
|
|
1908
|
+
if (typeof value !== "string")
|
|
1909
|
+
fail(table, identity, code);
|
|
1910
|
+
}
|
|
1911
|
+
function requireNonEmptyString(value, table, identity, code) {
|
|
1912
|
+
requireString(value, table, identity, code);
|
|
1913
|
+
if (value.length === 0)
|
|
1914
|
+
fail(table, identity, code);
|
|
1915
|
+
}
|
|
1916
|
+
function requireNullableString(value, table, identity, code) {
|
|
1917
|
+
if (value !== null && typeof value !== "string")
|
|
1918
|
+
fail(table, identity, code);
|
|
1919
|
+
}
|
|
1920
|
+
function requireInteger(value, table, identity, code) {
|
|
1921
|
+
if (!Number.isSafeInteger(value))
|
|
1922
|
+
fail(table, identity, code);
|
|
1923
|
+
}
|
|
1924
|
+
function requireNullableInteger(value, table, identity, code) {
|
|
1925
|
+
if (value !== null && !Number.isSafeInteger(value))
|
|
1926
|
+
fail(table, identity, code);
|
|
1927
|
+
}
|
|
1928
|
+
function requireIntegerAtLeast(value, minimum, table, identity, code) {
|
|
1929
|
+
if (typeof value !== "number" || !Number.isSafeInteger(value) || value < minimum) {
|
|
1930
|
+
fail(table, identity, code);
|
|
1931
|
+
}
|
|
1932
|
+
}
|
|
1933
|
+
function requireHash(value, table, identity, code) {
|
|
1934
|
+
if (typeof value !== "string" || !/^[0-9a-f]{64}$/i.test(value))
|
|
1935
|
+
fail(table, identity, code);
|
|
1936
|
+
}
|
|
1937
|
+
function requireKind(value, table, identity) {
|
|
1938
|
+
if (typeof value !== "string" || !KINDS_SET.has(value))
|
|
1939
|
+
fail(table, identity, "kind_invalid");
|
|
1940
|
+
}
|
|
1941
|
+
function requireSourceMessage(value, table, identity) {
|
|
1942
|
+
if (typeof value !== "string" || !value)
|
|
1943
|
+
fail(table, identity, "capture_identity_invalid");
|
|
1944
|
+
}
|
|
1945
|
+
function hasOwn(value, key) {
|
|
1946
|
+
return Object.prototype.hasOwnProperty.call(value, key);
|
|
1947
|
+
}
|
|
1948
|
+
function legacyNoteContentHash(kind, title, summary, content) {
|
|
1949
|
+
return createHash3("sha256").update(`${kind}\x00${title}\x00${summary}\x00${content}`, "utf8").digest("hex");
|
|
1950
|
+
}
|
|
1951
|
+
function sha256(value) {
|
|
1952
|
+
return createHash3("sha256").update(value, "utf8").digest("hex");
|
|
1953
|
+
}
|
|
1954
|
+
function capturePayloadHash(event) {
|
|
1955
|
+
const canonical = structuredClone(event);
|
|
1956
|
+
canonical.source.observedAt = 0;
|
|
1957
|
+
return hashTuple("capture-payload", 2, [JSON.stringify(canonical)]);
|
|
1958
|
+
}
|
|
1959
|
+
function fail(table, identity, code) {
|
|
1960
|
+
const safeIdentity = typeof identity === "string" && /^[A-Za-z0-9._:-]{1,120}$/.test(identity) ? identity : "redacted";
|
|
1961
|
+
throw new Error(`schema_v11_migration_${code} table=${table} row=${safeIdentity}`);
|
|
1962
|
+
}
|
|
1963
|
+
|
|
1964
|
+
// src/db/legacy-health.ts
|
|
1965
|
+
var V2_TABLES = [
|
|
1966
|
+
"memory_items",
|
|
1967
|
+
"memory_versions",
|
|
1968
|
+
"memory_identities",
|
|
1969
|
+
"document_sources",
|
|
1970
|
+
"document_chunks",
|
|
1971
|
+
"memories",
|
|
1972
|
+
"memory_links",
|
|
1973
|
+
"memory_edges",
|
|
1974
|
+
"memory_associations"
|
|
1975
|
+
];
|
|
1976
|
+
var PRE_V8_OBJECTS = new Set([
|
|
1977
|
+
"index:note_edges_source_idx",
|
|
1978
|
+
"index:note_edges_target_idx",
|
|
1979
|
+
"index:notes_project_idx",
|
|
1980
|
+
"table:note_edges",
|
|
1981
|
+
"table:notes",
|
|
1982
|
+
"table:notes_fts",
|
|
1983
|
+
"table:notes_fts_config",
|
|
1984
|
+
"table:notes_fts_content",
|
|
1985
|
+
"table:notes_fts_data",
|
|
1986
|
+
"table:notes_fts_docsize",
|
|
1987
|
+
"table:notes_fts_idx",
|
|
1988
|
+
"table:projects",
|
|
1989
|
+
"table:schema_state",
|
|
1990
|
+
...V2_TABLES.map((table) => `table:${table}`)
|
|
1991
|
+
]);
|
|
1992
|
+
var PRE_V8_COLUMNS = {
|
|
1993
|
+
schema_state: ["version"],
|
|
1994
|
+
projects: ["id", "name", "normalized_name", "created_at", "updated_at"],
|
|
1995
|
+
notes: [
|
|
1996
|
+
"id",
|
|
1997
|
+
"project_id",
|
|
1998
|
+
"kind",
|
|
1999
|
+
"title",
|
|
2000
|
+
"summary",
|
|
2001
|
+
"content",
|
|
2002
|
+
"size_class",
|
|
2003
|
+
"status",
|
|
2004
|
+
"supersedes_id",
|
|
2005
|
+
"created_at",
|
|
2006
|
+
"updated_at"
|
|
2007
|
+
],
|
|
2008
|
+
note_edges: ["id", "project_id", "source_id", "target_id", "predicate", "created_at"],
|
|
2009
|
+
notes_fts: ["id", "title", "summary", "content"]
|
|
2010
|
+
};
|
|
2011
|
+
var V2_COLUMNS = {
|
|
2012
|
+
memory_items: [
|
|
2013
|
+
"id",
|
|
2014
|
+
"identity_id",
|
|
2015
|
+
"subject_key",
|
|
2016
|
+
"kind",
|
|
2017
|
+
"lifecycle_state",
|
|
2018
|
+
"current_version_id",
|
|
2019
|
+
"created_at",
|
|
2020
|
+
"updated_at"
|
|
2021
|
+
],
|
|
2022
|
+
memory_versions: ["id", "summary", "content"],
|
|
2023
|
+
memory_identities: ["id", "project_id"]
|
|
2024
|
+
};
|
|
2025
|
+
var V8_OBJECTS = new Set([
|
|
2026
|
+
"table:note_edges",
|
|
2027
|
+
"table:notes",
|
|
2028
|
+
"table:notes_fts",
|
|
2029
|
+
"table:notes_fts_config",
|
|
2030
|
+
"table:notes_fts_content",
|
|
2031
|
+
"table:notes_fts_data",
|
|
2032
|
+
"table:notes_fts_docsize",
|
|
2033
|
+
"table:notes_fts_idx",
|
|
2034
|
+
"table:projects",
|
|
2035
|
+
"table:schema_state"
|
|
2036
|
+
]);
|
|
2037
|
+
var V9_V10_OBJECTS = new Set([
|
|
2038
|
+
"index:capture_checkpoints_due_idx",
|
|
2039
|
+
"index:capture_events_note_idx",
|
|
2040
|
+
"index:capture_events_session_idx",
|
|
2041
|
+
"index:capture_events_state_idx",
|
|
2042
|
+
"index:index_outbox_due_idx",
|
|
2043
|
+
"index:note_edges_source_idx",
|
|
2044
|
+
"index:note_edges_target_idx",
|
|
2045
|
+
"index:notes_active_subject_idx",
|
|
2046
|
+
"index:notes_project_idx",
|
|
2047
|
+
"table:capture_checkpoints",
|
|
2048
|
+
"table:capture_events",
|
|
2049
|
+
"table:index_outbox",
|
|
2050
|
+
"table:note_edges",
|
|
2051
|
+
"table:note_provenance",
|
|
2052
|
+
"table:note_revisions",
|
|
2053
|
+
"table:notes",
|
|
2054
|
+
"table:notes_fts",
|
|
2055
|
+
"table:notes_fts_config",
|
|
2056
|
+
"table:notes_fts_data",
|
|
2057
|
+
"table:notes_fts_docsize",
|
|
2058
|
+
"table:notes_fts_idx",
|
|
2059
|
+
"table:project_bindings",
|
|
2060
|
+
"table:projects",
|
|
2061
|
+
"table:schema_state",
|
|
2062
|
+
"trigger:notes_fts_ad",
|
|
2063
|
+
"trigger:notes_fts_ai",
|
|
2064
|
+
"trigger:notes_fts_au"
|
|
2065
|
+
]);
|
|
2066
|
+
var V9_V10_COLUMNS = {
|
|
2067
|
+
schema_state: ["version"],
|
|
2068
|
+
projects: ["id", "name", "normalized_name", "created_at", "updated_at"],
|
|
2069
|
+
notes: [
|
|
2070
|
+
"id",
|
|
2071
|
+
"project_id",
|
|
2072
|
+
"kind",
|
|
2073
|
+
"title",
|
|
2074
|
+
"summary",
|
|
2075
|
+
"content",
|
|
2076
|
+
"size_class",
|
|
2077
|
+
"pinned",
|
|
2078
|
+
"status",
|
|
2079
|
+
"supersedes_id",
|
|
2080
|
+
"current_revision",
|
|
2081
|
+
"subject_key",
|
|
2082
|
+
"content_hash",
|
|
2083
|
+
"created_at",
|
|
2084
|
+
"updated_at"
|
|
2085
|
+
],
|
|
2086
|
+
note_edges: ["id", "project_id", "source_id", "target_id", "predicate", "created_at"],
|
|
2087
|
+
project_bindings: [
|
|
2088
|
+
"binding_key",
|
|
2089
|
+
"project_id",
|
|
2090
|
+
"source",
|
|
2091
|
+
"source_project_id",
|
|
2092
|
+
"workspace_id",
|
|
2093
|
+
"canonical_path_hash",
|
|
2094
|
+
"created_at",
|
|
2095
|
+
"updated_at"
|
|
2096
|
+
],
|
|
2097
|
+
capture_checkpoints: [
|
|
2098
|
+
"session_id",
|
|
2099
|
+
"binding_key",
|
|
2100
|
+
"project_id",
|
|
2101
|
+
"state",
|
|
2102
|
+
"last_message_id",
|
|
2103
|
+
"last_reconciled_at",
|
|
2104
|
+
"next_reconcile_at",
|
|
2105
|
+
"failure_count",
|
|
2106
|
+
"lease_owner",
|
|
2107
|
+
"lease_expires_at",
|
|
2108
|
+
"created_at",
|
|
2109
|
+
"updated_at"
|
|
2110
|
+
],
|
|
2111
|
+
capture_events: [
|
|
2112
|
+
"idempotency_key",
|
|
2113
|
+
"contract",
|
|
2114
|
+
"project_id",
|
|
2115
|
+
"binding_key",
|
|
2116
|
+
"event_kind",
|
|
2117
|
+
"source_session_id",
|
|
2118
|
+
"source_message_id",
|
|
2119
|
+
"source_ordinal",
|
|
2120
|
+
"source_tool_call_id",
|
|
2121
|
+
"payload_json",
|
|
2122
|
+
"payload_hash",
|
|
2123
|
+
"redaction_version",
|
|
2124
|
+
"state",
|
|
2125
|
+
"attempt_count",
|
|
2126
|
+
"note_id",
|
|
2127
|
+
"last_error_code",
|
|
2128
|
+
"generation",
|
|
2129
|
+
"created_at",
|
|
2130
|
+
"updated_at",
|
|
2131
|
+
"processed_at"
|
|
2132
|
+
],
|
|
2133
|
+
note_provenance: [
|
|
2134
|
+
"id",
|
|
2135
|
+
"project_id",
|
|
2136
|
+
"note_id",
|
|
2137
|
+
"source_type",
|
|
2138
|
+
"capture_event_id",
|
|
2139
|
+
"source_session_id",
|
|
2140
|
+
"source_message_id",
|
|
2141
|
+
"source_ordinal",
|
|
2142
|
+
"source_tool_call_id",
|
|
2143
|
+
"redaction_version",
|
|
2144
|
+
"extractor_version",
|
|
2145
|
+
"confidence",
|
|
2146
|
+
"created_at"
|
|
2147
|
+
],
|
|
2148
|
+
note_revisions: [
|
|
2149
|
+
"project_id",
|
|
2150
|
+
"note_id",
|
|
2151
|
+
"revision",
|
|
2152
|
+
"kind",
|
|
2153
|
+
"title",
|
|
2154
|
+
"summary",
|
|
2155
|
+
"content",
|
|
2156
|
+
"size_class",
|
|
2157
|
+
"pinned",
|
|
2158
|
+
"status",
|
|
2159
|
+
"supersedes_id",
|
|
2160
|
+
"subject_key",
|
|
2161
|
+
"content_hash",
|
|
2162
|
+
"provenance_id",
|
|
2163
|
+
"created_at"
|
|
2164
|
+
],
|
|
2165
|
+
index_outbox: [
|
|
2166
|
+
"id",
|
|
2167
|
+
"backend",
|
|
2168
|
+
"operation",
|
|
2169
|
+
"project_id",
|
|
2170
|
+
"note_id",
|
|
2171
|
+
"revision",
|
|
2172
|
+
"content_hash",
|
|
2173
|
+
"state",
|
|
2174
|
+
"attempt_count",
|
|
2175
|
+
"available_at",
|
|
2176
|
+
"lease_owner",
|
|
2177
|
+
"lease_expires_at",
|
|
2178
|
+
"last_error_code",
|
|
2179
|
+
"created_at",
|
|
2180
|
+
"completed_at"
|
|
2181
|
+
]
|
|
2182
|
+
};
|
|
2183
|
+
function assertLegacySchemaIdentity(db, version) {
|
|
2184
|
+
if (version < 2 || version > 10)
|
|
2185
|
+
throw new Error("unrecognized_database");
|
|
2186
|
+
const applicationID = db.query("PRAGMA application_id").get().application_id;
|
|
2187
|
+
if (applicationID !== 0 || tableExists(db, "agz_meta"))
|
|
2188
|
+
throw new Error("unrecognized_database");
|
|
2189
|
+
if (version === 2 && tableExists(db, "memory_items")) {
|
|
2190
|
+
assertV2Identity(db);
|
|
2191
|
+
assertHealthyDatabase(db);
|
|
2192
|
+
return;
|
|
2193
|
+
}
|
|
2194
|
+
if (version < 8) {
|
|
2195
|
+
assertPreV8Identity(db, version);
|
|
2196
|
+
assertHealthyDatabase(db);
|
|
2197
|
+
return;
|
|
2198
|
+
}
|
|
2199
|
+
const states = db.query("SELECT version FROM schema_state").all();
|
|
2200
|
+
if (states.length !== 1 || states[0]?.version !== version)
|
|
2201
|
+
throw new Error("unrecognized_database");
|
|
2202
|
+
const rows = db.query("SELECT type, name FROM sqlite_master WHERE name NOT LIKE 'sqlite_%'").all();
|
|
2203
|
+
const actual = new Set(rows.map((row) => `${row.type}:${row.name}`));
|
|
2204
|
+
const allowed = version === 8 ? V8_OBJECTS : V9_V10_OBJECTS;
|
|
2205
|
+
const required = version === 8 ? V8_OBJECTS : new Set([...V9_V10_OBJECTS].filter((name) => !name.startsWith("index:capture_events_")));
|
|
2206
|
+
if (actual.size > allowed.size || [...actual].some((name) => !allowed.has(name)) || [...required].some((name) => !actual.has(name))) {
|
|
2207
|
+
throw new Error("unrecognized_database");
|
|
2208
|
+
}
|
|
2209
|
+
const columns = version === 8 ? {
|
|
2210
|
+
projects: V9_V10_COLUMNS.projects,
|
|
2211
|
+
notes: V9_V10_COLUMNS.notes.filter((name) => !["current_revision", "subject_key", "content_hash"].includes(name)),
|
|
2212
|
+
note_edges: V9_V10_COLUMNS.note_edges,
|
|
2213
|
+
schema_state: V9_V10_COLUMNS.schema_state
|
|
2214
|
+
} : V9_V10_COLUMNS;
|
|
2215
|
+
for (const [table, expected] of Object.entries(columns)) {
|
|
2216
|
+
const actualColumns = db.query(`PRAGMA table_info(${table})`).all().map((row) => row.name);
|
|
2217
|
+
if (JSON.stringify(actualColumns) !== JSON.stringify(expected)) {
|
|
2218
|
+
throw new Error("unrecognized_database");
|
|
2219
|
+
}
|
|
2220
|
+
}
|
|
2221
|
+
assertHealthyDatabase(db);
|
|
2222
|
+
if (version === 10)
|
|
2223
|
+
assertV10SourceDatabase(db);
|
|
2224
|
+
}
|
|
2225
|
+
function assertV2Identity(db) {
|
|
2226
|
+
const rows = db.query("SELECT type, name FROM sqlite_master WHERE name NOT LIKE 'sqlite_%'").all();
|
|
2227
|
+
const allowed = new Set(V2_TABLES.map((table) => `table:${table}`));
|
|
2228
|
+
const actual = new Set(rows.map((row) => `${row.type}:${row.name}`));
|
|
2229
|
+
if ([...actual].some((name) => !allowed.has(name)))
|
|
2230
|
+
throw new Error("unrecognized_database");
|
|
2231
|
+
for (const [table, expected] of Object.entries(V2_COLUMNS)) {
|
|
2232
|
+
assertColumns(db, table, expected);
|
|
2233
|
+
}
|
|
2234
|
+
}
|
|
2235
|
+
function assertPreV8Identity(db, version) {
|
|
2236
|
+
const rows = db.query("SELECT type, name FROM sqlite_master WHERE name NOT LIKE 'sqlite_%'").all();
|
|
2237
|
+
const actual = new Set(rows.map((row) => `${row.type}:${row.name}`));
|
|
2238
|
+
const required = ["table:notes", "table:note_edges", "table:schema_state"];
|
|
2239
|
+
if ([...actual].some((name) => !PRE_V8_OBJECTS.has(name)) || required.some((name) => !actual.has(name)) || version === 7 && !actual.has("table:projects")) {
|
|
2240
|
+
throw new Error("unrecognized_database");
|
|
2241
|
+
}
|
|
2242
|
+
const states = db.query("SELECT version FROM schema_state").all();
|
|
2243
|
+
if (states.length !== 1 || states[0]?.version !== version)
|
|
2244
|
+
throw new Error("unrecognized_database");
|
|
2245
|
+
const noteColumns = columns(db, "notes");
|
|
2246
|
+
const expectedNotes = [...PRE_V8_COLUMNS.notes];
|
|
2247
|
+
const expectedPinnedNotes = [...expectedNotes.slice(0, 7), "pinned", ...expectedNotes.slice(7)];
|
|
2248
|
+
if (JSON.stringify(noteColumns) !== JSON.stringify(expectedNotes) && JSON.stringify(noteColumns) !== JSON.stringify(expectedPinnedNotes)) {
|
|
2249
|
+
throw new Error("unrecognized_database");
|
|
2250
|
+
}
|
|
2251
|
+
assertColumns(db, "note_edges", PRE_V8_COLUMNS.note_edges);
|
|
2252
|
+
assertColumns(db, "schema_state", PRE_V8_COLUMNS.schema_state);
|
|
2253
|
+
if (tableExists(db, "projects"))
|
|
2254
|
+
assertColumns(db, "projects", PRE_V8_COLUMNS.projects);
|
|
2255
|
+
if (tableExists(db, "notes_fts"))
|
|
2256
|
+
assertColumns(db, "notes_fts", PRE_V8_COLUMNS.notes_fts);
|
|
38
2257
|
}
|
|
39
|
-
function
|
|
40
|
-
|
|
2258
|
+
function assertColumns(db, table, expected) {
|
|
2259
|
+
if (JSON.stringify(columns(db, table)) !== JSON.stringify(expected)) {
|
|
2260
|
+
throw new Error("unrecognized_database");
|
|
2261
|
+
}
|
|
2262
|
+
}
|
|
2263
|
+
function columns(db, table) {
|
|
2264
|
+
return db.query(`PRAGMA table_info(${table})`).all().map((row) => row.name);
|
|
2265
|
+
}
|
|
2266
|
+
function tableExists(db, table) {
|
|
2267
|
+
return db.query("SELECT COUNT(*) AS count FROM sqlite_master WHERE type = 'table' AND name = ?").get(table).count > 0;
|
|
41
2268
|
}
|
|
42
2269
|
|
|
43
|
-
// src/
|
|
44
|
-
|
|
45
|
-
var KINDS = ["decision", "fact", "procedure", "context", "research", "preference", "task"];
|
|
46
|
-
var PREDICATES = ["SUPPORTS", "DERIVED_FROM", "PART_OF", "ABOUT", "PRECEDES", "SUPERSEDES"];
|
|
47
|
-
|
|
48
|
-
// src/db/backup.ts
|
|
49
|
-
import { createHash as createHash2, randomUUID } from "crypto";
|
|
50
|
-
import { Database } from "bun:sqlite";
|
|
2270
|
+
// src/db/maintenance.ts
|
|
2271
|
+
import { randomUUID as randomUUID2 } from "crypto";
|
|
51
2272
|
import {
|
|
52
2273
|
chmodSync,
|
|
53
2274
|
closeSync,
|
|
54
|
-
|
|
2275
|
+
constants,
|
|
55
2276
|
existsSync,
|
|
56
2277
|
fsyncSync,
|
|
57
2278
|
lstatSync,
|
|
58
2279
|
mkdirSync,
|
|
59
2280
|
openSync,
|
|
60
2281
|
readFileSync,
|
|
2282
|
+
readdirSync,
|
|
61
2283
|
renameSync,
|
|
62
|
-
|
|
2284
|
+
rmdirSync,
|
|
2285
|
+
unlinkSync,
|
|
63
2286
|
writeFileSync
|
|
64
2287
|
} from "fs";
|
|
65
|
-
import {
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
const
|
|
70
|
-
const
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
"
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
if (
|
|
2288
|
+
import { hostname, platform } from "os";
|
|
2289
|
+
import { basename, dirname, join as join2, parse, resolve } from "path";
|
|
2290
|
+
function acquireDatabaseLease(databasePath) {
|
|
2291
|
+
const canonicalPath = canonicalDatabasePath(databasePath);
|
|
2292
|
+
const gatePath = maintenanceGatePath(canonicalPath);
|
|
2293
|
+
const leaseDirectory = databaseLeaseDirectory(canonicalPath);
|
|
2294
|
+
assertNoSymbolicLinks(dirname(canonicalPath));
|
|
2295
|
+
assertNoSymbolicLinks(canonicalPath, true);
|
|
2296
|
+
assertProtocolPath(gatePath, "maintenance gate", true);
|
|
2297
|
+
if (existsSync(gatePath))
|
|
2298
|
+
throw new Error("maintenance_gate_active");
|
|
2299
|
+
ensurePrivateDirectory(leaseDirectory, "database lease registry");
|
|
2300
|
+
const owner = ownerRecord();
|
|
2301
|
+
const temporaryPath = join2(leaseDirectory, `.${owner.ownerID}.tmp`);
|
|
2302
|
+
const leasePath = join2(leaseDirectory, `${owner.ownerID}.json`);
|
|
2303
|
+
try {
|
|
2304
|
+
writeExclusiveRecord(temporaryPath, owner);
|
|
2305
|
+
renameSync(temporaryPath, leasePath);
|
|
2306
|
+
fsyncPath(leaseDirectory);
|
|
2307
|
+
assertProtocolPath(gatePath, "maintenance gate", true);
|
|
2308
|
+
if (existsSync(gatePath)) {
|
|
2309
|
+
removeOwnedRecord(leasePath, owner.ownerID);
|
|
2310
|
+
throw new Error("maintenance_gate_active");
|
|
2311
|
+
}
|
|
2312
|
+
} catch (error) {
|
|
2313
|
+
removeFreshRecord(temporaryPath);
|
|
2314
|
+
removeOwnedRecord(leasePath, owner.ownerID);
|
|
2315
|
+
throw error;
|
|
2316
|
+
}
|
|
2317
|
+
let released = false;
|
|
2318
|
+
return {
|
|
2319
|
+
databasePath: canonicalPath,
|
|
2320
|
+
release: () => {
|
|
2321
|
+
if (released)
|
|
2322
|
+
return;
|
|
2323
|
+
removeOwnedRecord(leasePath, owner.ownerID);
|
|
2324
|
+
released = true;
|
|
2325
|
+
}
|
|
2326
|
+
};
|
|
2327
|
+
}
|
|
2328
|
+
function acquireMaintenanceGate(databasePath, recovery) {
|
|
2329
|
+
const canonicalPath = canonicalDatabasePath(databasePath);
|
|
2330
|
+
const gatePath = maintenanceGatePath(canonicalPath);
|
|
2331
|
+
const leaseDirectory = databaseLeaseDirectory(canonicalPath);
|
|
2332
|
+
assertNoSymbolicLinks(dirname(canonicalPath));
|
|
2333
|
+
assertNoSymbolicLinks(canonicalPath, true);
|
|
2334
|
+
assertProtocolPath(gatePath, "maintenance gate", true);
|
|
2335
|
+
assertProtocolPath(leaseDirectory, "database lease registry", true);
|
|
2336
|
+
const owner = ownerRecord();
|
|
2337
|
+
let created = false;
|
|
2338
|
+
let gateStat;
|
|
2339
|
+
try {
|
|
2340
|
+
mkdirSync(gatePath, { mode: 448 });
|
|
2341
|
+
created = true;
|
|
2342
|
+
gateStat = lstatSync(gatePath);
|
|
2343
|
+
} catch (error) {
|
|
2344
|
+
if (!isAlreadyExists(error))
|
|
2345
|
+
throw error;
|
|
2346
|
+
gateStat = claimStaleGate(gatePath, owner, recovery);
|
|
2347
|
+
}
|
|
2348
|
+
const ownerPath = join2(gatePath, "owner.json");
|
|
2349
|
+
try {
|
|
2350
|
+
if (created) {
|
|
2351
|
+
writeExclusiveRecord(ownerPath, owner);
|
|
2352
|
+
fsyncPath(gatePath);
|
|
2353
|
+
}
|
|
2354
|
+
rejectActiveLeases(leaseDirectory);
|
|
2355
|
+
} catch (error) {
|
|
2356
|
+
removeFreshGate(gatePath, owner.ownerID, gateStat.dev, gateStat.ino);
|
|
2357
|
+
throw error;
|
|
2358
|
+
}
|
|
2359
|
+
let released = false;
|
|
2360
|
+
let retained = false;
|
|
2361
|
+
const assertOwned = () => {
|
|
2362
|
+
if (released)
|
|
2363
|
+
throw new Error("maintenance_gate_not_owned");
|
|
2364
|
+
const record = readOwnerRecord(ownerPath, "maintenance gate owner");
|
|
2365
|
+
if (record.ownerID !== owner.ownerID)
|
|
2366
|
+
throw new Error("maintenance_gate_not_owned");
|
|
2367
|
+
};
|
|
2368
|
+
return {
|
|
2369
|
+
databasePath: canonicalPath,
|
|
2370
|
+
assertOwned,
|
|
2371
|
+
retain: () => {
|
|
2372
|
+
assertOwned();
|
|
2373
|
+
replaceOwnerRecord(ownerPath, owner.ownerID, {
|
|
2374
|
+
...owner,
|
|
2375
|
+
state: "recovery-required"
|
|
2376
|
+
});
|
|
2377
|
+
retained = true;
|
|
2378
|
+
},
|
|
2379
|
+
release: () => {
|
|
2380
|
+
if (released || retained)
|
|
2381
|
+
return;
|
|
2382
|
+
assertOwned();
|
|
2383
|
+
removeOwnedGate(gatePath, owner.ownerID);
|
|
2384
|
+
released = true;
|
|
2385
|
+
}
|
|
2386
|
+
};
|
|
2387
|
+
}
|
|
2388
|
+
function recoverStaleMaintenanceGate(databasePath, validate) {
|
|
2389
|
+
const canonicalPath = canonicalDatabasePath(databasePath);
|
|
2390
|
+
if (!existsSync(maintenanceGatePath(canonicalPath)))
|
|
2391
|
+
return false;
|
|
2392
|
+
const gate = acquireMaintenanceGate(canonicalPath);
|
|
2393
|
+
try {
|
|
2394
|
+
validate();
|
|
2395
|
+
} catch (error) {
|
|
2396
|
+
gate.retain();
|
|
2397
|
+
throw error;
|
|
2398
|
+
} finally {
|
|
2399
|
+
gate.release();
|
|
2400
|
+
}
|
|
2401
|
+
return true;
|
|
2402
|
+
}
|
|
2403
|
+
function assertMaintenanceGateFor(gate, databasePath) {
|
|
2404
|
+
if (gate.databasePath !== canonicalDatabasePath(databasePath)) {
|
|
2405
|
+
throw new Error("maintenance_gate_target_mismatch");
|
|
2406
|
+
}
|
|
2407
|
+
gate.assertOwned();
|
|
2408
|
+
}
|
|
2409
|
+
function canonicalDatabasePath(path) {
|
|
2410
|
+
const absolute = resolve(path);
|
|
2411
|
+
const parent = dirname(absolute);
|
|
2412
|
+
assertNoSymbolicLinks(parent);
|
|
2413
|
+
return join2(resolve(parent), basename(absolute));
|
|
2414
|
+
}
|
|
2415
|
+
function ensureDatabaseParent(path) {
|
|
2416
|
+
const parent = dirname(resolve(path));
|
|
2417
|
+
assertNoSymbolicLinks(parent);
|
|
2418
|
+
const existed = existsSync(parent);
|
|
2419
|
+
mkdirSync(parent, { recursive: true, mode: 448 });
|
|
2420
|
+
assertNoSymbolicLinks(parent);
|
|
2421
|
+
if (!existed)
|
|
2422
|
+
chmodSync(parent, 448);
|
|
2423
|
+
const stat = lstatSync(parent);
|
|
2424
|
+
if (!stat.isDirectory() || stat.isSymbolicLink()) {
|
|
2425
|
+
throw new Error("database parent must be a directory");
|
|
2426
|
+
}
|
|
2427
|
+
if (typeof process.getuid === "function" && stat.uid !== process.getuid()) {
|
|
2428
|
+
throw new Error("database parent must be owned by the current user");
|
|
2429
|
+
}
|
|
2430
|
+
if (platform() !== "win32" && (stat.mode & 18) !== 0) {
|
|
2431
|
+
throw new Error("database parent must not be group or world writable");
|
|
2432
|
+
}
|
|
2433
|
+
}
|
|
2434
|
+
function assertNoSymbolicLinks(path, allowMissingLeaf = false) {
|
|
2435
|
+
const absolute = resolve(path);
|
|
2436
|
+
const root = parse(absolute).root;
|
|
2437
|
+
const parts = absolute.slice(root.length).split(/[\\/]+/).filter(Boolean);
|
|
2438
|
+
let current = root;
|
|
2439
|
+
for (let index = 0;index < parts.length; index++) {
|
|
2440
|
+
current = join2(current, parts[index]);
|
|
2441
|
+
try {
|
|
2442
|
+
const stat = lstatSync(current);
|
|
2443
|
+
if (stat.isSymbolicLink())
|
|
2444
|
+
throw new Error(`symbolic links are not allowed: ${current}`);
|
|
2445
|
+
} catch (error) {
|
|
2446
|
+
if (isMissing(error) && (allowMissingLeaf ? index === parts.length - 1 : true))
|
|
2447
|
+
return;
|
|
2448
|
+
throw error;
|
|
2449
|
+
}
|
|
2450
|
+
}
|
|
2451
|
+
}
|
|
2452
|
+
function maintenanceGatePath(databasePath) {
|
|
2453
|
+
return `${databasePath}.maintenance`;
|
|
2454
|
+
}
|
|
2455
|
+
function databaseLeaseDirectory(databasePath) {
|
|
2456
|
+
return `${databasePath}.leases`;
|
|
2457
|
+
}
|
|
2458
|
+
function ensurePrivateDirectory(path, label) {
|
|
2459
|
+
assertProtocolPath(path, label, true);
|
|
2460
|
+
if (!existsSync(path)) {
|
|
2461
|
+
try {
|
|
2462
|
+
mkdirSync(path, { recursive: false, mode: 448 });
|
|
2463
|
+
} catch (error) {
|
|
2464
|
+
if (!isAlreadyExists(error))
|
|
2465
|
+
throw error;
|
|
2466
|
+
}
|
|
2467
|
+
}
|
|
2468
|
+
const stat = lstatSync(path);
|
|
2469
|
+
if (!stat.isDirectory() || stat.isSymbolicLink())
|
|
2470
|
+
throw new Error(`${label} must be a directory`);
|
|
2471
|
+
if (typeof process.getuid === "function" && stat.uid !== process.getuid()) {
|
|
2472
|
+
throw new Error(`${label} must be owned by the current user`);
|
|
2473
|
+
}
|
|
2474
|
+
chmodSync(path, 448);
|
|
2475
|
+
}
|
|
2476
|
+
function assertProtocolPath(path, label, allowMissing) {
|
|
2477
|
+
assertNoSymbolicLinks(path, allowMissing);
|
|
2478
|
+
if (!existsSync(path))
|
|
2479
|
+
return;
|
|
2480
|
+
const stat = lstatSync(path);
|
|
2481
|
+
if (stat.isSymbolicLink())
|
|
2482
|
+
throw new Error(`${label} must not be a symbolic link`);
|
|
2483
|
+
}
|
|
2484
|
+
function writeExclusiveRecord(path, owner) {
|
|
2485
|
+
writeFileSync(path, `${JSON.stringify(owner)}
|
|
2486
|
+
`, {
|
|
2487
|
+
encoding: "utf8",
|
|
2488
|
+
flag: "wx",
|
|
2489
|
+
mode: 384
|
|
2490
|
+
});
|
|
2491
|
+
fsyncPath(path);
|
|
2492
|
+
}
|
|
2493
|
+
function rejectActiveLeases(leaseDirectory) {
|
|
2494
|
+
if (!existsSync(leaseDirectory))
|
|
2495
|
+
return;
|
|
2496
|
+
const directoryStat = lstatSync(leaseDirectory);
|
|
2497
|
+
if (!directoryStat.isDirectory() || directoryStat.isSymbolicLink()) {
|
|
2498
|
+
throw new Error("database lease registry must be a directory");
|
|
2499
|
+
}
|
|
2500
|
+
for (const entry of readdirSync(leaseDirectory)) {
|
|
2501
|
+
if (!entry.endsWith(".json"))
|
|
86
2502
|
continue;
|
|
87
|
-
|
|
2503
|
+
const leasePath = join2(leaseDirectory, entry);
|
|
2504
|
+
const before = lstatSync(leasePath);
|
|
2505
|
+
if (!before.isFile() || before.isSymbolicLink())
|
|
2506
|
+
throw new Error("database lease must be a regular file");
|
|
2507
|
+
const lease = readOwnerRecord(leasePath, "database lease");
|
|
2508
|
+
if (ownerIsActive(lease))
|
|
2509
|
+
throw new Error("active_database_handles");
|
|
2510
|
+
const after = lstatSync(leasePath);
|
|
2511
|
+
if (before.dev !== after.dev || before.ino !== after.ino) {
|
|
2512
|
+
throw new Error("database lease changed during stale-owner verification");
|
|
2513
|
+
}
|
|
2514
|
+
unlinkSync(leasePath);
|
|
88
2515
|
}
|
|
89
|
-
|
|
2516
|
+
fsyncPath(leaseDirectory);
|
|
90
2517
|
}
|
|
91
|
-
function
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
2518
|
+
function ownerStatus(owner) {
|
|
2519
|
+
if (owner.state === "recovery-required")
|
|
2520
|
+
return "recovery-required";
|
|
2521
|
+
if (owner.hostname !== hostname())
|
|
2522
|
+
return "unverifiable";
|
|
2523
|
+
try {
|
|
2524
|
+
process.kill(owner.pid, 0);
|
|
2525
|
+
} catch (error) {
|
|
2526
|
+
if (error && typeof error === "object" && "code" in error && error.code === "ESRCH") {
|
|
2527
|
+
return "stale";
|
|
2528
|
+
}
|
|
2529
|
+
return "unverifiable";
|
|
95
2530
|
}
|
|
96
|
-
|
|
97
|
-
|
|
2531
|
+
const currentStart = processStartMarker(owner.pid);
|
|
2532
|
+
if (owner.processStart === null || currentStart === null)
|
|
2533
|
+
return "unverifiable";
|
|
2534
|
+
return owner.processStart === currentStart ? "active" : "stale";
|
|
2535
|
+
}
|
|
2536
|
+
function ownerIsActive(owner) {
|
|
2537
|
+
return ownerStatus(owner) !== "stale";
|
|
2538
|
+
}
|
|
2539
|
+
function ownerRecord() {
|
|
2540
|
+
return {
|
|
2541
|
+
ownerID: randomUUID2(),
|
|
2542
|
+
pid: process.pid,
|
|
2543
|
+
processStart: processStartMarker(process.pid),
|
|
2544
|
+
hostname: hostname(),
|
|
2545
|
+
createdAt: Date.now(),
|
|
2546
|
+
state: "active"
|
|
2547
|
+
};
|
|
2548
|
+
}
|
|
2549
|
+
function processStartMarker(pid) {
|
|
2550
|
+
if (platform() !== "linux")
|
|
2551
|
+
return null;
|
|
2552
|
+
try {
|
|
2553
|
+
const value = readFileSync(`/proc/${pid}/stat`, "utf8");
|
|
2554
|
+
const close = value.lastIndexOf(")");
|
|
2555
|
+
const fields = value.slice(close + 1).trim().split(/\s+/);
|
|
2556
|
+
const startTime = fields[19];
|
|
2557
|
+
return startTime ? `linux:${startTime}` : null;
|
|
2558
|
+
} catch {
|
|
2559
|
+
return null;
|
|
98
2560
|
}
|
|
99
|
-
return health;
|
|
100
2561
|
}
|
|
101
|
-
function
|
|
102
|
-
const
|
|
103
|
-
|
|
2562
|
+
function readOwnerRecord(path, label) {
|
|
2563
|
+
const stat = lstatSync(path);
|
|
2564
|
+
if (!stat.isFile() || stat.isSymbolicLink())
|
|
2565
|
+
throw new Error(`${label} must be a regular file`);
|
|
2566
|
+
const parsed = JSON.parse(readFileSync(path, "utf8"));
|
|
2567
|
+
if (typeof parsed.ownerID !== "string" || !parsed.ownerID || !Number.isSafeInteger(parsed.pid) || parsed.pid <= 0 || parsed.processStart !== null && (typeof parsed.processStart !== "string" || !parsed.processStart) || typeof parsed.hostname !== "string" || !parsed.hostname || !Number.isSafeInteger(parsed.createdAt) || parsed.createdAt < 0 || parsed.state !== undefined && parsed.state !== "active" && parsed.state !== "recovery-required") {
|
|
2568
|
+
throw new Error(`${label} is invalid`);
|
|
2569
|
+
}
|
|
2570
|
+
return { ...parsed, state: parsed.state ?? "active" };
|
|
2571
|
+
}
|
|
2572
|
+
function claimStaleGate(gatePath, owner, recovery) {
|
|
2573
|
+
const gate = lstatSync(gatePath);
|
|
2574
|
+
if (!gate.isDirectory() || gate.isSymbolicLink()) {
|
|
2575
|
+
throw new Error("maintenance gate must be a directory");
|
|
2576
|
+
}
|
|
2577
|
+
const ownerPath = join2(gatePath, "owner.json");
|
|
2578
|
+
const claimPath = join2(gatePath, "takeover.json");
|
|
2579
|
+
acquireTakeoverClaim(claimPath, owner);
|
|
2580
|
+
try {
|
|
2581
|
+
const before = lstatSync(ownerPath);
|
|
2582
|
+
const previous = readOwnerRecord(ownerPath, "maintenance gate owner");
|
|
2583
|
+
const status = ownerStatus(previous);
|
|
2584
|
+
if (status === "active")
|
|
2585
|
+
throw new Error("maintenance_gate_active");
|
|
2586
|
+
if (status === "unverifiable")
|
|
2587
|
+
throw new Error("maintenance_gate_unverifiable");
|
|
2588
|
+
if (status === "recovery-required") {
|
|
2589
|
+
if (recovery?.ownerID !== previous.ownerID || recovery.confirmation !== "RECOVER_RETAINED_MAINTENANCE_GATE") {
|
|
2590
|
+
throw new Error("maintenance_gate_recovery_required");
|
|
2591
|
+
}
|
|
2592
|
+
}
|
|
2593
|
+
const currentGate = lstatSync(gatePath);
|
|
2594
|
+
const currentOwner = lstatSync(ownerPath);
|
|
2595
|
+
if (currentGate.dev !== gate.dev || currentGate.ino !== gate.ino || currentOwner.dev !== before.dev || currentOwner.ino !== before.ino || readOwnerRecord(ownerPath, "maintenance gate owner").ownerID !== previous.ownerID) {
|
|
2596
|
+
throw new Error("maintenance gate changed during stale-owner verification");
|
|
2597
|
+
}
|
|
2598
|
+
renameSync(claimPath, ownerPath);
|
|
2599
|
+
fsyncPath(gatePath);
|
|
2600
|
+
return { dev: gate.dev, ino: gate.ino };
|
|
2601
|
+
} catch (error) {
|
|
2602
|
+
removeOwnedRecord(claimPath, owner.ownerID);
|
|
2603
|
+
throw error;
|
|
2604
|
+
}
|
|
2605
|
+
}
|
|
2606
|
+
function acquireTakeoverClaim(path, owner) {
|
|
2607
|
+
for (let attempt = 0;attempt < 2; attempt++) {
|
|
2608
|
+
try {
|
|
2609
|
+
writeExclusiveRecord(path, owner);
|
|
2610
|
+
return;
|
|
2611
|
+
} catch (error) {
|
|
2612
|
+
if (!isAlreadyExists(error))
|
|
2613
|
+
throw error;
|
|
2614
|
+
const before = lstatSync(path);
|
|
2615
|
+
const existing = readOwnerRecord(path, "maintenance takeover owner");
|
|
2616
|
+
const status = ownerStatus(existing);
|
|
2617
|
+
if (status === "active")
|
|
2618
|
+
throw new Error("maintenance_gate_active");
|
|
2619
|
+
if (status !== "stale")
|
|
2620
|
+
throw new Error("maintenance_gate_unverifiable");
|
|
2621
|
+
const after = lstatSync(path);
|
|
2622
|
+
if (before.dev !== after.dev || before.ino !== after.ino) {
|
|
2623
|
+
throw new Error("maintenance takeover changed during stale-owner verification");
|
|
2624
|
+
}
|
|
2625
|
+
unlinkSync(path);
|
|
2626
|
+
fsyncPath(dirname(path));
|
|
2627
|
+
}
|
|
2628
|
+
}
|
|
2629
|
+
throw new Error("maintenance_gate_active");
|
|
2630
|
+
}
|
|
2631
|
+
function replaceOwnerRecord(ownerPath, ownerID, replacement) {
|
|
2632
|
+
const temporary = `${ownerPath}.${ownerID}.tmp`;
|
|
2633
|
+
try {
|
|
2634
|
+
writeExclusiveRecord(temporary, replacement);
|
|
2635
|
+
if (readOwnerRecord(ownerPath, "maintenance gate owner").ownerID !== ownerID) {
|
|
2636
|
+
throw new Error("maintenance_gate_not_owned");
|
|
2637
|
+
}
|
|
2638
|
+
renameSync(temporary, ownerPath);
|
|
2639
|
+
fsyncPath(dirname(ownerPath));
|
|
2640
|
+
} catch (error) {
|
|
2641
|
+
removeFreshRecord(temporary);
|
|
2642
|
+
throw error;
|
|
2643
|
+
}
|
|
2644
|
+
}
|
|
2645
|
+
function removeOwnedRecord(path, ownerID) {
|
|
2646
|
+
try {
|
|
2647
|
+
if (readOwnerRecord(path, "owner record").ownerID !== ownerID)
|
|
2648
|
+
return;
|
|
2649
|
+
unlinkSync(path);
|
|
2650
|
+
} catch (error) {
|
|
2651
|
+
if (!isMissing(error))
|
|
2652
|
+
throw error;
|
|
2653
|
+
}
|
|
2654
|
+
}
|
|
2655
|
+
function removeOwnedGate(gatePath, ownerID) {
|
|
2656
|
+
const ownerPath = join2(gatePath, "owner.json");
|
|
2657
|
+
try {
|
|
2658
|
+
if (readOwnerRecord(ownerPath, "maintenance gate owner").ownerID !== ownerID)
|
|
2659
|
+
return;
|
|
2660
|
+
unlinkSync(ownerPath);
|
|
2661
|
+
rmdirSync(gatePath);
|
|
2662
|
+
} catch (error) {
|
|
2663
|
+
if (!isMissing(error))
|
|
2664
|
+
throw error;
|
|
2665
|
+
}
|
|
2666
|
+
}
|
|
2667
|
+
function removeFreshRecord(path) {
|
|
2668
|
+
try {
|
|
2669
|
+
const stat = lstatSync(path);
|
|
2670
|
+
if (!stat.isFile() || stat.isSymbolicLink())
|
|
2671
|
+
return;
|
|
2672
|
+
if (typeof process.getuid === "function" && stat.uid !== process.getuid())
|
|
2673
|
+
return;
|
|
2674
|
+
unlinkSync(path);
|
|
2675
|
+
} catch (error) {
|
|
2676
|
+
if (!isMissing(error))
|
|
2677
|
+
throw error;
|
|
2678
|
+
}
|
|
2679
|
+
}
|
|
2680
|
+
function removeFreshGate(gatePath, ownerID, device, inode) {
|
|
2681
|
+
try {
|
|
2682
|
+
const gate = lstatSync(gatePath);
|
|
2683
|
+
if (!gate.isDirectory() || gate.isSymbolicLink() || gate.dev !== device || gate.ino !== inode)
|
|
2684
|
+
return;
|
|
2685
|
+
const ownerPath = join2(gatePath, "owner.json");
|
|
2686
|
+
try {
|
|
2687
|
+
const record = readOwnerRecord(ownerPath, "maintenance gate owner");
|
|
2688
|
+
if (record.ownerID !== ownerID)
|
|
2689
|
+
return;
|
|
2690
|
+
unlinkSync(ownerPath);
|
|
2691
|
+
} catch (error) {
|
|
2692
|
+
if (!isMissing(error))
|
|
2693
|
+
removeFreshRecord(ownerPath);
|
|
2694
|
+
}
|
|
2695
|
+
rmdirSync(gatePath);
|
|
2696
|
+
} catch (error) {
|
|
2697
|
+
if (!isMissing(error))
|
|
2698
|
+
throw error;
|
|
2699
|
+
}
|
|
2700
|
+
}
|
|
2701
|
+
function fsyncPath(path) {
|
|
2702
|
+
const stat = lstatSync(path);
|
|
2703
|
+
if (platform() === "win32" && stat.isDirectory())
|
|
2704
|
+
return;
|
|
2705
|
+
const descriptor = openSync(path, platform() === "win32" ? constants.O_RDWR : constants.O_RDONLY);
|
|
2706
|
+
try {
|
|
2707
|
+
fsyncSync(descriptor);
|
|
2708
|
+
} finally {
|
|
2709
|
+
closeSync(descriptor);
|
|
2710
|
+
}
|
|
2711
|
+
}
|
|
2712
|
+
function isMissing(error) {
|
|
2713
|
+
return Boolean(error && typeof error === "object" && "code" in error && error.code === "ENOENT");
|
|
2714
|
+
}
|
|
2715
|
+
function isAlreadyExists(error) {
|
|
2716
|
+
return Boolean(error && typeof error === "object" && "code" in error && error.code === "EEXIST");
|
|
104
2717
|
}
|
|
105
2718
|
|
|
106
2719
|
// src/db/backup.ts
|
|
107
2720
|
var BACKUP_FORMAT = "agz-memory-backup/1";
|
|
108
2721
|
function createVerifiedBackup(db, databasePath, sourceSchema, targetSchema, productVersion) {
|
|
109
2722
|
const sourceHealth = assertHealthyDatabase(db);
|
|
110
|
-
|
|
111
|
-
|
|
2723
|
+
try {
|
|
2724
|
+
assertManifestSourceSchema(db, sourceHealth.schemaVersion, sourceSchema);
|
|
2725
|
+
} catch {
|
|
2726
|
+
throw new Error(`backup source schema v${sourceSchema} does not match database schema v${sourceHealth.schemaVersion ?? "unknown"}`);
|
|
112
2727
|
}
|
|
113
2728
|
const checkpoint = db.query("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
114
2729
|
if (checkpoint.busy !== 0)
|
|
115
2730
|
throw new Error("database WAL checkpoint is busy");
|
|
116
2731
|
const backupDirectory = `${databasePath}.backup`;
|
|
117
|
-
|
|
118
|
-
|
|
2732
|
+
assertNoSymbolicLinks(databasePath);
|
|
2733
|
+
assertNoSymbolicLinks(backupDirectory, true);
|
|
2734
|
+
mkdirSync2(backupDirectory, { recursive: true, mode: 448 });
|
|
2735
|
+
const backupRootStat = lstatSync2(backupDirectory);
|
|
2736
|
+
if (!backupRootStat.isDirectory() || backupRootStat.isSymbolicLink()) {
|
|
2737
|
+
throw new Error("backup root must be a directory and not a symbolic link");
|
|
2738
|
+
}
|
|
2739
|
+
chmodSync2(backupDirectory, 448);
|
|
119
2740
|
const stamp = new Date().toISOString().replace(/[:.]/g, "-");
|
|
120
|
-
const stem = `schema-v${sourceSchema}-${stamp}-${
|
|
121
|
-
const finalDatabasePath =
|
|
122
|
-
const finalManifestPath =
|
|
2741
|
+
const stem = `schema-v${sourceSchema}-${stamp}-${randomUUID3()}`;
|
|
2742
|
+
const finalDatabasePath = join3(backupDirectory, `${stem}.sqlite`);
|
|
2743
|
+
const finalManifestPath = join3(backupDirectory, `${stem}.manifest.json`);
|
|
123
2744
|
const temporaryDatabasePath = `${finalDatabasePath}.tmp`;
|
|
124
2745
|
const temporaryManifestPath = `${finalManifestPath}.tmp`;
|
|
125
2746
|
try {
|
|
126
2747
|
db.exec(`VACUUM INTO '${escapeSql(temporaryDatabasePath)}'`);
|
|
127
|
-
|
|
128
|
-
const verification = new
|
|
2748
|
+
chmodSync2(temporaryDatabasePath, 384);
|
|
2749
|
+
const verification = new Database2(temporaryDatabasePath, { readonly: true });
|
|
129
2750
|
let backupHealth;
|
|
130
2751
|
let sqliteVersion;
|
|
131
2752
|
try {
|
|
132
2753
|
backupHealth = assertHealthyDatabase(verification);
|
|
2754
|
+
assertManifestSourceSchema(verification, backupHealth.schemaVersion, sourceSchema);
|
|
133
2755
|
sqliteVersion = verification.query("SELECT sqlite_version() AS version").get().version;
|
|
134
2756
|
} finally {
|
|
135
2757
|
verification.close();
|
|
@@ -137,10 +2759,7 @@ function createVerifiedBackup(db, databasePath, sourceSchema, targetSchema, prod
|
|
|
137
2759
|
if (JSON.stringify(backupHealth.counts) !== JSON.stringify(sourceHealth.counts)) {
|
|
138
2760
|
throw new Error("backup row counts differ from source database");
|
|
139
2761
|
}
|
|
140
|
-
|
|
141
|
-
throw new Error("backup schema does not match source schema");
|
|
142
|
-
}
|
|
143
|
-
const bytes = readFileSync(temporaryDatabasePath);
|
|
2762
|
+
const digest = hashRegularFile(temporaryDatabasePath);
|
|
144
2763
|
const manifest = {
|
|
145
2764
|
format: BACKUP_FORMAT,
|
|
146
2765
|
productVersion,
|
|
@@ -148,22 +2767,22 @@ function createVerifiedBackup(db, databasePath, sourceSchema, targetSchema, prod
|
|
|
148
2767
|
targetSchema,
|
|
149
2768
|
createdAt: new Date().toISOString(),
|
|
150
2769
|
sqliteVersion,
|
|
151
|
-
databaseFile:
|
|
152
|
-
sha256:
|
|
153
|
-
size:
|
|
2770
|
+
databaseFile: basename2(finalDatabasePath),
|
|
2771
|
+
sha256: digest.sha256,
|
|
2772
|
+
size: digest.size,
|
|
154
2773
|
counts: backupHealth.counts,
|
|
155
2774
|
integrity: "ok",
|
|
156
2775
|
foreignKeyViolations: 0
|
|
157
2776
|
};
|
|
158
|
-
|
|
2777
|
+
writeFileSync2(temporaryManifestPath, `${JSON.stringify(manifest, null, 2)}
|
|
159
2778
|
`, {
|
|
160
2779
|
mode: 384
|
|
161
2780
|
});
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
2781
|
+
fsyncPath2(temporaryDatabasePath);
|
|
2782
|
+
fsyncPath2(temporaryManifestPath);
|
|
2783
|
+
renameSync2(temporaryDatabasePath, finalDatabasePath);
|
|
2784
|
+
renameSync2(temporaryManifestPath, finalManifestPath);
|
|
2785
|
+
fsyncPath2(backupDirectory);
|
|
167
2786
|
return { databasePath: finalDatabasePath, manifestPath: finalManifestPath, manifest };
|
|
168
2787
|
} catch (error) {
|
|
169
2788
|
rmSync(temporaryDatabasePath, { force: true });
|
|
@@ -172,16 +2791,20 @@ function createVerifiedBackup(db, databasePath, sourceSchema, targetSchema, prod
|
|
|
172
2791
|
}
|
|
173
2792
|
}
|
|
174
2793
|
function verifyBackupManifest(manifestPath) {
|
|
175
|
-
|
|
176
|
-
const
|
|
2794
|
+
assertNoSymbolicLinks(manifestPath);
|
|
2795
|
+
const resolvedManifestPath = resolve2(manifestPath);
|
|
2796
|
+
const manifestStat = lstatSync2(resolvedManifestPath);
|
|
177
2797
|
if (!manifestStat.isFile() || manifestStat.isSymbolicLink()) {
|
|
178
2798
|
throw new Error("backup manifest must be a regular file");
|
|
179
2799
|
}
|
|
180
|
-
const manifest = JSON.parse(
|
|
2800
|
+
const manifest = JSON.parse(readFileSync2(resolvedManifestPath, "utf8"));
|
|
181
2801
|
if (manifest.format !== BACKUP_FORMAT) {
|
|
182
2802
|
throw new Error("unsupported backup manifest format");
|
|
183
2803
|
}
|
|
184
|
-
if (
|
|
2804
|
+
if (!Number.isSafeInteger(manifest.sourceSchema) || manifest.sourceSchema < 2 || manifest.sourceSchema > 11) {
|
|
2805
|
+
throw new Error("backup manifest source schema is invalid");
|
|
2806
|
+
}
|
|
2807
|
+
if (typeof manifest.databaseFile !== "string" || !manifest.databaseFile || basename2(manifest.databaseFile) !== manifest.databaseFile) {
|
|
185
2808
|
throw new Error("backup databaseFile must be a basename");
|
|
186
2809
|
}
|
|
187
2810
|
if (typeof manifest.sha256 !== "string" || !/^[0-9a-f]{64}$/.test(manifest.sha256)) {
|
|
@@ -190,85 +2813,91 @@ function verifyBackupManifest(manifestPath) {
|
|
|
190
2813
|
if (!Number.isSafeInteger(manifest.size) || manifest.size < 0) {
|
|
191
2814
|
throw new Error("backup manifest size is invalid");
|
|
192
2815
|
}
|
|
193
|
-
const manifestDirectory =
|
|
194
|
-
const databasePath =
|
|
195
|
-
if (
|
|
2816
|
+
const manifestDirectory = dirname2(resolvedManifestPath);
|
|
2817
|
+
const databasePath = resolve2(manifestDirectory, manifest.databaseFile);
|
|
2818
|
+
if (dirname2(databasePath) !== manifestDirectory) {
|
|
196
2819
|
throw new Error("backup database file must stay inside the manifest directory");
|
|
197
2820
|
}
|
|
198
|
-
if (!
|
|
2821
|
+
if (!existsSync2(databasePath))
|
|
199
2822
|
throw new Error("backup database file is missing");
|
|
200
|
-
const databaseStat =
|
|
2823
|
+
const databaseStat = lstatSync2(databasePath);
|
|
201
2824
|
if (!databaseStat.isFile() || databaseStat.isSymbolicLink()) {
|
|
202
2825
|
throw new Error("backup database must be a regular file");
|
|
203
2826
|
}
|
|
204
|
-
const
|
|
205
|
-
|
|
206
|
-
if (hash !== manifest.sha256 || databaseStat.size !== manifest.size) {
|
|
2827
|
+
const digest = hashRegularFile(databasePath);
|
|
2828
|
+
if (digest.sha256 !== manifest.sha256 || digest.size !== manifest.size || databaseStat.size !== manifest.size) {
|
|
207
2829
|
throw new Error("backup hash or size mismatch");
|
|
208
2830
|
}
|
|
209
|
-
const db = new
|
|
2831
|
+
const db = new Database2(databasePath, { readonly: true });
|
|
210
2832
|
try {
|
|
211
2833
|
const health = assertHealthyDatabase(db);
|
|
212
2834
|
if (JSON.stringify(health.counts) !== JSON.stringify(manifest.counts)) {
|
|
213
2835
|
throw new Error("backup manifest row counts do not match");
|
|
214
2836
|
}
|
|
215
|
-
|
|
216
|
-
throw new Error("backup manifest source schema does not match database");
|
|
217
|
-
}
|
|
2837
|
+
assertManifestSourceSchema(db, health.schemaVersion, manifest.sourceSchema);
|
|
218
2838
|
} finally {
|
|
219
2839
|
db.close();
|
|
220
2840
|
}
|
|
221
2841
|
return { databasePath, manifestPath: resolvedManifestPath, manifest };
|
|
222
2842
|
}
|
|
223
|
-
function restoreVerifiedBackup(manifestPath, targetPath, confirmation) {
|
|
2843
|
+
function restoreVerifiedBackup(manifestPath, targetPath, confirmation, existingGate, expectedSha256) {
|
|
224
2844
|
if (confirmation !== "RESTORE_DATABASE_FROM_VERIFIED_BACKUP") {
|
|
225
2845
|
throw new Error("invalid restore confirmation");
|
|
226
2846
|
}
|
|
2847
|
+
ensureDatabaseParent(targetPath);
|
|
2848
|
+
const gate = existingGate ?? acquireMaintenanceGate(targetPath);
|
|
2849
|
+
const releaseGate = existingGate === undefined;
|
|
2850
|
+
try {
|
|
2851
|
+
return restoreUnderGate(manifestPath, targetPath, gate, expectedSha256);
|
|
2852
|
+
} finally {
|
|
2853
|
+
if (releaseGate)
|
|
2854
|
+
gate.release();
|
|
2855
|
+
}
|
|
2856
|
+
}
|
|
2857
|
+
function restoreUnderGate(manifestPath, targetPath, gate, expectedSha256) {
|
|
2858
|
+
assertMaintenanceGateFor(gate, targetPath);
|
|
227
2859
|
const verified = verifyBackupManifest(manifestPath);
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
2860
|
+
if (expectedSha256 !== undefined && verified.manifest.sha256 !== expectedSha256) {
|
|
2861
|
+
throw new Error("restore manifest hash mismatch");
|
|
2862
|
+
}
|
|
2863
|
+
mkdirSync2(dirname2(targetPath), { recursive: true, mode: 448 });
|
|
2864
|
+
const temporary = `${targetPath}.restore-${randomUUID3()}.tmp`;
|
|
2865
|
+
const preserved = `${targetPath}.failed-restore-source-${Date.now()}-${randomUUID3()}`;
|
|
231
2866
|
const movedSidecars = [];
|
|
232
2867
|
let hasPreservedSource = false;
|
|
233
2868
|
let preservedSourceHealthy = false;
|
|
234
2869
|
let targetInstalled = false;
|
|
235
2870
|
try {
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
fsyncPath(temporary);
|
|
239
|
-
if (existsSync(targetPath)) {
|
|
2871
|
+
copyVerifiedSource(verified, temporary);
|
|
2872
|
+
if (existsSync2(targetPath)) {
|
|
240
2873
|
preservedSourceHealthy = checkpointSource(targetPath);
|
|
241
|
-
|
|
242
|
-
chmodSync(preserved, 384);
|
|
243
|
-
fsyncPath(preserved);
|
|
244
|
-
if (preservedSourceHealthy)
|
|
245
|
-
verifyDatabaseFile(preserved);
|
|
2874
|
+
copyRegularFile(targetPath, preserved);
|
|
246
2875
|
for (const suffix of ["-wal", "-shm"]) {
|
|
247
2876
|
const source = `${targetPath}${suffix}`;
|
|
248
|
-
if (!
|
|
2877
|
+
if (!existsSync2(source))
|
|
249
2878
|
continue;
|
|
250
2879
|
const preservedSidecar = `${preserved}${suffix}`;
|
|
251
|
-
|
|
252
|
-
chmodSync(preservedSidecar, 384);
|
|
253
|
-
fsyncPath(preservedSidecar);
|
|
2880
|
+
copyRegularFile(source, preservedSidecar);
|
|
254
2881
|
}
|
|
255
|
-
|
|
2882
|
+
if (preservedSourceHealthy)
|
|
2883
|
+
verifyDatabaseFile(preserved);
|
|
2884
|
+
fsyncPath2(dirname2(targetPath));
|
|
256
2885
|
hasPreservedSource = true;
|
|
257
2886
|
}
|
|
258
2887
|
for (const suffix of ["-wal", "-shm"]) {
|
|
259
2888
|
const source = `${targetPath}${suffix}`;
|
|
260
|
-
if (!
|
|
2889
|
+
if (!existsSync2(source))
|
|
261
2890
|
continue;
|
|
262
|
-
const quarantine = `${source}.quarantine-${
|
|
263
|
-
|
|
2891
|
+
const quarantine = `${source}.quarantine-${randomUUID3()}`;
|
|
2892
|
+
quarantineRegularFile(source, quarantine);
|
|
264
2893
|
movedSidecars.push({ source, quarantine });
|
|
265
2894
|
}
|
|
266
|
-
|
|
2895
|
+
renameSync2(temporary, targetPath);
|
|
267
2896
|
targetInstalled = true;
|
|
268
|
-
|
|
269
|
-
|
|
2897
|
+
fsyncPath2(dirname2(targetPath));
|
|
2898
|
+
verifyInstalledBackup(targetPath, verified.manifest);
|
|
270
2899
|
for (const { quarantine } of movedSidecars) {
|
|
271
|
-
|
|
2900
|
+
unlinkRegularFile(quarantine);
|
|
272
2901
|
}
|
|
273
2902
|
} catch (error) {
|
|
274
2903
|
const rollbackErrors = [];
|
|
@@ -276,62 +2905,231 @@ function restoreVerifiedBackup(manifestPath, targetPath, confirmation) {
|
|
|
276
2905
|
if (targetInstalled) {
|
|
277
2906
|
try {
|
|
278
2907
|
for (const suffix of ["-wal", "-shm"]) {
|
|
279
|
-
|
|
2908
|
+
unlinkRegularFile(`${targetPath}${suffix}`, true);
|
|
280
2909
|
}
|
|
281
2910
|
if (hasPreservedSource) {
|
|
282
|
-
const rollback = `${targetPath}.rollback-${
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
rmSync(targetPath, { force: true });
|
|
287
|
-
renameSync(rollback, targetPath);
|
|
2911
|
+
const rollback = `${targetPath}.rollback-${randomUUID3()}.tmp`;
|
|
2912
|
+
copyRegularFile(preserved, rollback);
|
|
2913
|
+
unlinkRegularFile(targetPath);
|
|
2914
|
+
renameSync2(rollback, targetPath);
|
|
288
2915
|
} else {
|
|
289
|
-
|
|
2916
|
+
unlinkRegularFile(targetPath);
|
|
290
2917
|
}
|
|
291
2918
|
} catch (rollbackError) {
|
|
292
2919
|
rollbackErrors.push(rollbackError);
|
|
293
2920
|
}
|
|
294
2921
|
}
|
|
295
2922
|
for (const { source, quarantine } of movedSidecars.reverse()) {
|
|
296
|
-
if (!
|
|
2923
|
+
if (!existsSync2(quarantine))
|
|
297
2924
|
continue;
|
|
298
2925
|
try {
|
|
299
|
-
|
|
300
|
-
|
|
2926
|
+
if (existsSync2(source))
|
|
2927
|
+
throw new Error("restore sidecar path was replaced during rollback");
|
|
2928
|
+
renameSync2(quarantine, source);
|
|
301
2929
|
} catch (rollbackError) {
|
|
302
2930
|
rollbackErrors.push(rollbackError);
|
|
303
2931
|
}
|
|
304
2932
|
}
|
|
305
2933
|
if (targetInstalled && hasPreservedSource && preservedSourceHealthy && rollbackErrors.length === 0) {
|
|
306
2934
|
try {
|
|
307
|
-
|
|
2935
|
+
fsyncPath2(dirname2(targetPath));
|
|
308
2936
|
verifyDatabaseFile(targetPath);
|
|
309
2937
|
} catch (rollbackError) {
|
|
310
2938
|
rollbackErrors.push(rollbackError);
|
|
311
2939
|
}
|
|
312
2940
|
}
|
|
313
|
-
if (rollbackErrors.length > 0) {
|
|
314
|
-
|
|
2941
|
+
if (rollbackErrors.length > 0) {
|
|
2942
|
+
gate.retain();
|
|
2943
|
+
throw new AggregateError([error, ...rollbackErrors], "restore failed and rollback was incomplete");
|
|
2944
|
+
}
|
|
2945
|
+
throw error;
|
|
2946
|
+
}
|
|
2947
|
+
return preserved;
|
|
2948
|
+
}
|
|
2949
|
+
function copyVerifiedSource(verified, target) {
|
|
2950
|
+
assertNoSymbolicLinks(verified.databasePath);
|
|
2951
|
+
const source = openSync2(verified.databasePath, constants2.O_RDONLY | constants2.O_NOFOLLOW);
|
|
2952
|
+
let destination;
|
|
2953
|
+
try {
|
|
2954
|
+
const before = fstatSync(source);
|
|
2955
|
+
if (!before.isFile())
|
|
2956
|
+
throw new Error("backup database must be a regular file");
|
|
2957
|
+
destination = openSync2(target, constants2.O_WRONLY | constants2.O_CREAT | constants2.O_EXCL | constants2.O_NOFOLLOW, 384);
|
|
2958
|
+
const hash = createHash4("sha256");
|
|
2959
|
+
const buffer = Buffer.allocUnsafe(64 * 1024);
|
|
2960
|
+
let size = 0;
|
|
2961
|
+
while (true) {
|
|
2962
|
+
const read = readSync(source, buffer, 0, buffer.byteLength, null);
|
|
2963
|
+
if (read === 0)
|
|
2964
|
+
break;
|
|
2965
|
+
hash.update(buffer.subarray(0, read));
|
|
2966
|
+
let written = 0;
|
|
2967
|
+
while (written < read) {
|
|
2968
|
+
written += writeSync(destination, buffer, written, read - written);
|
|
2969
|
+
}
|
|
2970
|
+
size += read;
|
|
2971
|
+
}
|
|
2972
|
+
const after = fstatSync(source);
|
|
2973
|
+
if (before.dev !== after.dev || before.ino !== after.ino || before.size !== after.size) {
|
|
2974
|
+
throw new Error("backup database changed while it was copied");
|
|
2975
|
+
}
|
|
2976
|
+
if (size !== verified.manifest.size || hash.digest("hex") !== verified.manifest.sha256) {
|
|
2977
|
+
throw new Error("backup hash or size mismatch during restore copy");
|
|
2978
|
+
}
|
|
2979
|
+
fsyncSync2(destination);
|
|
2980
|
+
} finally {
|
|
2981
|
+
if (destination !== undefined)
|
|
2982
|
+
closeSync2(destination);
|
|
2983
|
+
closeSync2(source);
|
|
2984
|
+
}
|
|
2985
|
+
chmodSync2(target, 384);
|
|
2986
|
+
verifyDatabaseMatchesManifest(target, verified.manifest);
|
|
2987
|
+
}
|
|
2988
|
+
function verifyDatabaseMatchesManifest(path, manifest) {
|
|
2989
|
+
const db = new Database2(path, { readonly: true });
|
|
2990
|
+
try {
|
|
2991
|
+
const health = assertHealthyDatabase(db);
|
|
2992
|
+
assertManifestSourceSchema(db, health.schemaVersion, manifest.sourceSchema);
|
|
2993
|
+
if (JSON.stringify(health.counts) !== JSON.stringify(manifest.counts)) {
|
|
2994
|
+
throw new Error("backup manifest row counts do not match");
|
|
2995
|
+
}
|
|
2996
|
+
} finally {
|
|
2997
|
+
db.close();
|
|
2998
|
+
}
|
|
2999
|
+
}
|
|
3000
|
+
function verifyInstalledBackup(path, manifest) {
|
|
3001
|
+
const digest = hashRegularFile(path);
|
|
3002
|
+
if (digest.size !== manifest.size) {
|
|
3003
|
+
throw new Error("installed backup size does not match manifest");
|
|
3004
|
+
}
|
|
3005
|
+
if (digest.sha256 !== manifest.sha256) {
|
|
3006
|
+
throw new Error("installed backup hash does not match manifest");
|
|
3007
|
+
}
|
|
3008
|
+
verifyDatabaseMatchesManifest(path, manifest);
|
|
3009
|
+
}
|
|
3010
|
+
function assertManifestSourceSchema(db, actualSchema, sourceSchema) {
|
|
3011
|
+
if (!Number.isSafeInteger(sourceSchema) || sourceSchema < 2 || sourceSchema > 11) {
|
|
3012
|
+
throw new Error("backup manifest source schema does not match database");
|
|
3013
|
+
}
|
|
3014
|
+
if (actualSchema === sourceSchema) {
|
|
3015
|
+
if (sourceSchema >= 2 && sourceSchema <= 10) {
|
|
3016
|
+
try {
|
|
3017
|
+
assertLegacySchemaIdentity(db, sourceSchema);
|
|
3018
|
+
} catch {
|
|
3019
|
+
throw new Error("backup manifest source schema does not match database");
|
|
3020
|
+
}
|
|
3021
|
+
}
|
|
3022
|
+
return;
|
|
3023
|
+
}
|
|
3024
|
+
if (sourceSchema === 2 && actualSchema === undefined && hasTable3(db, "memory_items")) {
|
|
3025
|
+
try {
|
|
3026
|
+
assertLegacySchemaIdentity(db, sourceSchema);
|
|
3027
|
+
return;
|
|
3028
|
+
} catch {
|
|
3029
|
+
throw new Error("backup manifest source schema does not match database");
|
|
3030
|
+
}
|
|
3031
|
+
}
|
|
3032
|
+
throw new Error("backup manifest source schema does not match database");
|
|
3033
|
+
}
|
|
3034
|
+
function hasTable3(db, table) {
|
|
3035
|
+
return db.query("SELECT COUNT(*) AS count FROM sqlite_master WHERE type = 'table' AND name = ?").get(table).count > 0;
|
|
3036
|
+
}
|
|
3037
|
+
function hashRegularFile(path) {
|
|
3038
|
+
const descriptor = openSync2(path, constants2.O_RDONLY | constants2.O_NOFOLLOW);
|
|
3039
|
+
try {
|
|
3040
|
+
const before = fstatSync(descriptor);
|
|
3041
|
+
if (!before.isFile())
|
|
3042
|
+
throw new Error(`${path} must be a regular file`);
|
|
3043
|
+
const hash = createHash4("sha256");
|
|
3044
|
+
const buffer = Buffer.allocUnsafe(64 * 1024);
|
|
3045
|
+
let size = 0;
|
|
3046
|
+
while (true) {
|
|
3047
|
+
const read = readSync(descriptor, buffer, 0, buffer.byteLength, null);
|
|
3048
|
+
if (read === 0)
|
|
3049
|
+
break;
|
|
3050
|
+
hash.update(buffer.subarray(0, read));
|
|
3051
|
+
size += read;
|
|
3052
|
+
}
|
|
3053
|
+
const after = fstatSync(descriptor);
|
|
3054
|
+
if (before.dev !== after.dev || before.ino !== after.ino || before.size !== after.size || size !== after.size) {
|
|
3055
|
+
throw new Error(`${path} changed while it was hashed`);
|
|
3056
|
+
}
|
|
3057
|
+
return { sha256: hash.digest("hex"), size };
|
|
3058
|
+
} finally {
|
|
3059
|
+
closeSync2(descriptor);
|
|
3060
|
+
}
|
|
3061
|
+
}
|
|
3062
|
+
function copyRegularFile(sourcePath, targetPath) {
|
|
3063
|
+
assertNoSymbolicLinks(sourcePath);
|
|
3064
|
+
const source = openSync2(sourcePath, constants2.O_RDONLY | constants2.O_NOFOLLOW);
|
|
3065
|
+
let target;
|
|
3066
|
+
try {
|
|
3067
|
+
const before = fstatSync(source);
|
|
3068
|
+
if (!before.isFile())
|
|
3069
|
+
throw new Error(`${sourcePath} must be a regular file`);
|
|
3070
|
+
target = openSync2(targetPath, constants2.O_WRONLY | constants2.O_CREAT | constants2.O_EXCL | constants2.O_NOFOLLOW, 384);
|
|
3071
|
+
const buffer = Buffer.allocUnsafe(64 * 1024);
|
|
3072
|
+
while (true) {
|
|
3073
|
+
const read = readSync(source, buffer, 0, buffer.byteLength, null);
|
|
3074
|
+
if (read === 0)
|
|
3075
|
+
break;
|
|
3076
|
+
let written = 0;
|
|
3077
|
+
while (written < read)
|
|
3078
|
+
written += writeSync(target, buffer, written, read - written);
|
|
3079
|
+
}
|
|
3080
|
+
const after = fstatSync(source);
|
|
3081
|
+
if (before.dev !== after.dev || before.ino !== after.ino || before.size !== after.size) {
|
|
3082
|
+
throw new Error(`${sourcePath} changed while it was copied`);
|
|
3083
|
+
}
|
|
3084
|
+
fsyncSync2(target);
|
|
3085
|
+
} finally {
|
|
3086
|
+
if (target !== undefined)
|
|
3087
|
+
closeSync2(target);
|
|
3088
|
+
closeSync2(source);
|
|
3089
|
+
}
|
|
3090
|
+
}
|
|
3091
|
+
function quarantineRegularFile(source, quarantine) {
|
|
3092
|
+
const before = lstatSync2(source);
|
|
3093
|
+
if (!before.isFile() || before.isSymbolicLink()) {
|
|
3094
|
+
throw new Error("database sidecar must be a regular file");
|
|
3095
|
+
}
|
|
3096
|
+
renameSync2(source, quarantine);
|
|
3097
|
+
const after = lstatSync2(quarantine);
|
|
3098
|
+
if (!after.isFile() || after.isSymbolicLink() || before.dev !== after.dev || before.ino !== after.ino) {
|
|
3099
|
+
throw new Error("database sidecar changed while it was quarantined");
|
|
3100
|
+
}
|
|
3101
|
+
}
|
|
3102
|
+
function unlinkRegularFile(path, allowMissing = false) {
|
|
3103
|
+
try {
|
|
3104
|
+
const stat = lstatSync2(path);
|
|
3105
|
+
if (!stat.isFile() || stat.isSymbolicLink())
|
|
3106
|
+
throw new Error(`${path} must be a regular file`);
|
|
3107
|
+
rmSync(path);
|
|
3108
|
+
} catch (error) {
|
|
3109
|
+
if (allowMissing && error && typeof error === "object" && "code" in error && error.code === "ENOENT") {
|
|
3110
|
+
return;
|
|
315
3111
|
}
|
|
316
3112
|
throw error;
|
|
317
3113
|
}
|
|
318
|
-
return preserved;
|
|
319
3114
|
}
|
|
320
3115
|
function escapeSql(value) {
|
|
321
3116
|
return value.replaceAll("'", "''");
|
|
322
3117
|
}
|
|
323
|
-
function
|
|
324
|
-
const
|
|
3118
|
+
function fsyncPath2(path) {
|
|
3119
|
+
const stat = lstatSync2(path);
|
|
3120
|
+
if (platform2() === "win32" && stat.isDirectory())
|
|
3121
|
+
return;
|
|
3122
|
+
const descriptor = openSync2(path, platform2() === "win32" ? "r+" : "r");
|
|
325
3123
|
try {
|
|
326
|
-
|
|
3124
|
+
fsyncSync2(descriptor);
|
|
327
3125
|
} finally {
|
|
328
|
-
|
|
3126
|
+
closeSync2(descriptor);
|
|
329
3127
|
}
|
|
330
3128
|
}
|
|
331
3129
|
function checkpointSource(path) {
|
|
332
3130
|
let db;
|
|
333
3131
|
try {
|
|
334
|
-
db = new
|
|
3132
|
+
db = new Database2(path);
|
|
335
3133
|
const checkpoint = db.query("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
336
3134
|
if (checkpoint.busy !== 0)
|
|
337
3135
|
throw new Error("source database WAL checkpoint is busy");
|
|
@@ -350,7 +3148,7 @@ function checkpointSource(path) {
|
|
|
350
3148
|
}
|
|
351
3149
|
}
|
|
352
3150
|
function verifyDatabaseFile(path) {
|
|
353
|
-
const db = new
|
|
3151
|
+
const db = new Database2(path, { readonly: true });
|
|
354
3152
|
try {
|
|
355
3153
|
assertHealthyDatabase(db);
|
|
356
3154
|
} finally {
|
|
@@ -367,33 +3165,33 @@ function isBusyError(error) {
|
|
|
367
3165
|
}
|
|
368
3166
|
|
|
369
3167
|
// src/db/migration-lock.ts
|
|
370
|
-
import { randomUUID as
|
|
371
|
-
import { existsSync as
|
|
372
|
-
import { hostname } from "os";
|
|
3168
|
+
import { randomUUID as randomUUID4 } from "crypto";
|
|
3169
|
+
import { existsSync as existsSync3, mkdirSync as mkdirSync3, readFileSync as readFileSync3, renameSync as renameSync3, rmSync as rmSync2, writeFileSync as writeFileSync3 } from "fs";
|
|
3170
|
+
import { hostname as hostname2 } from "os";
|
|
373
3171
|
function migrationLockPath(databasePath) {
|
|
374
3172
|
return `${databasePath}.migration.lock`;
|
|
375
3173
|
}
|
|
376
3174
|
function acquireMigrationLock(databasePath, targetSchema, timeoutMs = 30000) {
|
|
377
3175
|
const path = migrationLockPath(databasePath);
|
|
378
3176
|
const owner = {
|
|
379
|
-
ownerID:
|
|
3177
|
+
ownerID: randomUUID4(),
|
|
380
3178
|
pid: process.pid,
|
|
381
|
-
processStartMarker:
|
|
382
|
-
hostname:
|
|
3179
|
+
processStartMarker: processStartMarker2(process.pid) ?? "unavailable",
|
|
3180
|
+
hostname: hostname2(),
|
|
383
3181
|
startedAt: Date.now(),
|
|
384
3182
|
targetSchema
|
|
385
3183
|
};
|
|
386
3184
|
const deadline = Date.now() + timeoutMs;
|
|
387
3185
|
const stagedOwner = `${path}.owner-${owner.ownerID}.tmp`;
|
|
388
|
-
|
|
3186
|
+
writeFileSync3(stagedOwner, `${JSON.stringify(owner, null, 2)}
|
|
389
3187
|
`, { mode: 384 });
|
|
390
3188
|
try {
|
|
391
3189
|
while (true) {
|
|
392
3190
|
let created = false;
|
|
393
3191
|
try {
|
|
394
|
-
|
|
3192
|
+
mkdirSync3(path, { mode: 448 });
|
|
395
3193
|
created = true;
|
|
396
|
-
|
|
3194
|
+
renameSync3(stagedOwner, `${path}/owner.json`);
|
|
397
3195
|
break;
|
|
398
3196
|
} catch (error) {
|
|
399
3197
|
if (created) {
|
|
@@ -430,7 +3228,7 @@ function acquireMigrationLock(databasePath, targetSchema, timeoutMs = 30000) {
|
|
|
430
3228
|
}
|
|
431
3229
|
function readMigrationLockOwner(path) {
|
|
432
3230
|
try {
|
|
433
|
-
return JSON.parse(
|
|
3231
|
+
return JSON.parse(readFileSync3(`${path}/owner.json`, "utf8"));
|
|
434
3232
|
} catch {
|
|
435
3233
|
return;
|
|
436
3234
|
}
|
|
@@ -442,24 +3240,24 @@ function breakMigrationLock(databasePath, ownerID, confirmation) {
|
|
|
442
3240
|
const path = migrationLockPath(databasePath);
|
|
443
3241
|
const owner = readMigrationLockOwner(path);
|
|
444
3242
|
if (!owner) {
|
|
445
|
-
if (ownerID !== "ORPHANED" || !
|
|
3243
|
+
if (ownerID !== "ORPHANED" || !existsSync3(path))
|
|
446
3244
|
throw new Error("migration lock owner mismatch");
|
|
447
3245
|
rmSync2(path, { recursive: true, force: true });
|
|
448
3246
|
return;
|
|
449
3247
|
}
|
|
450
3248
|
if (owner.ownerID !== ownerID)
|
|
451
3249
|
throw new Error("migration lock owner mismatch");
|
|
452
|
-
if (owner.hostname ===
|
|
453
|
-
const currentMarker =
|
|
3250
|
+
if (owner.hostname === hostname2() && processIsAlive(owner.pid)) {
|
|
3251
|
+
const currentMarker = processStartMarker2(owner.pid);
|
|
454
3252
|
if (!currentMarker || currentMarker === owner.processStartMarker) {
|
|
455
3253
|
throw new Error(`migration lock process ${owner.pid} is still alive`);
|
|
456
3254
|
}
|
|
457
3255
|
}
|
|
458
3256
|
rmSync2(path, { recursive: true, force: true });
|
|
459
3257
|
}
|
|
460
|
-
function
|
|
3258
|
+
function processStartMarker2(pid) {
|
|
461
3259
|
try {
|
|
462
|
-
const fields =
|
|
3260
|
+
const fields = readFileSync3(`/proc/${pid}/stat`, "utf8").trim().split(/\s+/);
|
|
463
3261
|
return fields[21];
|
|
464
3262
|
} catch {
|
|
465
3263
|
return;
|
|
@@ -473,272 +3271,12 @@ function processIsAlive(pid) {
|
|
|
473
3271
|
return false;
|
|
474
3272
|
}
|
|
475
3273
|
}
|
|
476
|
-
function isAlreadyExistsError(error) {
|
|
477
|
-
return Boolean(error && typeof error === "object" && "code" in error && String(error.code) === "EEXIST");
|
|
478
|
-
}
|
|
479
|
-
|
|
480
|
-
// src/db/migrations/v009.ts
|
|
481
|
-
import { createHash as createHash3, randomUUID as randomUUID3 } from "crypto";
|
|
482
|
-
|
|
483
|
-
// src/capture/contract.ts
|
|
484
|
-
import * as z from "zod/v4";
|
|
485
|
-
var CAPTURE_SCHEMA = "agz-memory.capture/1";
|
|
486
|
-
var SUPPORTED_OPENCODE_VERSION = "0.0.0-beta-18743";
|
|
487
|
-
var CAPTURE_EVENT_MAX_BYTES = 16 * 1024;
|
|
488
|
-
var CAPTURE_CONTENT_MAX_CHARACTERS = 4800;
|
|
489
|
-
var CAPTURE_SUMMARY_MAX_CHARACTERS = 1200;
|
|
490
|
-
var CAPTURE_SUBJECT_MAX_CHARACTERS = 240;
|
|
491
|
-
var candidateSchema = z.object({
|
|
492
|
-
kind: z.enum(KINDS),
|
|
493
|
-
title: z.string().min(1).max(240),
|
|
494
|
-
summary: z.string().min(1).max(CAPTURE_SUMMARY_MAX_CHARACTERS),
|
|
495
|
-
content: z.string().min(1).max(CAPTURE_CONTENT_MAX_CHARACTERS),
|
|
496
|
-
subjectKey: z.string().min(1).max(CAPTURE_SUBJECT_MAX_CHARACTERS).optional(),
|
|
497
|
-
intent: z.enum(["create", "supersede", "ignore", "review"]),
|
|
498
|
-
targetNoteID: z.string().min(1).max(240).optional(),
|
|
499
|
-
confidence: z.number().finite().min(0).max(1),
|
|
500
|
-
evidence: z.enum(["explicit-user", "verified-outcome", "session-summary"])
|
|
501
|
-
}).strict();
|
|
502
|
-
var signalSchema = z.object({
|
|
503
|
-
tool: z.string().min(1).max(160),
|
|
504
|
-
status: z.enum(["completed", "error"]),
|
|
505
|
-
errorType: z.string().min(1).max(160).optional()
|
|
506
|
-
}).strict();
|
|
507
|
-
var captureEventSchema = z.object({
|
|
508
|
-
schema: z.literal(CAPTURE_SCHEMA),
|
|
509
|
-
idempotencyKey: z.string().regex(/^[0-9a-f]{64}$/),
|
|
510
|
-
projectID: z.uuid(),
|
|
511
|
-
bindingKey: z.string().regex(/^[0-9a-f]{64}$/),
|
|
512
|
-
kind: z.enum(["user-candidate", "assistant-candidate", "session-summary", "tool-signal"]),
|
|
513
|
-
source: z.object({
|
|
514
|
-
system: z.literal("opencode-v2"),
|
|
515
|
-
opencodeVersion: z.literal(SUPPORTED_OPENCODE_VERSION),
|
|
516
|
-
pluginVersion: z.string().min(1).max(80),
|
|
517
|
-
sessionID: z.string().min(1).max(240),
|
|
518
|
-
messageID: z.string().min(1).max(240).optional(),
|
|
519
|
-
ordinal: z.number().int().nonnegative().optional(),
|
|
520
|
-
toolCallID: z.string().min(1).max(240).optional(),
|
|
521
|
-
observedAt: z.number().int().nonnegative()
|
|
522
|
-
}).strict(),
|
|
523
|
-
candidate: candidateSchema.optional(),
|
|
524
|
-
signal: signalSchema.optional(),
|
|
525
|
-
redaction: z.object({
|
|
526
|
-
policyVersion: z.string().min(1).max(80),
|
|
527
|
-
replacements: z.number().int().nonnegative(),
|
|
528
|
-
truncated: z.boolean()
|
|
529
|
-
}).strict()
|
|
530
|
-
}).strict().superRefine((event, context) => {
|
|
531
|
-
if (event.kind === "tool-signal" && !event.signal) {
|
|
532
|
-
context.addIssue({ code: "custom", message: "tool-signal requires signal" });
|
|
533
|
-
}
|
|
534
|
-
if (event.kind !== "tool-signal" && !event.candidate) {
|
|
535
|
-
context.addIssue({ code: "custom", message: `${event.kind} requires candidate` });
|
|
536
|
-
}
|
|
537
|
-
if (event.kind === "tool-signal" && event.candidate) {
|
|
538
|
-
context.addIssue({ code: "custom", message: "tool-signal cannot carry candidate" });
|
|
539
|
-
}
|
|
540
|
-
if (event.kind !== "tool-signal" && event.signal) {
|
|
541
|
-
context.addIssue({ code: "custom", message: `${event.kind} cannot carry signal` });
|
|
542
|
-
}
|
|
543
|
-
});
|
|
544
|
-
function parseCaptureEvent(value) {
|
|
545
|
-
const event = captureEventSchema.parse(value);
|
|
546
|
-
if (Buffer.byteLength(JSON.stringify(event), "utf8") > CAPTURE_EVENT_MAX_BYTES) {
|
|
547
|
-
throw new Error(`capture event exceeds ${CAPTURE_EVENT_MAX_BYTES} UTF-8 bytes`);
|
|
548
|
-
}
|
|
549
|
-
return event;
|
|
550
|
-
}
|
|
551
|
-
|
|
552
|
-
// src/db/schema.ts
|
|
553
|
-
var SCHEMA_TABLES = `
|
|
554
|
-
CREATE TABLE IF NOT EXISTS projects (
|
|
555
|
-
id TEXT PRIMARY KEY,
|
|
556
|
-
name TEXT NOT NULL,
|
|
557
|
-
normalized_name TEXT NOT NULL UNIQUE,
|
|
558
|
-
created_at INTEGER NOT NULL,
|
|
559
|
-
updated_at INTEGER NOT NULL
|
|
560
|
-
);
|
|
561
|
-
CREATE TABLE IF NOT EXISTS notes (
|
|
562
|
-
id TEXT PRIMARY KEY,
|
|
563
|
-
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
564
|
-
kind TEXT NOT NULL CHECK (kind IN ('decision','fact','procedure','context','research','preference','task')),
|
|
565
|
-
title TEXT NOT NULL,
|
|
566
|
-
summary TEXT NOT NULL,
|
|
567
|
-
content TEXT NOT NULL,
|
|
568
|
-
size_class TEXT NOT NULL CHECK (size_class IN ('inline','indexed')),
|
|
569
|
-
pinned INTEGER NOT NULL DEFAULT 0 CHECK (pinned IN (0,1)),
|
|
570
|
-
status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('active','superseded','archived')),
|
|
571
|
-
supersedes_id TEXT,
|
|
572
|
-
current_revision INTEGER NOT NULL DEFAULT 1 CHECK (current_revision >= 1),
|
|
573
|
-
subject_key TEXT,
|
|
574
|
-
content_hash TEXT NOT NULL CHECK (length(content_hash) = 64),
|
|
575
|
-
created_at INTEGER NOT NULL,
|
|
576
|
-
updated_at INTEGER NOT NULL,
|
|
577
|
-
UNIQUE(project_id, id)
|
|
578
|
-
);
|
|
579
|
-
CREATE INDEX IF NOT EXISTS notes_project_idx ON notes(project_id, status);
|
|
580
|
-
CREATE UNIQUE INDEX IF NOT EXISTS notes_active_subject_idx
|
|
581
|
-
ON notes(project_id, kind, subject_key)
|
|
582
|
-
WHERE status = 'active' AND subject_key IS NOT NULL;
|
|
583
|
-
CREATE TABLE IF NOT EXISTS note_edges (
|
|
584
|
-
id TEXT PRIMARY KEY,
|
|
585
|
-
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
586
|
-
source_id TEXT NOT NULL,
|
|
587
|
-
target_id TEXT NOT NULL,
|
|
588
|
-
predicate TEXT NOT NULL CHECK (predicate IN ('SUPPORTS','DERIVED_FROM','PART_OF','ABOUT','PRECEDES','SUPERSEDES')),
|
|
589
|
-
created_at INTEGER NOT NULL,
|
|
590
|
-
UNIQUE(project_id, source_id, target_id, predicate),
|
|
591
|
-
FOREIGN KEY (project_id, source_id) REFERENCES notes(project_id, id) ON DELETE CASCADE,
|
|
592
|
-
FOREIGN KEY (project_id, target_id) REFERENCES notes(project_id, id) ON DELETE CASCADE
|
|
593
|
-
);
|
|
594
|
-
CREATE INDEX IF NOT EXISTS note_edges_source_idx ON note_edges(project_id, source_id);
|
|
595
|
-
CREATE INDEX IF NOT EXISTS note_edges_target_idx ON note_edges(project_id, target_id);
|
|
596
|
-
CREATE TABLE IF NOT EXISTS project_bindings (
|
|
597
|
-
binding_key TEXT PRIMARY KEY,
|
|
598
|
-
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
599
|
-
source TEXT NOT NULL CHECK (source = 'opencode-v2'),
|
|
600
|
-
source_project_id TEXT NOT NULL,
|
|
601
|
-
workspace_id TEXT NOT NULL,
|
|
602
|
-
canonical_path_hash TEXT NOT NULL CHECK (length(canonical_path_hash) = 64),
|
|
603
|
-
created_at INTEGER NOT NULL,
|
|
604
|
-
updated_at INTEGER NOT NULL,
|
|
605
|
-
UNIQUE(source, source_project_id, workspace_id)
|
|
606
|
-
);
|
|
607
|
-
CREATE TABLE IF NOT EXISTS capture_checkpoints (
|
|
608
|
-
session_id TEXT PRIMARY KEY,
|
|
609
|
-
binding_key TEXT NOT NULL REFERENCES project_bindings(binding_key) ON DELETE CASCADE,
|
|
610
|
-
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
611
|
-
state TEXT NOT NULL CHECK (state IN ('active','idle','unavailable','closed')),
|
|
612
|
-
last_message_id TEXT,
|
|
613
|
-
last_reconciled_at INTEGER,
|
|
614
|
-
next_reconcile_at INTEGER NOT NULL,
|
|
615
|
-
failure_count INTEGER NOT NULL DEFAULT 0 CHECK (failure_count >= 0),
|
|
616
|
-
lease_owner TEXT,
|
|
617
|
-
lease_expires_at INTEGER,
|
|
618
|
-
created_at INTEGER NOT NULL,
|
|
619
|
-
updated_at INTEGER NOT NULL
|
|
620
|
-
);
|
|
621
|
-
CREATE INDEX IF NOT EXISTS capture_checkpoints_due_idx
|
|
622
|
-
ON capture_checkpoints(state, next_reconcile_at);
|
|
623
|
-
${captureEventsTable()}
|
|
624
|
-
CREATE INDEX IF NOT EXISTS capture_events_state_idx ON capture_events(state, updated_at);
|
|
625
|
-
CREATE INDEX IF NOT EXISTS capture_events_session_idx ON capture_events(project_id, source_session_id);
|
|
626
|
-
CREATE INDEX IF NOT EXISTS capture_events_note_idx ON capture_events(project_id, note_id);
|
|
627
|
-
CREATE TABLE IF NOT EXISTS note_provenance (
|
|
628
|
-
id TEXT PRIMARY KEY,
|
|
629
|
-
project_id TEXT NOT NULL,
|
|
630
|
-
note_id TEXT NOT NULL,
|
|
631
|
-
source_type TEXT NOT NULL CHECK (source_type IN ('mcp-manual','opencode-capture','migration','legacy-import','admin')),
|
|
632
|
-
capture_event_id TEXT,
|
|
633
|
-
source_session_id TEXT,
|
|
634
|
-
source_message_id TEXT,
|
|
635
|
-
source_ordinal INTEGER,
|
|
636
|
-
source_tool_call_id TEXT,
|
|
637
|
-
redaction_version TEXT,
|
|
638
|
-
extractor_version TEXT,
|
|
639
|
-
confidence REAL CHECK (confidence IS NULL OR (confidence >= 0 AND confidence <= 1)),
|
|
640
|
-
created_at INTEGER NOT NULL,
|
|
641
|
-
UNIQUE(project_id, id),
|
|
642
|
-
FOREIGN KEY (project_id, note_id) REFERENCES notes(project_id, id) ON DELETE CASCADE
|
|
643
|
-
);
|
|
644
|
-
CREATE TABLE IF NOT EXISTS note_revisions (
|
|
645
|
-
project_id TEXT NOT NULL,
|
|
646
|
-
note_id TEXT NOT NULL,
|
|
647
|
-
revision INTEGER NOT NULL CHECK (revision >= 1),
|
|
648
|
-
kind TEXT NOT NULL,
|
|
649
|
-
title TEXT NOT NULL,
|
|
650
|
-
summary TEXT NOT NULL,
|
|
651
|
-
content TEXT NOT NULL,
|
|
652
|
-
size_class TEXT NOT NULL,
|
|
653
|
-
pinned INTEGER NOT NULL CHECK (pinned IN (0,1)),
|
|
654
|
-
status TEXT NOT NULL CHECK (status IN ('active','superseded','archived')),
|
|
655
|
-
supersedes_id TEXT,
|
|
656
|
-
subject_key TEXT,
|
|
657
|
-
content_hash TEXT NOT NULL,
|
|
658
|
-
provenance_id TEXT NOT NULL,
|
|
659
|
-
created_at INTEGER NOT NULL,
|
|
660
|
-
PRIMARY KEY(project_id, note_id, revision),
|
|
661
|
-
FOREIGN KEY (project_id, note_id) REFERENCES notes(project_id, id) ON DELETE CASCADE,
|
|
662
|
-
FOREIGN KEY (project_id, provenance_id) REFERENCES note_provenance(project_id, id)
|
|
663
|
-
);
|
|
664
|
-
CREATE TABLE IF NOT EXISTS index_outbox (
|
|
665
|
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
666
|
-
backend TEXT NOT NULL,
|
|
667
|
-
operation TEXT NOT NULL CHECK (operation IN ('upsert-note','delete-note','purge-project')),
|
|
668
|
-
project_id TEXT NOT NULL,
|
|
669
|
-
note_id TEXT,
|
|
670
|
-
revision INTEGER,
|
|
671
|
-
content_hash TEXT,
|
|
672
|
-
state TEXT NOT NULL DEFAULT 'pending' CHECK (state IN ('pending','leased','succeeded','dead')),
|
|
673
|
-
attempt_count INTEGER NOT NULL DEFAULT 0 CHECK (attempt_count >= 0),
|
|
674
|
-
available_at INTEGER NOT NULL,
|
|
675
|
-
lease_owner TEXT,
|
|
676
|
-
lease_expires_at INTEGER,
|
|
677
|
-
last_error_code TEXT,
|
|
678
|
-
created_at INTEGER NOT NULL,
|
|
679
|
-
completed_at INTEGER,
|
|
680
|
-
UNIQUE(backend, operation, project_id, note_id, revision)
|
|
681
|
-
);
|
|
682
|
-
CREATE INDEX IF NOT EXISTS index_outbox_due_idx
|
|
683
|
-
ON index_outbox(backend, project_id, state, available_at, id);
|
|
684
|
-
CREATE TABLE IF NOT EXISTS schema_state (version INTEGER PRIMARY KEY);
|
|
685
|
-
`;
|
|
686
|
-
var FTS_V9 = `
|
|
687
|
-
CREATE VIRTUAL TABLE IF NOT EXISTS notes_fts USING fts5(
|
|
688
|
-
title, summary, content,
|
|
689
|
-
content='notes', content_rowid='rowid',
|
|
690
|
-
tokenize='unicode61'
|
|
691
|
-
);
|
|
692
|
-
CREATE TRIGGER IF NOT EXISTS notes_fts_ai AFTER INSERT ON notes BEGIN
|
|
693
|
-
INSERT INTO notes_fts(rowid, title, summary, content)
|
|
694
|
-
VALUES (new.rowid, new.title, new.summary, new.content);
|
|
695
|
-
END;
|
|
696
|
-
CREATE TRIGGER IF NOT EXISTS notes_fts_ad AFTER DELETE ON notes BEGIN
|
|
697
|
-
INSERT INTO notes_fts(notes_fts, rowid, title, summary, content)
|
|
698
|
-
VALUES ('delete', old.rowid, old.title, old.summary, old.content);
|
|
699
|
-
END;
|
|
700
|
-
CREATE TRIGGER IF NOT EXISTS notes_fts_au AFTER UPDATE OF title, summary, content ON notes BEGIN
|
|
701
|
-
INSERT INTO notes_fts(notes_fts, rowid, title, summary, content)
|
|
702
|
-
VALUES ('delete', old.rowid, old.title, old.summary, old.content);
|
|
703
|
-
INSERT INTO notes_fts(rowid, title, summary, content)
|
|
704
|
-
VALUES (new.rowid, new.title, new.summary, new.content);
|
|
705
|
-
END;
|
|
706
|
-
`;
|
|
707
|
-
function createSchema(db) {
|
|
708
|
-
db.exec(SCHEMA_TABLES);
|
|
709
|
-
db.exec(FTS_V9);
|
|
710
|
-
db.query("DELETE FROM schema_state").run();
|
|
711
|
-
db.query("INSERT INTO schema_state(version) VALUES (10)").run();
|
|
712
|
-
}
|
|
713
|
-
function rebuildFts(db) {
|
|
714
|
-
db.query("INSERT INTO notes_fts(notes_fts) VALUES ('rebuild')").run();
|
|
715
|
-
}
|
|
716
|
-
function captureEventsTable(table = "capture_events") {
|
|
717
|
-
return `CREATE TABLE IF NOT EXISTS ${table} (
|
|
718
|
-
idempotency_key TEXT PRIMARY KEY,
|
|
719
|
-
contract TEXT NOT NULL CHECK (contract = '${CAPTURE_SCHEMA}'),
|
|
720
|
-
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
721
|
-
binding_key TEXT NOT NULL REFERENCES project_bindings(binding_key) ON DELETE CASCADE,
|
|
722
|
-
event_kind TEXT NOT NULL CHECK (event_kind IN ('user-candidate','assistant-candidate','session-summary','tool-signal')),
|
|
723
|
-
source_session_id TEXT NOT NULL,
|
|
724
|
-
source_message_id TEXT,
|
|
725
|
-
source_ordinal INTEGER CHECK (source_ordinal IS NULL OR source_ordinal >= 0),
|
|
726
|
-
source_tool_call_id TEXT,
|
|
727
|
-
payload_json TEXT CHECK (payload_json IS NULL OR json_valid(payload_json)),
|
|
728
|
-
payload_hash TEXT,
|
|
729
|
-
redaction_version TEXT NOT NULL,
|
|
730
|
-
state TEXT NOT NULL CHECK (state IN ('pending','shadowed','review','materialized','duplicate','ignored','rejected','quarantined','failed','dead')),
|
|
731
|
-
attempt_count INTEGER NOT NULL DEFAULT 0 CHECK (attempt_count >= 0),
|
|
732
|
-
note_id TEXT,
|
|
733
|
-
last_error_code TEXT,
|
|
734
|
-
generation INTEGER NOT NULL DEFAULT 0 CHECK (generation >= 0),
|
|
735
|
-
created_at INTEGER NOT NULL,
|
|
736
|
-
updated_at INTEGER NOT NULL,
|
|
737
|
-
processed_at INTEGER
|
|
738
|
-
);`;
|
|
3274
|
+
function isAlreadyExistsError(error) {
|
|
3275
|
+
return Boolean(error && typeof error === "object" && "code" in error && String(error.code) === "EEXIST");
|
|
739
3276
|
}
|
|
740
3277
|
|
|
741
3278
|
// src/db/migrations/v009.ts
|
|
3279
|
+
import { createHash as createHash5, randomUUID as randomUUID5 } from "crypto";
|
|
742
3280
|
function migrateV8ToV9(db) {
|
|
743
3281
|
const notes = db.query("SELECT * FROM notes ORDER BY rowid").all();
|
|
744
3282
|
db.exec(`
|
|
@@ -791,7 +3329,7 @@ function migrateV8ToV9(db) {
|
|
|
791
3329
|
`);
|
|
792
3330
|
const hashes = new Map;
|
|
793
3331
|
for (const note of notes) {
|
|
794
|
-
const hash =
|
|
3332
|
+
const hash = noteContentHash2(note.kind, note.title, note.summary, note.content);
|
|
795
3333
|
hashes.set(note.id, hash);
|
|
796
3334
|
insert.run(note.id, note.project_id, note.kind, note.title, note.summary, note.content, note.size_class, note.pinned, note.status, note.supersedes_id, hash, note.created_at, note.updated_at);
|
|
797
3335
|
}
|
|
@@ -807,7 +3345,7 @@ function migrateV8ToV9(db) {
|
|
|
807
3345
|
`);
|
|
808
3346
|
db.exec(SCHEMA_TABLES);
|
|
809
3347
|
for (const note of notes) {
|
|
810
|
-
const provenanceID =
|
|
3348
|
+
const provenanceID = randomUUID5();
|
|
811
3349
|
db.query(`
|
|
812
3350
|
INSERT INTO note_provenance
|
|
813
3351
|
(id, project_id, note_id, source_type, created_at)
|
|
@@ -825,12 +3363,12 @@ function migrateV8ToV9(db) {
|
|
|
825
3363
|
db.query("DELETE FROM schema_state").run();
|
|
826
3364
|
db.query("INSERT INTO schema_state(version) VALUES (9)").run();
|
|
827
3365
|
}
|
|
828
|
-
function
|
|
829
|
-
return
|
|
3366
|
+
function noteContentHash2(kind, title, summary, content) {
|
|
3367
|
+
return createHash5("sha256").update(`${kind}\x00${title}\x00${summary}\x00${content}`, "utf8").digest("hex");
|
|
830
3368
|
}
|
|
831
3369
|
|
|
832
3370
|
// src/db/migrations/v010.ts
|
|
833
|
-
import { createHash as
|
|
3371
|
+
import { createHash as createHash6 } from "crypto";
|
|
834
3372
|
function migrateV9ToV10(db) {
|
|
835
3373
|
const payloads = db.query("SELECT idempotency_key, payload_json FROM capture_events WHERE payload_json IS NOT NULL").all();
|
|
836
3374
|
const migratedPayloads = payloads.map((row) => {
|
|
@@ -843,7 +3381,7 @@ function migrateV9ToV10(db) {
|
|
|
843
3381
|
return {
|
|
844
3382
|
idempotencyKey: row.idempotency_key,
|
|
845
3383
|
payload,
|
|
846
|
-
payloadHash:
|
|
3384
|
+
payloadHash: createHash6("sha256").update(payload, "utf8").digest("hex")
|
|
847
3385
|
};
|
|
848
3386
|
});
|
|
849
3387
|
db.exec("DROP TABLE IF EXISTS capture_events_v10");
|
|
@@ -878,7 +3416,7 @@ function migrateV9ToV10(db) {
|
|
|
878
3416
|
}
|
|
879
3417
|
|
|
880
3418
|
// src/version.ts
|
|
881
|
-
var PRODUCT_VERSION = "0.
|
|
3419
|
+
var PRODUCT_VERSION = "0.5.0";
|
|
882
3420
|
|
|
883
3421
|
// src/db.ts
|
|
884
3422
|
var DDL = `
|
|
@@ -920,41 +3458,144 @@ CREATE INDEX IF NOT EXISTS note_edges_source_idx ON note_edges(project_id, sourc
|
|
|
920
3458
|
CREATE INDEX IF NOT EXISTS note_edges_target_idx ON note_edges(project_id, target_id);
|
|
921
3459
|
CREATE TABLE IF NOT EXISTS schema_state (version INTEGER PRIMARY KEY);
|
|
922
3460
|
`;
|
|
3461
|
+
var PRE_OPEN_PROBE_TIMEOUT_MS = 5000;
|
|
923
3462
|
function openMemoryDatabase(path) {
|
|
924
|
-
|
|
3463
|
+
ensureDatabaseParent(path);
|
|
3464
|
+
assertSupportedDatabaseBeforeOpen(path);
|
|
3465
|
+
recoverStaleMaintenanceGate(path, () => assertSupportedDatabaseBeforeOpen(path));
|
|
3466
|
+
let lock = acquireMigrationLock(path, SCHEMA_VERSION);
|
|
3467
|
+
let lease = acquireDatabaseLease(path);
|
|
3468
|
+
let db;
|
|
3469
|
+
try {
|
|
3470
|
+
assertSupportedDatabaseBeforeOpen(path);
|
|
3471
|
+
db = openDatabase(path);
|
|
3472
|
+
} catch (error) {
|
|
3473
|
+
lease.release();
|
|
3474
|
+
lock.release();
|
|
3475
|
+
throw error;
|
|
3476
|
+
}
|
|
925
3477
|
let dbOpen = true;
|
|
926
|
-
let
|
|
3478
|
+
let maintenance;
|
|
927
3479
|
let backup;
|
|
928
3480
|
try {
|
|
929
3481
|
const existingVersion = getSchemaVersion(db);
|
|
930
3482
|
if (existingVersion && existingVersion.version > SCHEMA_VERSION) {
|
|
931
3483
|
throw new Error(`database schema v${existingVersion.version} is newer than supported v${SCHEMA_VERSION}`);
|
|
932
3484
|
}
|
|
933
|
-
const
|
|
934
|
-
if (!
|
|
3485
|
+
const hasObjects = hasApplicationObjects(db);
|
|
3486
|
+
if (!hasObjects) {
|
|
3487
|
+
db.close();
|
|
3488
|
+
dbOpen = false;
|
|
3489
|
+
lease.release();
|
|
3490
|
+
lease = undefined;
|
|
3491
|
+
lease = acquireDatabaseLease(path);
|
|
3492
|
+
assertSupportedDatabaseBeforeOpen(path);
|
|
3493
|
+
db = openDatabase(path);
|
|
3494
|
+
dbOpen = true;
|
|
3495
|
+
if (hasApplicationObjects(db)) {
|
|
3496
|
+
const initializedVersion = getSchemaVersion(db);
|
|
3497
|
+
if (initializedVersion?.version !== SCHEMA_VERSION) {
|
|
3498
|
+
throw new Error("database changed during initialization");
|
|
3499
|
+
}
|
|
3500
|
+
db.exec("PRAGMA foreign_keys=ON");
|
|
3501
|
+
assertHealthyDatabase(db);
|
|
3502
|
+
lock.release();
|
|
3503
|
+
lock = undefined;
|
|
3504
|
+
const opened3 = openedWithLease(db, lease);
|
|
3505
|
+
lease = undefined;
|
|
3506
|
+
return opened3;
|
|
3507
|
+
}
|
|
3508
|
+
db.exec("PRAGMA foreign_keys=ON");
|
|
3509
|
+
db.exec(`PRAGMA application_id = ${APPLICATION_ID}`);
|
|
3510
|
+
db.transaction(() => createSchemaV11(db))();
|
|
3511
|
+
assertHealthyDatabase(db);
|
|
3512
|
+
lock.release();
|
|
3513
|
+
lock = undefined;
|
|
3514
|
+
const opened2 = openedWithLease(db, lease);
|
|
3515
|
+
lease = undefined;
|
|
3516
|
+
return opened2;
|
|
3517
|
+
}
|
|
3518
|
+
const hasV11Marker = hasV11IdentityMarker(db);
|
|
3519
|
+
if (!existingVersion && hasV11Marker) {
|
|
3520
|
+
assertSchemaV11(db);
|
|
3521
|
+
}
|
|
3522
|
+
if (existingVersion && existingVersion.version < SCHEMA_VERSION && hasV11Marker) {
|
|
3523
|
+
assertSchemaV11(db);
|
|
3524
|
+
}
|
|
3525
|
+
if (!existingVersion && !hasLegacyV2(db))
|
|
3526
|
+
throw new Error("unrecognized_database");
|
|
3527
|
+
if (existingVersion?.version === SCHEMA_VERSION) {
|
|
935
3528
|
db.exec("PRAGMA foreign_keys=ON");
|
|
936
|
-
db.transaction(() => createSchema(db))();
|
|
937
3529
|
assertHealthyDatabase(db);
|
|
938
|
-
|
|
3530
|
+
lock.release();
|
|
3531
|
+
lock = undefined;
|
|
3532
|
+
const opened2 = openedWithLease(db, lease);
|
|
3533
|
+
lease = undefined;
|
|
3534
|
+
return opened2;
|
|
939
3535
|
}
|
|
940
3536
|
if ((existingVersion?.version ?? 0) < SCHEMA_VERSION) {
|
|
941
3537
|
db.close();
|
|
942
3538
|
dbOpen = false;
|
|
943
|
-
|
|
3539
|
+
lease.release();
|
|
3540
|
+
lease = undefined;
|
|
3541
|
+
lease = acquireDatabaseLease(path);
|
|
3542
|
+
assertSupportedDatabaseBeforeOpen(path);
|
|
944
3543
|
db = openDatabase(path);
|
|
945
3544
|
dbOpen = true;
|
|
946
|
-
|
|
3545
|
+
let migrationVersion = getSchemaVersion(db);
|
|
947
3546
|
if (migrationVersion && migrationVersion.version > SCHEMA_VERSION) {
|
|
948
3547
|
throw new Error(`database schema v${migrationVersion.version} is newer than supported v${SCHEMA_VERSION}`);
|
|
949
3548
|
}
|
|
3549
|
+
if (!migrationVersion && hasV11IdentityMarker(db))
|
|
3550
|
+
assertSchemaV11(db);
|
|
3551
|
+
if (migrationVersion && migrationVersion.version < SCHEMA_VERSION && hasV11IdentityMarker(db)) {
|
|
3552
|
+
assertSchemaV11(db);
|
|
3553
|
+
}
|
|
3554
|
+
if (!migrationVersion && !hasLegacyV2(db))
|
|
3555
|
+
throw new Error("unrecognized_database");
|
|
950
3556
|
if ((migrationVersion?.version ?? 0) === SCHEMA_VERSION) {
|
|
3557
|
+
db.exec("PRAGMA foreign_keys=ON");
|
|
3558
|
+
assertHealthyDatabase(db);
|
|
951
3559
|
lock.release();
|
|
952
3560
|
lock = undefined;
|
|
3561
|
+
const opened3 = openedWithLease(db, lease);
|
|
3562
|
+
lease = undefined;
|
|
3563
|
+
return opened3;
|
|
3564
|
+
}
|
|
3565
|
+
db.close();
|
|
3566
|
+
dbOpen = false;
|
|
3567
|
+
lease.release();
|
|
3568
|
+
lease = undefined;
|
|
3569
|
+
maintenance = acquireMaintenanceGate(path);
|
|
3570
|
+
assertSupportedDatabaseBeforeOpen(path);
|
|
3571
|
+
db = openDatabase(path);
|
|
3572
|
+
dbOpen = true;
|
|
3573
|
+
migrationVersion = getSchemaVersion(db);
|
|
3574
|
+
if (migrationVersion && migrationVersion.version > SCHEMA_VERSION) {
|
|
3575
|
+
throw new Error(`database schema v${migrationVersion.version} is newer than supported v${SCHEMA_VERSION}`);
|
|
3576
|
+
}
|
|
3577
|
+
if (!migrationVersion && hasV11IdentityMarker(db))
|
|
3578
|
+
assertSchemaV11(db);
|
|
3579
|
+
if (migrationVersion && migrationVersion.version < SCHEMA_VERSION && hasV11IdentityMarker(db)) {
|
|
3580
|
+
assertSchemaV11(db);
|
|
3581
|
+
}
|
|
3582
|
+
if (!migrationVersion && !hasLegacyV2(db))
|
|
3583
|
+
throw new Error("unrecognized_database");
|
|
3584
|
+
if ((migrationVersion?.version ?? 0) === SCHEMA_VERSION) {
|
|
3585
|
+
db.close();
|
|
3586
|
+
dbOpen = false;
|
|
3587
|
+
const handoff2 = handoffMaintenanceToLease(path, maintenance);
|
|
3588
|
+
maintenance = undefined;
|
|
3589
|
+
db = handoff2.db;
|
|
3590
|
+
dbOpen = true;
|
|
3591
|
+
lease = handoff2.lease;
|
|
953
3592
|
db.exec("PRAGMA foreign_keys=ON");
|
|
954
|
-
db.exec(SCHEMA_TABLES);
|
|
955
|
-
db.exec(FTS_V9);
|
|
956
3593
|
assertHealthyDatabase(db);
|
|
957
|
-
|
|
3594
|
+
lock.release();
|
|
3595
|
+
lock = undefined;
|
|
3596
|
+
const opened3 = openedWithLease(db, lease);
|
|
3597
|
+
lease = undefined;
|
|
3598
|
+
return opened3;
|
|
958
3599
|
}
|
|
959
3600
|
backup = createVerifiedBackup(db, path, migrationVersion?.version ?? 2, SCHEMA_VERSION, PRODUCT_VERSION);
|
|
960
3601
|
db.exec("PRAGMA foreign_keys=OFF");
|
|
@@ -980,43 +3621,120 @@ function openMemoryDatabase(path) {
|
|
|
980
3621
|
db.transaction(() => migrateV8ToV9(db))();
|
|
981
3622
|
version = 9;
|
|
982
3623
|
}
|
|
983
|
-
if (version < 10)
|
|
3624
|
+
if (version < 10) {
|
|
984
3625
|
db.transaction(() => migrateV9ToV10(db))();
|
|
3626
|
+
version = 10;
|
|
3627
|
+
}
|
|
3628
|
+
if (version < SCHEMA_VERSION) {
|
|
3629
|
+
db.transaction(() => {
|
|
3630
|
+
db.exec(`PRAGMA application_id = ${APPLICATION_ID}`);
|
|
3631
|
+
migrateV10ToV11(db);
|
|
3632
|
+
})();
|
|
3633
|
+
}
|
|
985
3634
|
db.exec("PRAGMA foreign_keys=ON");
|
|
986
3635
|
if (db.query("PRAGMA foreign_keys").get().foreign_keys !== 1) {
|
|
987
3636
|
throw new Error("failed to enable database foreign keys");
|
|
988
3637
|
}
|
|
989
3638
|
assertHealthyDatabase(db);
|
|
990
3639
|
console.warn(`[agz-memory] migrated to v${SCHEMA_VERSION} (backup: ${backup.manifestPath})`);
|
|
3640
|
+
backup = undefined;
|
|
3641
|
+
db.close();
|
|
3642
|
+
dbOpen = false;
|
|
3643
|
+
const handoff = handoffMaintenanceToLease(path, maintenance);
|
|
3644
|
+
maintenance = undefined;
|
|
3645
|
+
db = handoff.db;
|
|
3646
|
+
dbOpen = true;
|
|
3647
|
+
lease = handoff.lease;
|
|
3648
|
+
db.exec("PRAGMA foreign_keys=ON");
|
|
3649
|
+
assertHealthyDatabase(db);
|
|
991
3650
|
lock.release();
|
|
992
3651
|
lock = undefined;
|
|
993
|
-
|
|
3652
|
+
const opened2 = openedWithLease(db, lease);
|
|
3653
|
+
lease = undefined;
|
|
3654
|
+
return opened2;
|
|
994
3655
|
}
|
|
995
3656
|
db.exec("PRAGMA foreign_keys=ON");
|
|
996
|
-
db.exec(SCHEMA_TABLES);
|
|
997
|
-
db.exec(FTS_V9);
|
|
998
3657
|
assertHealthyDatabase(db);
|
|
999
3658
|
db.exec("PRAGMA foreign_keys=ON");
|
|
1000
|
-
|
|
3659
|
+
lock.release();
|
|
3660
|
+
lock = undefined;
|
|
3661
|
+
const opened = openedWithLease(db, lease);
|
|
3662
|
+
lease = undefined;
|
|
3663
|
+
return opened;
|
|
1001
3664
|
} catch (error) {
|
|
1002
3665
|
if (dbOpen)
|
|
1003
3666
|
db.close();
|
|
1004
3667
|
if (backup) {
|
|
1005
3668
|
try {
|
|
1006
|
-
restoreVerifiedBackup(backup.manifestPath, path, "RESTORE_DATABASE_FROM_VERIFIED_BACKUP");
|
|
3669
|
+
restoreVerifiedBackup(backup.manifestPath, path, "RESTORE_DATABASE_FROM_VERIFIED_BACKUP", maintenance);
|
|
1007
3670
|
} catch (restoreError) {
|
|
1008
3671
|
throw new AggregateError([error, restoreError], "migration and automatic restore failed");
|
|
1009
3672
|
}
|
|
1010
3673
|
}
|
|
1011
3674
|
throw error;
|
|
1012
3675
|
} finally {
|
|
3676
|
+
lease?.release();
|
|
3677
|
+
maintenance?.release();
|
|
1013
3678
|
lock?.release();
|
|
1014
3679
|
}
|
|
1015
3680
|
}
|
|
3681
|
+
function openReadOnlyMemoryDatabase(path) {
|
|
3682
|
+
assertSupportedDatabaseBeforeOpen(path);
|
|
3683
|
+
recoverStaleMaintenanceGate(path, () => assertSupportedDatabaseBeforeOpen(path));
|
|
3684
|
+
const lease = acquireDatabaseLease(path);
|
|
3685
|
+
try {
|
|
3686
|
+
assertSupportedDatabaseBeforeOpen(path);
|
|
3687
|
+
assertDatabasePath(path);
|
|
3688
|
+
const db = new Database3(path, { readonly: true });
|
|
3689
|
+
try {
|
|
3690
|
+
assertDatabasePath(path);
|
|
3691
|
+
assertSupportedDatabase(db);
|
|
3692
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
3693
|
+
return openedWithLease(db, lease);
|
|
3694
|
+
} catch (error) {
|
|
3695
|
+
db.close();
|
|
3696
|
+
throw error;
|
|
3697
|
+
}
|
|
3698
|
+
} catch (error) {
|
|
3699
|
+
lease.release();
|
|
3700
|
+
throw error;
|
|
3701
|
+
}
|
|
3702
|
+
}
|
|
3703
|
+
function handoffMaintenanceToLease(path, maintenance) {
|
|
3704
|
+
maintenance.release();
|
|
3705
|
+
const lease = acquireDatabaseLease(path);
|
|
3706
|
+
try {
|
|
3707
|
+
return { db: openDatabase(path), lease };
|
|
3708
|
+
} catch (error) {
|
|
3709
|
+
lease.release();
|
|
3710
|
+
throw error;
|
|
3711
|
+
}
|
|
3712
|
+
}
|
|
3713
|
+
function openedWithLease(db, lease) {
|
|
3714
|
+
let closed = false;
|
|
3715
|
+
return {
|
|
3716
|
+
db,
|
|
3717
|
+
close: () => {
|
|
3718
|
+
if (closed)
|
|
3719
|
+
return;
|
|
3720
|
+
closed = true;
|
|
3721
|
+
try {
|
|
3722
|
+
db.close();
|
|
3723
|
+
} finally {
|
|
3724
|
+
lease.release();
|
|
3725
|
+
}
|
|
3726
|
+
}
|
|
3727
|
+
};
|
|
3728
|
+
}
|
|
1016
3729
|
function openDatabase(path) {
|
|
1017
|
-
|
|
3730
|
+
assertDatabasePath(path);
|
|
3731
|
+
const db = new Database3(path, { create: true });
|
|
1018
3732
|
try {
|
|
1019
|
-
|
|
3733
|
+
assertDatabasePath(path);
|
|
3734
|
+
db.exec("PRAGMA query_only=ON");
|
|
3735
|
+
assertSupportedDatabase(db);
|
|
3736
|
+
db.exec("PRAGMA query_only=OFF");
|
|
3737
|
+
chmodSync3(path, 384);
|
|
1020
3738
|
db.exec("PRAGMA busy_timeout=5000");
|
|
1021
3739
|
db.exec("PRAGMA journal_mode=WAL");
|
|
1022
3740
|
return db;
|
|
@@ -1025,14 +3743,97 @@ function openDatabase(path) {
|
|
|
1025
3743
|
throw error;
|
|
1026
3744
|
}
|
|
1027
3745
|
}
|
|
3746
|
+
function assertSupportedDatabaseBeforeOpen(path) {
|
|
3747
|
+
const deadline = Date.now() + PRE_OPEN_PROBE_TIMEOUT_MS;
|
|
3748
|
+
while (true) {
|
|
3749
|
+
try {
|
|
3750
|
+
assertSupportedDatabaseBeforeOpenOnce(path);
|
|
3751
|
+
return;
|
|
3752
|
+
} catch (error) {
|
|
3753
|
+
if (!isSQLiteBusyError(error) || Date.now() >= deadline)
|
|
3754
|
+
throw error;
|
|
3755
|
+
Bun.sleepSync(Math.min(50, Math.max(1, deadline - Date.now())));
|
|
3756
|
+
}
|
|
3757
|
+
}
|
|
3758
|
+
}
|
|
3759
|
+
function assertSupportedDatabaseBeforeOpenOnce(path) {
|
|
3760
|
+
assertDatabasePath(path);
|
|
3761
|
+
if (!existsSync4(path))
|
|
3762
|
+
return;
|
|
3763
|
+
const db = new Database3(path, { readonly: true });
|
|
3764
|
+
try {
|
|
3765
|
+
assertDatabasePath(path);
|
|
3766
|
+
assertSupportedDatabase(db);
|
|
3767
|
+
} finally {
|
|
3768
|
+
db.close();
|
|
3769
|
+
}
|
|
3770
|
+
}
|
|
3771
|
+
function assertSupportedDatabase(db) {
|
|
3772
|
+
const existingVersion = getSchemaVersion(db);
|
|
3773
|
+
if (existingVersion && existingVersion.version > SCHEMA_VERSION) {
|
|
3774
|
+
throw new Error(`database schema v${existingVersion.version} is newer than supported v${SCHEMA_VERSION}`);
|
|
3775
|
+
}
|
|
3776
|
+
if (!hasApplicationObjects(db))
|
|
3777
|
+
return;
|
|
3778
|
+
const hasV11Marker = hasV11IdentityMarker(db);
|
|
3779
|
+
if (!existingVersion) {
|
|
3780
|
+
if (!hasLegacyV2(db))
|
|
3781
|
+
throw new Error("unrecognized_database");
|
|
3782
|
+
assertLegacySchemaIdentity(db, 2);
|
|
3783
|
+
return;
|
|
3784
|
+
}
|
|
3785
|
+
if (existingVersion.version === SCHEMA_VERSION || hasV11Marker) {
|
|
3786
|
+
assertSchemaV11(db);
|
|
3787
|
+
return;
|
|
3788
|
+
}
|
|
3789
|
+
if (existingVersion.version < 2 || existingVersion.version > 10) {
|
|
3790
|
+
throw new Error("unrecognized_database");
|
|
3791
|
+
}
|
|
3792
|
+
assertLegacySchemaIdentity(db, existingVersion.version);
|
|
3793
|
+
}
|
|
3794
|
+
function assertDatabasePath(path) {
|
|
3795
|
+
try {
|
|
3796
|
+
if (lstatSync3(path).isSymbolicLink()) {
|
|
3797
|
+
throw new Error("database path must not be a symbolic link");
|
|
3798
|
+
}
|
|
3799
|
+
} catch (error) {
|
|
3800
|
+
if (error instanceof Error && "code" in error && error.code === "ENOENT")
|
|
3801
|
+
return;
|
|
3802
|
+
throw error;
|
|
3803
|
+
}
|
|
3804
|
+
}
|
|
1028
3805
|
function getSchemaVersion(db) {
|
|
1029
|
-
if (!
|
|
3806
|
+
if (!hasTable4(db, "schema_state"))
|
|
1030
3807
|
return;
|
|
1031
|
-
|
|
3808
|
+
let rows;
|
|
3809
|
+
try {
|
|
3810
|
+
rows = db.query("SELECT version FROM schema_state").all();
|
|
3811
|
+
} catch (error) {
|
|
3812
|
+
if (isSQLiteBusyError(error))
|
|
3813
|
+
throw error;
|
|
3814
|
+
throw new Error("unrecognized_database");
|
|
3815
|
+
}
|
|
3816
|
+
if (rows.length !== 1 || typeof rows[0]?.version !== "number" || !Number.isSafeInteger(rows[0].version)) {
|
|
3817
|
+
throw new Error("unrecognized_database");
|
|
3818
|
+
}
|
|
3819
|
+
return { version: rows[0].version };
|
|
1032
3820
|
}
|
|
1033
3821
|
function migrateToV8(db) {
|
|
1034
3822
|
db.transaction(() => {
|
|
1035
3823
|
adoptLegacyProjectIDs(db);
|
|
3824
|
+
importLegacyAssociations(db);
|
|
3825
|
+
if (hasColumn(db, "notes", "current_revision")) {
|
|
3826
|
+
dropLegacyV2Tables(db);
|
|
3827
|
+
db.query("DELETE FROM schema_state").run();
|
|
3828
|
+
db.query("INSERT INTO schema_state (version) VALUES (8)").run();
|
|
3829
|
+
return;
|
|
3830
|
+
}
|
|
3831
|
+
db.exec(`
|
|
3832
|
+
DROP TRIGGER IF EXISTS notes_fts_ai;
|
|
3833
|
+
DROP TRIGGER IF EXISTS notes_fts_ad;
|
|
3834
|
+
DROP TRIGGER IF EXISTS notes_fts_au;
|
|
3835
|
+
DROP TABLE IF EXISTS notes_fts;
|
|
3836
|
+
`);
|
|
1036
3837
|
const pinned = hasColumn(db, "notes", "pinned") ? "pinned" : "0";
|
|
1037
3838
|
db.exec(`
|
|
1038
3839
|
CREATE TABLE notes_v7 (
|
|
@@ -1077,7 +3878,7 @@ function migrateToV8(db) {
|
|
|
1077
3878
|
ALTER TABLE notes_v7 RENAME TO notes;
|
|
1078
3879
|
ALTER TABLE note_edges_v7 RENAME TO note_edges;
|
|
1079
3880
|
`);
|
|
1080
|
-
|
|
3881
|
+
dropLegacyV2Tables(db);
|
|
1081
3882
|
db.query("DELETE FROM schema_state").run();
|
|
1082
3883
|
db.query("INSERT INTO schema_state (version) VALUES (8)").run();
|
|
1083
3884
|
})();
|
|
@@ -1087,7 +3888,7 @@ function adoptLegacyProjectIDs(db) {
|
|
|
1087
3888
|
for (const { id: legacyID } of existingProjects) {
|
|
1088
3889
|
if (isUUID(legacyID))
|
|
1089
3890
|
continue;
|
|
1090
|
-
const id =
|
|
3891
|
+
const id = randomUUID6();
|
|
1091
3892
|
db.query("UPDATE projects SET id = ? WHERE id = ?").run(id, legacyID);
|
|
1092
3893
|
db.query("UPDATE notes SET project_id = ? WHERE project_id = ?").run(id, legacyID);
|
|
1093
3894
|
db.query("UPDATE note_edges SET project_id = ? WHERE project_id = ?").run(id, legacyID);
|
|
@@ -1096,7 +3897,7 @@ function adoptLegacyProjectIDs(db) {
|
|
|
1096
3897
|
for (const { project_id: legacyID } of rows) {
|
|
1097
3898
|
if (db.query("SELECT id FROM projects WHERE id = ?").get(legacyID))
|
|
1098
3899
|
continue;
|
|
1099
|
-
const id =
|
|
3900
|
+
const id = randomUUID6();
|
|
1100
3901
|
const name = uniqueLegacyProjectName(db, legacyID);
|
|
1101
3902
|
const now = Date.now();
|
|
1102
3903
|
db.query("INSERT INTO projects (id, name, normalized_name, created_at, updated_at) VALUES (?, ?, ?, ?, ?)").run(id, name, normalizeProjectName(name), now, now);
|
|
@@ -1121,9 +3922,18 @@ function hasColumn(db, table, column) {
|
|
|
1121
3922
|
return rows.some((row) => row.name === column);
|
|
1122
3923
|
}
|
|
1123
3924
|
function hasLegacyV2(db) {
|
|
1124
|
-
return
|
|
3925
|
+
return hasTable4(db, "memory_items");
|
|
1125
3926
|
}
|
|
1126
|
-
function
|
|
3927
|
+
function hasApplicationObjects(db) {
|
|
3928
|
+
const row = db.query("SELECT COUNT(*) AS count FROM sqlite_master WHERE name NOT LIKE 'sqlite_%'").get();
|
|
3929
|
+
return row.count > 0;
|
|
3930
|
+
}
|
|
3931
|
+
function hasV11IdentityMarker(db) {
|
|
3932
|
+
if (hasTable4(db, "agz_meta"))
|
|
3933
|
+
return true;
|
|
3934
|
+
return db.query("PRAGMA application_id").get().application_id === APPLICATION_ID;
|
|
3935
|
+
}
|
|
3936
|
+
function hasTable4(db, table) {
|
|
1127
3937
|
const row = db.query("SELECT COUNT(*) AS n FROM sqlite_master WHERE type='table' AND name = ?").get(table);
|
|
1128
3938
|
return (row?.n ?? 0) > 0;
|
|
1129
3939
|
}
|
|
@@ -1145,26 +3955,42 @@ var KIND_MAP = {
|
|
|
1145
3955
|
};
|
|
1146
3956
|
function migrateFromV2(db, path) {
|
|
1147
3957
|
const requiredTables = ["memory_items", "memory_versions", "memory_identities"];
|
|
1148
|
-
const missingTables = requiredTables.filter((table) => !
|
|
3958
|
+
const missingTables = requiredTables.filter((table) => !hasTable4(db, table));
|
|
1149
3959
|
if (missingTables.length > 0) {
|
|
1150
3960
|
throw new Error(`unsupported legacy schema; missing tables: ${missingTables.join(", ")}`);
|
|
1151
3961
|
}
|
|
1152
3962
|
const backup = `${path}.v2-backup`;
|
|
1153
|
-
if (!
|
|
3963
|
+
if (!existsSync4(backup)) {
|
|
1154
3964
|
db.exec("PRAGMA wal_checkpoint(TRUNCATE)");
|
|
1155
3965
|
copyFileSync2(path, backup);
|
|
1156
3966
|
}
|
|
1157
3967
|
db.transaction(() => {
|
|
1158
3968
|
migrateFromV2Data(db, backup, {
|
|
1159
|
-
documents: ["document_sources", "document_chunks", "memories"].every((table) =>
|
|
1160
|
-
links:
|
|
1161
|
-
edges:
|
|
3969
|
+
documents: ["document_sources", "document_chunks", "memories"].every((table) => hasTable4(db, table)),
|
|
3970
|
+
links: hasTable4(db, "memory_links"),
|
|
3971
|
+
edges: hasTable4(db, "memory_edges")
|
|
1162
3972
|
});
|
|
1163
3973
|
adoptLegacyProjectIDs(db);
|
|
3974
|
+
dropLegacyV2Tables(db);
|
|
1164
3975
|
db.query("DELETE FROM schema_state").run();
|
|
1165
3976
|
db.query("INSERT INTO schema_state (version) VALUES (8)").run();
|
|
1166
3977
|
})();
|
|
1167
3978
|
}
|
|
3979
|
+
function dropLegacyV2Tables(db) {
|
|
3980
|
+
for (const table of [
|
|
3981
|
+
"document_chunks",
|
|
3982
|
+
"document_sources",
|
|
3983
|
+
"memories",
|
|
3984
|
+
"memory_associations",
|
|
3985
|
+
"memory_edges",
|
|
3986
|
+
"memory_links",
|
|
3987
|
+
"memory_identities",
|
|
3988
|
+
"memory_versions",
|
|
3989
|
+
"memory_items"
|
|
3990
|
+
]) {
|
|
3991
|
+
db.exec(`DROP TABLE IF EXISTS ${table}`);
|
|
3992
|
+
}
|
|
3993
|
+
}
|
|
1168
3994
|
function migrateFromV2Data(db, backup, options) {
|
|
1169
3995
|
const now = Date.now();
|
|
1170
3996
|
db.query("DELETE FROM notes_fts").run();
|
|
@@ -1208,7 +4034,7 @@ function migrateFromV2Data(db, backup, options) {
|
|
|
1208
4034
|
const content = source.body ?? "";
|
|
1209
4035
|
if (!content.trim())
|
|
1210
4036
|
continue;
|
|
1211
|
-
const id =
|
|
4037
|
+
const id = randomUUID6();
|
|
1212
4038
|
db.query(`INSERT INTO notes (id, project_id, kind, title, summary, content, size_class, status, supersedes_id, created_at, updated_at)
|
|
1213
4039
|
VALUES (?, ?, 'research', ?, ?, ?, 'indexed', 'active', NULL, ?, ?)`).run(id, source.project_root ? hashRoot(source.project_root) : "legacy", source.title, content.slice(0, 200), content, source.created_at, source.updated_at);
|
|
1214
4040
|
db.query("INSERT INTO notes_fts (id, title, summary, content) VALUES (?, ?, ?, ?)").run(id, source.title, content.slice(0, 200), content);
|
|
@@ -1247,7 +4073,7 @@ function migrateFromV2Data(db, backup, options) {
|
|
|
1247
4073
|
console.warn(`[agz-memory] v2\u2192v3 migration complete: ${migratedNotes} notes, ${migratedEdges} edges (backup: ${backup})`);
|
|
1248
4074
|
}
|
|
1249
4075
|
function importLegacyAssociations(db) {
|
|
1250
|
-
if (!
|
|
4076
|
+
if (!hasTable4(db, "memory_associations"))
|
|
1251
4077
|
return 0;
|
|
1252
4078
|
const associations = db.query(`SELECT id, left_item_id, right_item_id, kind, created_at
|
|
1253
4079
|
FROM memory_associations
|
|
@@ -1287,6 +4113,13 @@ function doctorDatabase(db) {
|
|
|
1287
4113
|
failures.push("schema_newer_than_runtime");
|
|
1288
4114
|
else if (health.schemaVersion < SCHEMA_VERSION)
|
|
1289
4115
|
warnings.push("schema_upgrade_required");
|
|
4116
|
+
if (health.schemaVersion === SCHEMA_VERSION) {
|
|
4117
|
+
try {
|
|
4118
|
+
assertSchemaV11(db);
|
|
4119
|
+
} catch {
|
|
4120
|
+
failures.push("schema_fingerprint_mismatch");
|
|
4121
|
+
}
|
|
4122
|
+
}
|
|
1290
4123
|
if (hasTable(db, "notes_fts") && hasTable(db, "notes")) {
|
|
1291
4124
|
invariants.notes = count(db, "SELECT COUNT(*) AS count FROM notes");
|
|
1292
4125
|
invariants.fts = count(db, "SELECT COUNT(*) AS count FROM notes_fts");
|
|
@@ -1301,6 +4134,34 @@ function doctorDatabase(db) {
|
|
|
1301
4134
|
AND r.note_id = n.id
|
|
1302
4135
|
AND r.revision = n.current_revision
|
|
1303
4136
|
WHERE r.note_id IS NULL`);
|
|
4137
|
+
invariants.noteContentHashMismatches = canonicalHashMismatches(db, "notes");
|
|
4138
|
+
invariants.revisionContentHashMismatches = canonicalHashMismatches(db, "note_revisions");
|
|
4139
|
+
invariants.currentRevisionMismatches = count(db, `SELECT COUNT(*) AS count
|
|
4140
|
+
FROM notes n
|
|
4141
|
+
JOIN note_revisions r
|
|
4142
|
+
ON r.project_id = n.project_id
|
|
4143
|
+
AND r.note_id = n.id
|
|
4144
|
+
AND r.revision = n.current_revision
|
|
4145
|
+
WHERE n.kind IS NOT r.kind
|
|
4146
|
+
OR n.title IS NOT r.title
|
|
4147
|
+
OR n.summary IS NOT r.summary
|
|
4148
|
+
OR n.content IS NOT r.content
|
|
4149
|
+
OR n.size_class IS NOT r.size_class
|
|
4150
|
+
OR n.pinned IS NOT r.pinned
|
|
4151
|
+
OR n.status IS NOT r.status
|
|
4152
|
+
OR n.supersedes_id IS NOT r.supersedes_id
|
|
4153
|
+
OR n.subject_key IS NOT r.subject_key
|
|
4154
|
+
OR n.content_hash IS NOT r.content_hash`);
|
|
4155
|
+
invariants.revisionGaps = count(db, `SELECT COUNT(*) AS count FROM (
|
|
4156
|
+
SELECT n.project_id, n.id
|
|
4157
|
+
FROM notes n
|
|
4158
|
+
LEFT JOIN note_revisions r
|
|
4159
|
+
ON r.project_id = n.project_id AND r.note_id = n.id
|
|
4160
|
+
GROUP BY n.project_id, n.id, n.current_revision
|
|
4161
|
+
HAVING COUNT(r.revision) <> n.current_revision
|
|
4162
|
+
OR MIN(r.revision) IS NOT 1
|
|
4163
|
+
OR MAX(r.revision) IS NOT n.current_revision
|
|
4164
|
+
)`);
|
|
1304
4165
|
invariants.missingProvenance = count(db, `SELECT COUNT(*) AS count
|
|
1305
4166
|
FROM note_revisions r
|
|
1306
4167
|
LEFT JOIN note_provenance p
|
|
@@ -1313,120 +4174,104 @@ function doctorDatabase(db) {
|
|
|
1313
4174
|
HAVING COUNT(DISTINCT project_id) > 1
|
|
1314
4175
|
)`);
|
|
1315
4176
|
invariants.deadOutbox = count(db, "SELECT COUNT(*) AS count FROM index_outbox WHERE state = 'dead'");
|
|
4177
|
+
invariants.invalidOutboxOperations = count(db, `SELECT COUNT(*) AS count FROM index_outbox
|
|
4178
|
+
WHERE (operation = 'upsert-note'
|
|
4179
|
+
AND (note_id IS NULL OR revision IS NULL OR revision < 1
|
|
4180
|
+
OR content_hash IS NULL OR length(content_hash) <> 64))
|
|
4181
|
+
OR (operation = 'delete-note'
|
|
4182
|
+
AND (note_id IS NULL OR revision IS NULL OR revision < 1
|
|
4183
|
+
OR content_hash IS NOT NULL))
|
|
4184
|
+
OR (operation = 'purge-project'
|
|
4185
|
+
AND (note_id IS NOT NULL OR revision IS NOT NULL OR content_hash IS NOT NULL))
|
|
4186
|
+
OR operation NOT IN ('upsert-note', 'delete-note', 'purge-project')`);
|
|
4187
|
+
invariants.invalidOutboxLeases = count(db, `SELECT COUNT(*) AS count FROM index_outbox
|
|
4188
|
+
WHERE (state = 'leased' AND (lease_owner IS NULL OR lease_expires_at IS NULL))
|
|
4189
|
+
OR (state <> 'leased'
|
|
4190
|
+
AND (lease_owner IS NOT NULL OR lease_expires_at IS NOT NULL OR heartbeat_at IS NOT NULL))`);
|
|
4191
|
+
invariants.orphanedOutboxProjects = count(db, `SELECT COUNT(*) AS count FROM index_outbox o
|
|
4192
|
+
LEFT JOIN projects p ON p.id = o.project_id
|
|
4193
|
+
WHERE p.id IS NULL`);
|
|
4194
|
+
invariants.outboxOperationKeyMismatches = outboxOperationKeyMismatches(db);
|
|
4195
|
+
invariants.derivedHashMismatches = derivedHashMismatches(db);
|
|
1316
4196
|
invariants.dueCheckpoints = count(db, "SELECT COUNT(*) AS count FROM capture_checkpoints WHERE next_reconcile_at <= ? AND state = 'active'", Date.now());
|
|
1317
4197
|
for (const [name, value] of Object.entries(invariants)) {
|
|
1318
|
-
if ([
|
|
4198
|
+
if ([
|
|
4199
|
+
"missingCurrentRevisions",
|
|
4200
|
+
"noteContentHashMismatches",
|
|
4201
|
+
"revisionContentHashMismatches",
|
|
4202
|
+
"currentRevisionMismatches",
|
|
4203
|
+
"revisionGaps",
|
|
4204
|
+
"missingProvenance",
|
|
4205
|
+
"bindingConflicts",
|
|
4206
|
+
"invalidOutboxOperations",
|
|
4207
|
+
"invalidOutboxLeases",
|
|
4208
|
+
"outboxOperationKeyMismatches",
|
|
4209
|
+
"derivedHashMismatches"
|
|
4210
|
+
].includes(name) && value > 0) {
|
|
1319
4211
|
failures.push(name);
|
|
1320
4212
|
}
|
|
1321
4213
|
}
|
|
1322
4214
|
}
|
|
1323
4215
|
return { ok: failures.length === 0, health, warnings, failures, invariants };
|
|
1324
4216
|
}
|
|
4217
|
+
function canonicalHashMismatches(db, table) {
|
|
4218
|
+
const rows = db.query(`SELECT kind, title, summary, content, content_hash FROM ${table}`).iterate();
|
|
4219
|
+
let mismatches = 0;
|
|
4220
|
+
for (const row of rows) {
|
|
4221
|
+
if (row.content_hash !== noteContentHash(row.kind, row.title, row.summary, row.content)) {
|
|
4222
|
+
mismatches++;
|
|
4223
|
+
}
|
|
4224
|
+
}
|
|
4225
|
+
return mismatches;
|
|
4226
|
+
}
|
|
1325
4227
|
function count(db, sql, ...bindings) {
|
|
1326
4228
|
return db.query(sql).get(...bindings).count;
|
|
1327
4229
|
}
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
{
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
{ name: "github-token", pattern: /\b(?:gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,})\b/g },
|
|
1347
|
-
{ name: "gitlab-token", pattern: /\bglpat-[A-Za-z0-9_-]{16,}\b/g },
|
|
1348
|
-
{ name: "aws-access-key", pattern: /\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/g },
|
|
1349
|
-
{ name: "jwt", pattern: /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g },
|
|
1350
|
-
{
|
|
1351
|
-
name: "secret-assignment",
|
|
1352
|
-
pattern: /\b(?:PASSWORD|PASSWD|SECRET|TOKEN|API_KEY|PRIVATE_KEY)\s*[:=]\s*["']?[^\s,"']{8,}["']?/gi
|
|
1353
|
-
}
|
|
1354
|
-
];
|
|
1355
|
-
function redactText(value, options = {}) {
|
|
1356
|
-
const maxCharacters = options.maxCharacters ?? Number.MAX_SAFE_INTEGER;
|
|
1357
|
-
let text = value;
|
|
1358
|
-
let replacements = 0;
|
|
1359
|
-
let highRisk = 0;
|
|
1360
|
-
const classes = {};
|
|
1361
|
-
for (const literal2 of options.denylist ?? []) {
|
|
1362
|
-
if (!literal2)
|
|
1363
|
-
continue;
|
|
1364
|
-
const count2 = text.split(literal2).length - 1;
|
|
1365
|
-
if (count2 === 0)
|
|
1366
|
-
continue;
|
|
1367
|
-
replacements += count2;
|
|
1368
|
-
classes.denylist = (classes.denylist ?? 0) + count2;
|
|
1369
|
-
text = text.replaceAll(literal2, "[REDACTED:denylist]");
|
|
1370
|
-
}
|
|
1371
|
-
for (const rule of RULES) {
|
|
1372
|
-
text = text.replace(rule.pattern, () => {
|
|
1373
|
-
replacements++;
|
|
1374
|
-
classes[rule.name] = (classes[rule.name] ?? 0) + 1;
|
|
1375
|
-
if (rule.highRisk)
|
|
1376
|
-
highRisk++;
|
|
1377
|
-
return `[REDACTED:${rule.name}]`;
|
|
1378
|
-
});
|
|
4230
|
+
function outboxOperationKeyMismatches(db) {
|
|
4231
|
+
const rows = db.query(`SELECT backend, operation, project_id, note_id, revision, content_hash,
|
|
4232
|
+
generation, operation_key
|
|
4233
|
+
FROM index_outbox
|
|
4234
|
+
ORDER BY id`).all();
|
|
4235
|
+
let mismatches = 0;
|
|
4236
|
+
for (const row of rows) {
|
|
4237
|
+
const expected = hashTuple("outbox-operation", 2, [
|
|
4238
|
+
row.backend,
|
|
4239
|
+
row.operation,
|
|
4240
|
+
row.project_id,
|
|
4241
|
+
row.note_id,
|
|
4242
|
+
row.revision,
|
|
4243
|
+
row.content_hash,
|
|
4244
|
+
row.generation
|
|
4245
|
+
]);
|
|
4246
|
+
if (row.operation_key !== expected)
|
|
4247
|
+
mismatches++;
|
|
1379
4248
|
}
|
|
1380
|
-
|
|
1381
|
-
if (!looksHighEntropy(candidate))
|
|
1382
|
-
return candidate;
|
|
1383
|
-
replacements++;
|
|
1384
|
-
classes.entropy = (classes.entropy ?? 0) + 1;
|
|
1385
|
-
return "[REDACTED:entropy]";
|
|
1386
|
-
});
|
|
1387
|
-
const truncated = text.length > maxCharacters;
|
|
1388
|
-
if (truncated)
|
|
1389
|
-
text = text.slice(0, maxCharacters);
|
|
1390
|
-
return {
|
|
1391
|
-
text,
|
|
1392
|
-
replacements,
|
|
1393
|
-
classes,
|
|
1394
|
-
truncated,
|
|
1395
|
-
quarantined: highRisk > 0 || replacements >= 3
|
|
1396
|
-
};
|
|
4249
|
+
return mismatches;
|
|
1397
4250
|
}
|
|
1398
|
-
function
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
4251
|
+
function derivedHashMismatches(db) {
|
|
4252
|
+
const rows = db.query(`SELECT o.content_hash, n.project_id, n.id, n.current_revision,
|
|
4253
|
+
n.kind, n.title, n.summary, n.content
|
|
4254
|
+
FROM index_outbox o
|
|
4255
|
+
JOIN notes n ON n.project_id = o.project_id AND n.id = o.note_id
|
|
4256
|
+
WHERE o.operation = 'upsert-note'
|
|
4257
|
+
AND o.state IN ('pending', 'leased')
|
|
4258
|
+
AND n.status = 'active'
|
|
4259
|
+
AND n.current_revision = o.revision`).all();
|
|
4260
|
+
let mismatches = 0;
|
|
4261
|
+
for (const row of rows) {
|
|
4262
|
+
const document = deriveDocument({
|
|
4263
|
+
projectID: row.project_id,
|
|
4264
|
+
noteID: row.id,
|
|
4265
|
+
revision: row.current_revision,
|
|
4266
|
+
kind: row.kind,
|
|
4267
|
+
title: row.title,
|
|
4268
|
+
summary: row.summary,
|
|
4269
|
+
content: row.content
|
|
4270
|
+
});
|
|
4271
|
+
if (!document || document.contentHash !== row.content_hash)
|
|
4272
|
+
mismatches++;
|
|
1408
4273
|
}
|
|
1409
|
-
return
|
|
1410
|
-
}
|
|
1411
|
-
|
|
1412
|
-
// src/retrieval/derived.ts
|
|
1413
|
-
function deriveDocument(source) {
|
|
1414
|
-
const title = redactText(source.title);
|
|
1415
|
-
const summary = redactText(source.summary);
|
|
1416
|
-
const content = redactText(source.content);
|
|
1417
|
-
if (title.quarantined || summary.quarantined || content.quarantined)
|
|
1418
|
-
return;
|
|
1419
|
-
const contentHash = createHash5("sha256").update(`${source.kind}\x00${title.text}\x00${summary.text}\x00${content.text}`, "utf8").digest("hex");
|
|
1420
|
-
return {
|
|
1421
|
-
projectID: source.projectID,
|
|
1422
|
-
noteID: source.noteID,
|
|
1423
|
-
revision: source.revision,
|
|
1424
|
-
kind: source.kind,
|
|
1425
|
-
title: title.text,
|
|
1426
|
-
summary: summary.text,
|
|
1427
|
-
content: content.text,
|
|
1428
|
-
contentHash
|
|
1429
|
-
};
|
|
4274
|
+
return mismatches;
|
|
1430
4275
|
}
|
|
1431
4276
|
|
|
1432
4277
|
// src/admin/index.ts
|
|
@@ -1437,24 +4282,24 @@ async function runAdmin(argv = process.argv.slice(2)) {
|
|
|
1437
4282
|
throw new Error("admin command is required");
|
|
1438
4283
|
if (command === "doctor") {
|
|
1439
4284
|
requireExistingDatabase(databasePath);
|
|
1440
|
-
const
|
|
4285
|
+
const opened = openReadOnlyMemoryDatabase(databasePath);
|
|
1441
4286
|
try {
|
|
1442
|
-
return doctorDatabase(db);
|
|
4287
|
+
return doctorDatabase(opened.db);
|
|
1443
4288
|
} finally {
|
|
1444
|
-
|
|
4289
|
+
opened.close();
|
|
1445
4290
|
}
|
|
1446
4291
|
}
|
|
1447
4292
|
if (command === "backup" && subcommand !== "prune") {
|
|
1448
4293
|
requireExistingDatabase(databasePath);
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
}
|
|
4294
|
+
return withExclusiveMaintenance(databasePath, readSchemaVersion(databasePath), () => {
|
|
4295
|
+
const db = new Database4(databasePath);
|
|
4296
|
+
try {
|
|
4297
|
+
const version = schemaVersion(db);
|
|
4298
|
+
return createVerifiedBackup(db, databasePath, version, version, PRODUCT_VERSION);
|
|
4299
|
+
} finally {
|
|
4300
|
+
db.close();
|
|
4301
|
+
}
|
|
4302
|
+
});
|
|
1458
4303
|
}
|
|
1459
4304
|
if (command === "upgrade") {
|
|
1460
4305
|
if (option(argv, "--to") !== String(SCHEMA_VERSION)) {
|
|
@@ -1480,13 +4325,22 @@ async function runAdmin(argv = process.argv.slice(2)) {
|
|
|
1480
4325
|
}
|
|
1481
4326
|
if (expectedHash !== verified.manifest.sha256)
|
|
1482
4327
|
throw new Error("restore manifest hash mismatch");
|
|
1483
|
-
const
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
}
|
|
1488
|
-
|
|
4328
|
+
const maintenanceOwner = option(argv, "--maintenance-owner");
|
|
4329
|
+
const maintenanceConfirmation = option(argv, "--maintenance-confirm");
|
|
4330
|
+
if (maintenanceOwner === undefined !== (maintenanceConfirmation === undefined)) {
|
|
4331
|
+
throw new Error("retained maintenance recovery requires both owner and confirmation");
|
|
4332
|
+
}
|
|
4333
|
+
const recovery = maintenanceOwner && maintenanceConfirmation ? {
|
|
4334
|
+
ownerID: maintenanceOwner,
|
|
4335
|
+
confirmation: maintenanceConfirmation
|
|
4336
|
+
} : undefined;
|
|
4337
|
+
if (recovery && recovery.confirmation !== "RECOVER_RETAINED_MAINTENANCE_GATE") {
|
|
4338
|
+
throw new Error("invalid retained maintenance recovery confirmation");
|
|
1489
4339
|
}
|
|
4340
|
+
return withExclusiveMaintenance(databasePath, verified.manifest.sourceSchema, (maintenance) => {
|
|
4341
|
+
const preservedPath = restoreVerifiedBackup(manifestPath, databasePath, confirmation, maintenance, expectedHash);
|
|
4342
|
+
return { restored: true, preservedPath, manifest: verified.manifest };
|
|
4343
|
+
}, recovery);
|
|
1490
4344
|
}
|
|
1491
4345
|
if (command === "unlock") {
|
|
1492
4346
|
const ownerID = option(argv, "--owner");
|
|
@@ -1505,14 +4359,26 @@ async function runAdmin(argv = process.argv.slice(2)) {
|
|
|
1505
4359
|
try {
|
|
1506
4360
|
const now = Date.now();
|
|
1507
4361
|
let queued = 0;
|
|
1508
|
-
|
|
4362
|
+
let purges = 0;
|
|
4363
|
+
const quarantined = {};
|
|
4364
|
+
let generation = 0;
|
|
1509
4365
|
const insert = opened.db.query(`
|
|
1510
|
-
INSERT
|
|
1511
|
-
(backend, operation, project_id, note_id, revision, content_hash,
|
|
1512
|
-
state, attempt_count, available_at,
|
|
1513
|
-
|
|
4366
|
+
INSERT INTO index_outbox
|
|
4367
|
+
(backend, operation_key, operation, project_id, note_id, revision, content_hash,
|
|
4368
|
+
generation, lease_generation, fence, state, attempt_count, available_at,
|
|
4369
|
+
heartbeat_at, created_at)
|
|
4370
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0, 0, 'pending', 0, ?, NULL, ?)
|
|
1514
4371
|
`);
|
|
1515
|
-
opened.db.
|
|
4372
|
+
opened.db.exec("BEGIN IMMEDIATE");
|
|
4373
|
+
try {
|
|
4374
|
+
generation = opened.db.query("SELECT COALESCE(MAX(generation), 0) + 1 AS generation FROM index_outbox WHERE backend = ?").get(backend).generation;
|
|
4375
|
+
const projects = opened.db.query("SELECT id FROM projects ORDER BY id").all();
|
|
4376
|
+
const notes = opened.db.query("SELECT * FROM notes WHERE status = 'active' ORDER BY project_id, id").all();
|
|
4377
|
+
for (const project of projects) {
|
|
4378
|
+
const operation = "purge-project";
|
|
4379
|
+
const operationKey = outboxOperationKey(backend, operation, project.id, null, null, null, generation);
|
|
4380
|
+
purges += insert.run(backend, operationKey, operation, project.id, null, null, null, generation, now, now).changes;
|
|
4381
|
+
}
|
|
1516
4382
|
for (const note of notes) {
|
|
1517
4383
|
const document = deriveDocument({
|
|
1518
4384
|
projectID: note.project_id,
|
|
@@ -1523,10 +4389,22 @@ async function runAdmin(argv = process.argv.slice(2)) {
|
|
|
1523
4389
|
summary: note.summary,
|
|
1524
4390
|
content: note.content
|
|
1525
4391
|
});
|
|
1526
|
-
|
|
4392
|
+
if (!document) {
|
|
4393
|
+
quarantined.derived_document_unavailable = (quarantined.derived_document_unavailable ?? 0) + 1;
|
|
4394
|
+
continue;
|
|
4395
|
+
}
|
|
4396
|
+
const operation = "upsert-note";
|
|
4397
|
+
const operationKey = outboxOperationKey(backend, operation, note.project_id, note.id, note.current_revision, document.contentHash, generation);
|
|
4398
|
+
queued += insert.run(backend, operationKey, operation, note.project_id, note.id, note.current_revision, document.contentHash, generation, now, now).changes;
|
|
1527
4399
|
}
|
|
1528
|
-
|
|
1529
|
-
|
|
4400
|
+
opened.db.exec("COMMIT");
|
|
4401
|
+
} catch (error) {
|
|
4402
|
+
try {
|
|
4403
|
+
opened.db.exec("ROLLBACK");
|
|
4404
|
+
} catch {}
|
|
4405
|
+
throw error;
|
|
4406
|
+
}
|
|
4407
|
+
return { backend, generation, purges, queued, quarantined };
|
|
1530
4408
|
} finally {
|
|
1531
4409
|
opened.close();
|
|
1532
4410
|
}
|
|
@@ -1544,7 +4422,8 @@ async function runAdmin(argv = process.argv.slice(2)) {
|
|
|
1544
4422
|
return withDatabase(databasePath, (db) => {
|
|
1545
4423
|
const result = db.query(`UPDATE index_outbox
|
|
1546
4424
|
SET state = 'pending', available_at = ?, lease_owner = NULL,
|
|
1547
|
-
lease_expires_at = NULL,
|
|
4425
|
+
lease_expires_at = NULL, heartbeat_at = NULL,
|
|
4426
|
+
completed_at = NULL, last_error_code = NULL
|
|
1548
4427
|
WHERE id = ? AND state = 'dead'`).run(Date.now(), id);
|
|
1549
4428
|
return { id, retried: result.changes === 1 };
|
|
1550
4429
|
});
|
|
@@ -1556,28 +4435,30 @@ async function runAdmin(argv = process.argv.slice(2)) {
|
|
|
1556
4435
|
}));
|
|
1557
4436
|
}
|
|
1558
4437
|
if (command === "backup" && subcommand === "prune") {
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
4438
|
+
return withExclusiveMaintenance(databasePath, SCHEMA_VERSION, () => {
|
|
4439
|
+
const entries = backupEntries(databasePath);
|
|
4440
|
+
const root = resolve3(`${databasePath}.backup`);
|
|
4441
|
+
const digest = createHash7("sha256").update(`${resolve3(databasePath)}\x00${root}
|
|
4442
|
+
${entries.map((entry) => `${basename3(entry.manifest)}\x00${basename3(entry.database)}\x00${entry.sha256}\x00${entry.size}\x00${entry.manifestHash}`).join(`
|
|
1563
4443
|
`)}`).digest("hex");
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
4444
|
+
if (option(argv, "--confirm") !== "DELETE_VERIFIED_BACKUPS") {
|
|
4445
|
+
return { dryRun: true, digest, backups: entries };
|
|
4446
|
+
}
|
|
4447
|
+
if (option(argv, "--digest") !== digest)
|
|
4448
|
+
throw new Error("backup prune digest mismatch");
|
|
4449
|
+
const currentEntries = entries.map((entry) => {
|
|
4450
|
+
const current = verifiedBackupEntry(root, entry.manifest);
|
|
4451
|
+
if (current.database !== entry.database || current.sha256 !== entry.sha256 || current.size !== entry.size || current.manifestHash !== entry.manifestHash) {
|
|
4452
|
+
throw new Error(`backup changed after confirmation: ${basename3(entry.manifest)}`);
|
|
4453
|
+
}
|
|
4454
|
+
return current;
|
|
4455
|
+
});
|
|
4456
|
+
for (const current of currentEntries) {
|
|
4457
|
+
rmSync3(current.database, { force: true });
|
|
4458
|
+
rmSync3(current.manifest, { force: true });
|
|
1573
4459
|
}
|
|
1574
|
-
return
|
|
4460
|
+
return { deleted: entries.length, digest };
|
|
1575
4461
|
});
|
|
1576
|
-
for (const current of currentEntries) {
|
|
1577
|
-
rmSync3(current.database, { force: true });
|
|
1578
|
-
rmSync3(current.manifest, { force: true });
|
|
1579
|
-
}
|
|
1580
|
-
return { deleted: entries.length, digest };
|
|
1581
4462
|
}
|
|
1582
4463
|
throw new Error(`unknown admin command: ${argv.join(" ")}`);
|
|
1583
4464
|
}
|
|
@@ -1589,16 +4470,29 @@ function withDatabase(databasePath, action) {
|
|
|
1589
4470
|
opened.close();
|
|
1590
4471
|
}
|
|
1591
4472
|
}
|
|
4473
|
+
function withExclusiveMaintenance(databasePath, schemaVersion, action, recovery) {
|
|
4474
|
+
const lock = acquireMigrationLock(databasePath, schemaVersion);
|
|
4475
|
+
try {
|
|
4476
|
+
const maintenance = acquireMaintenanceGate(databasePath, recovery);
|
|
4477
|
+
try {
|
|
4478
|
+
return action(maintenance);
|
|
4479
|
+
} finally {
|
|
4480
|
+
maintenance.release();
|
|
4481
|
+
}
|
|
4482
|
+
} finally {
|
|
4483
|
+
lock.release();
|
|
4484
|
+
}
|
|
4485
|
+
}
|
|
1592
4486
|
function option(argv, name) {
|
|
1593
4487
|
const index = argv.indexOf(name);
|
|
1594
4488
|
return index >= 0 ? argv[index + 1] : undefined;
|
|
1595
4489
|
}
|
|
1596
4490
|
function requireExistingDatabase(databasePath) {
|
|
1597
|
-
if (!
|
|
4491
|
+
if (!existsSync5(databasePath))
|
|
1598
4492
|
throw new Error(`database does not exist: ${databasePath}`);
|
|
1599
4493
|
}
|
|
1600
4494
|
function readSchemaVersion(databasePath) {
|
|
1601
|
-
const db = new
|
|
4495
|
+
const db = new Database4(databasePath, { readonly: true });
|
|
1602
4496
|
try {
|
|
1603
4497
|
return schemaVersion(db);
|
|
1604
4498
|
} finally {
|
|
@@ -1612,27 +4506,27 @@ function schemaVersion(db) {
|
|
|
1612
4506
|
return row.version;
|
|
1613
4507
|
}
|
|
1614
4508
|
function assertInsideBackupRoot(databasePath, manifestPath) {
|
|
1615
|
-
const root =
|
|
1616
|
-
const candidate =
|
|
1617
|
-
if (
|
|
4509
|
+
const root = resolve3(`${databasePath}.backup`);
|
|
4510
|
+
const candidate = resolve3(manifestPath);
|
|
4511
|
+
if (dirname3(candidate) !== root)
|
|
1618
4512
|
throw new Error("manifest must be inside the database backup directory");
|
|
1619
4513
|
}
|
|
1620
4514
|
function backupEntries(databasePath) {
|
|
1621
|
-
const root =
|
|
1622
|
-
if (!
|
|
4515
|
+
const root = resolve3(`${databasePath}.backup`);
|
|
4516
|
+
if (!existsSync5(root))
|
|
1623
4517
|
return [];
|
|
1624
|
-
return
|
|
4518
|
+
return readdirSync2(root).filter((name) => name.endsWith(".manifest.json")).sort().map((name) => verifiedBackupEntry(root, resolve3(root, name)));
|
|
1625
4519
|
}
|
|
1626
4520
|
function verifiedBackupEntry(root, manifest) {
|
|
1627
|
-
const candidate =
|
|
1628
|
-
if (
|
|
4521
|
+
const candidate = resolve3(manifest);
|
|
4522
|
+
if (dirname3(candidate) !== root)
|
|
1629
4523
|
throw new Error("backup manifest escaped the backup directory");
|
|
1630
|
-
const stat =
|
|
4524
|
+
const stat = lstatSync4(candidate);
|
|
1631
4525
|
if (!stat.isFile() || stat.isSymbolicLink())
|
|
1632
4526
|
throw new Error("backup manifest must be a regular file");
|
|
1633
|
-
const bytes =
|
|
4527
|
+
const bytes = readFileSync4(candidate);
|
|
1634
4528
|
const verified = verifyBackupManifest(candidate);
|
|
1635
|
-
if (
|
|
4529
|
+
if (dirname3(verified.databasePath) !== root) {
|
|
1636
4530
|
throw new Error("backup database escaped the backup directory");
|
|
1637
4531
|
}
|
|
1638
4532
|
return {
|
|
@@ -1640,9 +4534,20 @@ function verifiedBackupEntry(root, manifest) {
|
|
|
1640
4534
|
database: verified.databasePath,
|
|
1641
4535
|
sha256: verified.manifest.sha256,
|
|
1642
4536
|
size: verified.manifest.size,
|
|
1643
|
-
manifestHash:
|
|
4537
|
+
manifestHash: createHash7("sha256").update(bytes).digest("hex")
|
|
1644
4538
|
};
|
|
1645
4539
|
}
|
|
4540
|
+
function outboxOperationKey(backend, operation, projectID, noteID, revision, contentHash, generation) {
|
|
4541
|
+
return hashTuple("outbox-operation", 2, [
|
|
4542
|
+
backend,
|
|
4543
|
+
operation,
|
|
4544
|
+
projectID,
|
|
4545
|
+
noteID,
|
|
4546
|
+
revision,
|
|
4547
|
+
contentHash,
|
|
4548
|
+
generation
|
|
4549
|
+
]);
|
|
4550
|
+
}
|
|
1646
4551
|
if (import.meta.main) {
|
|
1647
4552
|
try {
|
|
1648
4553
|
const result = await runAdmin();
|