@vaur94/agz-memory 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +41 -25
- package/CHANGELOG.md +78 -0
- package/README.md +39 -16
- package/README.tr.md +40 -17
- package/dist/admin.js +3563 -618
- package/dist/core.js +4378 -914
- package/dist/server.js +4105 -864
- package/dist/types/capture/contract.d.ts +7 -5
- package/dist/types/capture/redact.d.ts +1 -0
- package/dist/types/context.d.ts +1 -1
- package/dist/types/core.d.ts +2 -2
- package/dist/types/db/backup.d.ts +2 -1
- package/dist/types/db/health.d.ts +2 -0
- package/dist/types/db/legacy-health.d.ts +2 -0
- package/dist/types/db/maintenance.d.ts +21 -0
- package/dist/types/db/migrations/v011.d.ts +3 -0
- package/dist/types/db/schema.d.ts +10 -1
- package/dist/types/db.d.ts +1 -0
- package/dist/types/hash.d.ts +4 -0
- package/dist/types/retrieval/contract.d.ts +15 -3
- package/dist/types/server.d.ts +1 -1
- package/dist/types/store/capture.d.ts +12 -4
- package/dist/types/store/outbox.d.ts +6 -3
- package/dist/types/store/retrieval.d.ts +2 -2
- package/dist/types/store.d.ts +12 -3
- package/dist/types/types.d.ts +1 -1
- package/dist/types/version.d.ts +1 -1
- package/docs/adr/hash-identity-v2.md +57 -0
- package/docs/adr/maintenance-lock.md +93 -0
- package/docs/backup-restore-runbook.md +36 -16
- package/docs/backup-restore-runbook.tr.md +36 -16
- package/docs/repository-hardening.md +125 -0
- package/docs/review-resolution.md +78 -0
- package/docs/schema-v11.md +75 -0
- package/package.json +19 -4
- package/skills/agz-memory/agz-memory.md +39 -0
- package/skills/index.json +9 -0
package/dist/admin.js
CHANGED
|
@@ -2,16 +2,16 @@
|
|
|
2
2
|
// @bun
|
|
3
3
|
|
|
4
4
|
// src/admin/index.ts
|
|
5
|
-
import { createHash as
|
|
6
|
-
import { Database as
|
|
5
|
+
import { createHash as createHash7 } from "crypto";
|
|
6
|
+
import { Database as Database4 } from "bun:sqlite";
|
|
7
7
|
import {
|
|
8
|
-
existsSync as
|
|
9
|
-
lstatSync as
|
|
10
|
-
readdirSync,
|
|
11
|
-
readFileSync as
|
|
8
|
+
existsSync as existsSync5,
|
|
9
|
+
lstatSync as lstatSync4,
|
|
10
|
+
readdirSync as readdirSync2,
|
|
11
|
+
readFileSync as readFileSync4,
|
|
12
12
|
rmSync as rmSync3
|
|
13
13
|
} from "fs";
|
|
14
|
-
import { basename as
|
|
14
|
+
import { basename as basename3, dirname as dirname3, resolve as resolve3 } from "path";
|
|
15
15
|
|
|
16
16
|
// src/config.ts
|
|
17
17
|
import { homedir } from "os";
|
|
@@ -22,111 +22,2736 @@ function resolveConfig(environment = process.env) {
|
|
|
22
22
|
}
|
|
23
23
|
|
|
24
24
|
// src/db.ts
|
|
25
|
-
import { randomUUID as
|
|
26
|
-
import { Database as
|
|
27
|
-
import { chmodSync as
|
|
25
|
+
import { randomUUID as randomUUID6 } from "crypto";
|
|
26
|
+
import { Database as Database3 } from "bun:sqlite";
|
|
27
|
+
import { chmodSync as chmodSync3, copyFileSync as copyFileSync2, existsSync as existsSync4, lstatSync as lstatSync3 } from "fs";
|
|
28
28
|
|
|
29
29
|
// src/identity.ts
|
|
30
30
|
import { createHash } from "crypto";
|
|
31
31
|
function hashRoot(directory) {
|
|
32
32
|
return createHash("sha256").update(directory).digest("hex");
|
|
33
33
|
}
|
|
34
|
-
|
|
35
|
-
// src/project.ts
|
|
36
|
-
function cleanProjectName(value) {
|
|
37
|
-
return value.trim().replace(/\s+/g, " ");
|
|
34
|
+
|
|
35
|
+
// src/project.ts
|
|
36
|
+
function cleanProjectName(value) {
|
|
37
|
+
return value.trim().replace(/\s+/g, " ");
|
|
38
|
+
}
|
|
39
|
+
function normalizeProjectName(value) {
|
|
40
|
+
return cleanProjectName(value).normalize("NFKC").toLowerCase();
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// src/types.ts
|
|
44
|
+
var SCHEMA_VERSION = 11;
|
|
45
|
+
var KINDS = ["decision", "fact", "procedure", "context", "research", "preference", "task"];
|
|
46
|
+
var PREDICATES = ["SUPPORTS", "DERIVED_FROM", "PART_OF", "ABOUT", "PRECEDES", "SUPERSEDES"];
|
|
47
|
+
|
|
48
|
+
// src/db/backup.ts
|
|
49
|
+
import { createHash as createHash4, randomUUID as randomUUID3 } from "crypto";
|
|
50
|
+
import { Database as Database2 } from "bun:sqlite";
|
|
51
|
+
import {
|
|
52
|
+
chmodSync as chmodSync2,
|
|
53
|
+
closeSync as closeSync2,
|
|
54
|
+
constants as constants2,
|
|
55
|
+
existsSync as existsSync2,
|
|
56
|
+
fstatSync,
|
|
57
|
+
fsyncSync as fsyncSync2,
|
|
58
|
+
lstatSync as lstatSync2,
|
|
59
|
+
mkdirSync as mkdirSync2,
|
|
60
|
+
openSync as openSync2,
|
|
61
|
+
readFileSync as readFileSync2,
|
|
62
|
+
readSync,
|
|
63
|
+
renameSync as renameSync2,
|
|
64
|
+
rmSync,
|
|
65
|
+
writeFileSync as writeFileSync2,
|
|
66
|
+
writeSync
|
|
67
|
+
} from "fs";
|
|
68
|
+
import { basename as basename2, dirname as dirname2, join as join3, resolve as resolve2 } from "path";
|
|
69
|
+
import { platform as platform2 } from "os";
|
|
70
|
+
|
|
71
|
+
// src/db/schema.ts
|
|
72
|
+
import { Database } from "bun:sqlite";
|
|
73
|
+
import { randomUUID } from "crypto";
|
|
74
|
+
|
|
75
|
+
// src/capture/contract.ts
|
|
76
|
+
import * as z from "zod/v4";
|
|
77
|
+
var CAPTURE_SCHEMA = "agz-memory.capture/2";
|
|
78
|
+
var SUPPORTED_OPENCODE_VERSION = "0.0.0-beta-18743";
|
|
79
|
+
var CAPTURE_EVENT_MAX_BYTES = 16 * 1024;
|
|
80
|
+
var CAPTURE_CONTENT_MAX_CHARACTERS = 4800;
|
|
81
|
+
var CAPTURE_SUMMARY_MAX_CHARACTERS = 1200;
|
|
82
|
+
var CAPTURE_SUBJECT_MAX_CHARACTERS = 240;
|
|
83
|
+
var candidateSchema = z.object({
|
|
84
|
+
kind: z.enum(KINDS),
|
|
85
|
+
title: z.string().min(1).max(240),
|
|
86
|
+
summary: z.string().min(1).max(CAPTURE_SUMMARY_MAX_CHARACTERS),
|
|
87
|
+
content: z.string().min(1).max(CAPTURE_CONTENT_MAX_CHARACTERS),
|
|
88
|
+
subjectKey: z.string().min(1).max(CAPTURE_SUBJECT_MAX_CHARACTERS).optional(),
|
|
89
|
+
intent: z.enum(["create", "supersede", "ignore", "review"]),
|
|
90
|
+
targetNoteID: z.string().min(1).max(240).optional(),
|
|
91
|
+
confidence: z.number().finite().min(0).max(1),
|
|
92
|
+
evidence: z.enum(["explicit-user", "verified-outcome", "session-summary"])
|
|
93
|
+
}).strict();
|
|
94
|
+
var signalSchema = z.object({
|
|
95
|
+
tool: z.string().min(1).max(160),
|
|
96
|
+
status: z.enum(["completed", "error"]),
|
|
97
|
+
errorType: z.string().min(1).max(160).optional()
|
|
98
|
+
}).strict();
|
|
99
|
+
var captureEventSchema = z.object({
|
|
100
|
+
schema: z.literal(CAPTURE_SCHEMA),
|
|
101
|
+
idempotencyKey: z.string().regex(/^[0-9a-f]{64}$/),
|
|
102
|
+
projectID: z.uuid(),
|
|
103
|
+
bindingKey: z.string().regex(/^[0-9a-f]{64}$/),
|
|
104
|
+
kind: z.enum(["user-candidate", "assistant-candidate", "session-summary", "tool-signal"]),
|
|
105
|
+
source: z.object({
|
|
106
|
+
system: z.literal("opencode-v2"),
|
|
107
|
+
opencodeVersion: z.literal(SUPPORTED_OPENCODE_VERSION),
|
|
108
|
+
pluginVersion: z.string().min(1).max(80),
|
|
109
|
+
sessionID: z.string().min(1).max(240),
|
|
110
|
+
messageID: z.string().min(1).max(240).optional(),
|
|
111
|
+
ordinal: z.number().finite().int().nonnegative().refine(Number.isSafeInteger, "must be a safe integer").optional(),
|
|
112
|
+
toolCallID: z.string().min(1).max(240).optional(),
|
|
113
|
+
observedAt: z.number().finite().int().nonnegative().refine(Number.isSafeInteger, "must be a safe integer")
|
|
114
|
+
}).strict(),
|
|
115
|
+
candidate: candidateSchema.optional(),
|
|
116
|
+
signal: signalSchema.optional(),
|
|
117
|
+
redaction: z.object({
|
|
118
|
+
policyVersion: z.string().min(1).max(80),
|
|
119
|
+
replacements: z.number().int().nonnegative(),
|
|
120
|
+
truncated: z.boolean()
|
|
121
|
+
}).strict()
|
|
122
|
+
}).strict().superRefine((event, context) => {
|
|
123
|
+
if (event.kind === "tool-signal" && !event.signal) {
|
|
124
|
+
context.addIssue({ code: "custom", message: "tool-signal requires signal" });
|
|
125
|
+
}
|
|
126
|
+
if (event.kind !== "tool-signal" && !event.candidate) {
|
|
127
|
+
context.addIssue({ code: "custom", message: `${event.kind} requires candidate` });
|
|
128
|
+
}
|
|
129
|
+
if (event.kind === "tool-signal" && event.candidate) {
|
|
130
|
+
context.addIssue({ code: "custom", message: "tool-signal cannot carry candidate" });
|
|
131
|
+
}
|
|
132
|
+
if (event.kind !== "tool-signal" && event.signal) {
|
|
133
|
+
context.addIssue({ code: "custom", message: `${event.kind} cannot carry signal` });
|
|
134
|
+
}
|
|
135
|
+
const hasSourceField = (field) => Object.prototype.hasOwnProperty.call(event.source, field);
|
|
136
|
+
const requiredFields = event.kind === "user-candidate" ? ["messageID"] : event.kind === "assistant-candidate" ? ["messageID", "ordinal"] : event.kind === "session-summary" ? ["messageID"] : ["messageID", "toolCallID"];
|
|
137
|
+
const forbiddenFields = event.kind === "assistant-candidate" ? ["toolCallID"] : event.kind === "tool-signal" ? ["ordinal"] : ["ordinal", "toolCallID"];
|
|
138
|
+
for (const field of requiredFields) {
|
|
139
|
+
if (event.source[field] === undefined) {
|
|
140
|
+
context.addIssue({ code: "custom", message: `${event.kind} requires source.${field}` });
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
for (const field of forbiddenFields) {
|
|
144
|
+
if (hasSourceField(field)) {
|
|
145
|
+
context.addIssue({ code: "custom", message: `${event.kind} forbids source.${field}` });
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
});
|
|
149
|
+
function parseCaptureEvent(value) {
|
|
150
|
+
const event = captureEventSchema.parse(value);
|
|
151
|
+
if (Buffer.byteLength(JSON.stringify(event), "utf8") > CAPTURE_EVENT_MAX_BYTES) {
|
|
152
|
+
throw new Error(`capture event exceeds ${CAPTURE_EVENT_MAX_BYTES} UTF-8 bytes`);
|
|
153
|
+
}
|
|
154
|
+
return event;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// src/hash.ts
|
|
158
|
+
import { createHash as createHash2 } from "crypto";
|
|
159
|
+
var HASH_TUPLE_MARKER = Buffer.from("agz-memory/hash-tuple", "utf8");
|
|
160
|
+
var MAX_UINT32 = 4294967295;
|
|
161
|
+
function hashTuple(domain, version, fields) {
|
|
162
|
+
if (typeof domain !== "string")
|
|
163
|
+
throw new TypeError("hash tuple domain must be a string");
|
|
164
|
+
if (!Number.isSafeInteger(version) || version < 0 || version > MAX_UINT32) {
|
|
165
|
+
throw new RangeError("hash tuple version must be an unsigned safe integer");
|
|
166
|
+
}
|
|
167
|
+
if (!Array.isArray(fields))
|
|
168
|
+
throw new TypeError("hash tuple fields must be an array");
|
|
169
|
+
if (fields.length > MAX_UINT32)
|
|
170
|
+
throw new RangeError("hash tuple has too many fields");
|
|
171
|
+
const domainBytes = utf8(domain);
|
|
172
|
+
const chunks = [
|
|
173
|
+
HASH_TUPLE_MARKER,
|
|
174
|
+
lengthPrefix(domainBytes),
|
|
175
|
+
uint32(version),
|
|
176
|
+
uint32(fields.length)
|
|
177
|
+
];
|
|
178
|
+
for (const field of fields) {
|
|
179
|
+
const encoded = encodeField(field);
|
|
180
|
+
chunks.push(Buffer.from([encoded.tag]), lengthPrefix(encoded.payload));
|
|
181
|
+
}
|
|
182
|
+
return createHash2("sha256").update(Buffer.concat(chunks)).digest("hex");
|
|
183
|
+
}
|
|
184
|
+
function noteContentHash(kind, title, summary, content) {
|
|
185
|
+
return hashTuple("canonical-note", 2, [kind, title, summary, content]);
|
|
186
|
+
}
|
|
187
|
+
function encodeField(value) {
|
|
188
|
+
if (value === null)
|
|
189
|
+
return { tag: 0, payload: Buffer.alloc(0) };
|
|
190
|
+
if (typeof value === "string")
|
|
191
|
+
return { tag: 1, payload: utf8(value) };
|
|
192
|
+
if (typeof value === "boolean")
|
|
193
|
+
return { tag: 2, payload: Buffer.from([value ? 1 : 0]) };
|
|
194
|
+
if (typeof value === "number") {
|
|
195
|
+
if (!Number.isFinite(value))
|
|
196
|
+
throw new RangeError("hash tuple numbers must be finite");
|
|
197
|
+
if (Number.isInteger(value) && !Number.isSafeInteger(value)) {
|
|
198
|
+
throw new RangeError("hash tuple integer numbers must be safe");
|
|
199
|
+
}
|
|
200
|
+
return { tag: 3, payload: utf8(String(value)) };
|
|
201
|
+
}
|
|
202
|
+
if (value instanceof Uint8Array)
|
|
203
|
+
return { tag: 4, payload: Buffer.from(value) };
|
|
204
|
+
throw new TypeError("hash tuple field has an unsupported type");
|
|
205
|
+
}
|
|
206
|
+
function utf8(value) {
|
|
207
|
+
if (!isWellFormedUnicode(value)) {
|
|
208
|
+
throw new TypeError("hash tuple strings must be well-formed Unicode");
|
|
209
|
+
}
|
|
210
|
+
const bytes = Buffer.from(value, "utf8");
|
|
211
|
+
if (bytes.length > MAX_UINT32)
|
|
212
|
+
throw new RangeError("hash tuple field is too large");
|
|
213
|
+
return bytes;
|
|
214
|
+
}
|
|
215
|
+
function isWellFormedUnicode(value) {
|
|
216
|
+
for (let index = 0;index < value.length; index++) {
|
|
217
|
+
const code = value.charCodeAt(index);
|
|
218
|
+
if (code >= 55296 && code <= 56319) {
|
|
219
|
+
if (index + 1 >= value.length)
|
|
220
|
+
return false;
|
|
221
|
+
const next = value.charCodeAt(index + 1);
|
|
222
|
+
if (next < 56320 || next > 57343)
|
|
223
|
+
return false;
|
|
224
|
+
index++;
|
|
225
|
+
} else if (code >= 56320 && code <= 57343) {
|
|
226
|
+
return false;
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
return true;
|
|
230
|
+
}
|
|
231
|
+
function lengthPrefix(bytes) {
|
|
232
|
+
return Buffer.concat([uint32(bytes.length), Buffer.from(bytes)]);
|
|
233
|
+
}
|
|
234
|
+
function uint32(value) {
|
|
235
|
+
const result = Buffer.alloc(4);
|
|
236
|
+
result.writeUInt32BE(value, 0);
|
|
237
|
+
return result;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
// src/db/schema.ts
|
|
241
|
+
var APPLICATION_ID = 1095195213;
|
|
242
|
+
var PRODUCT_ID = "agz-memory";
|
|
243
|
+
var HASH_POLICY = "hash-tuple/2";
|
|
244
|
+
var SQLITE_MANAGED_FTS_TABLES = new Set([
|
|
245
|
+
"notes_fts_config",
|
|
246
|
+
"notes_fts_data",
|
|
247
|
+
"notes_fts_docsize",
|
|
248
|
+
"notes_fts_idx"
|
|
249
|
+
]);
|
|
250
|
+
var SCHEMA_TABLES = `
|
|
251
|
+
CREATE TABLE IF NOT EXISTS projects (
|
|
252
|
+
id TEXT PRIMARY KEY,
|
|
253
|
+
name TEXT NOT NULL,
|
|
254
|
+
normalized_name TEXT NOT NULL UNIQUE,
|
|
255
|
+
created_at INTEGER NOT NULL,
|
|
256
|
+
updated_at INTEGER NOT NULL
|
|
257
|
+
);
|
|
258
|
+
CREATE TABLE IF NOT EXISTS notes (
|
|
259
|
+
id TEXT PRIMARY KEY,
|
|
260
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
261
|
+
kind TEXT NOT NULL CHECK (kind IN ('decision','fact','procedure','context','research','preference','task')),
|
|
262
|
+
title TEXT NOT NULL,
|
|
263
|
+
summary TEXT NOT NULL,
|
|
264
|
+
content TEXT NOT NULL,
|
|
265
|
+
size_class TEXT NOT NULL CHECK (size_class IN ('inline','indexed')),
|
|
266
|
+
pinned INTEGER NOT NULL DEFAULT 0 CHECK (pinned IN (0,1)),
|
|
267
|
+
status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('active','superseded','archived')),
|
|
268
|
+
supersedes_id TEXT,
|
|
269
|
+
current_revision INTEGER NOT NULL DEFAULT 1 CHECK (current_revision >= 1),
|
|
270
|
+
subject_key TEXT,
|
|
271
|
+
content_hash TEXT NOT NULL CHECK (length(content_hash) = 64),
|
|
272
|
+
created_at INTEGER NOT NULL,
|
|
273
|
+
updated_at INTEGER NOT NULL,
|
|
274
|
+
UNIQUE(project_id, id)
|
|
275
|
+
);
|
|
276
|
+
CREATE INDEX IF NOT EXISTS notes_project_idx ON notes(project_id, status);
|
|
277
|
+
CREATE UNIQUE INDEX IF NOT EXISTS notes_active_subject_idx
|
|
278
|
+
ON notes(project_id, kind, subject_key)
|
|
279
|
+
WHERE status = 'active' AND subject_key IS NOT NULL;
|
|
280
|
+
CREATE TABLE IF NOT EXISTS note_edges (
|
|
281
|
+
id TEXT PRIMARY KEY,
|
|
282
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
283
|
+
source_id TEXT NOT NULL,
|
|
284
|
+
target_id TEXT NOT NULL,
|
|
285
|
+
predicate TEXT NOT NULL CHECK (predicate IN ('SUPPORTS','DERIVED_FROM','PART_OF','ABOUT','PRECEDES','SUPERSEDES')),
|
|
286
|
+
created_at INTEGER NOT NULL,
|
|
287
|
+
UNIQUE(project_id, source_id, target_id, predicate),
|
|
288
|
+
FOREIGN KEY (project_id, source_id) REFERENCES notes(project_id, id) ON DELETE CASCADE,
|
|
289
|
+
FOREIGN KEY (project_id, target_id) REFERENCES notes(project_id, id) ON DELETE CASCADE
|
|
290
|
+
);
|
|
291
|
+
CREATE INDEX IF NOT EXISTS note_edges_source_idx ON note_edges(project_id, source_id);
|
|
292
|
+
CREATE INDEX IF NOT EXISTS note_edges_target_idx ON note_edges(project_id, target_id);
|
|
293
|
+
CREATE TABLE IF NOT EXISTS project_bindings (
|
|
294
|
+
binding_key TEXT PRIMARY KEY,
|
|
295
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
296
|
+
source TEXT NOT NULL CHECK (source = 'opencode-v2'),
|
|
297
|
+
source_project_id TEXT NOT NULL,
|
|
298
|
+
workspace_id TEXT NOT NULL,
|
|
299
|
+
canonical_path_hash TEXT NOT NULL CHECK (length(canonical_path_hash) = 64),
|
|
300
|
+
created_at INTEGER NOT NULL,
|
|
301
|
+
updated_at INTEGER NOT NULL,
|
|
302
|
+
UNIQUE(source, source_project_id, workspace_id)
|
|
303
|
+
);
|
|
304
|
+
CREATE TABLE IF NOT EXISTS capture_checkpoints (
|
|
305
|
+
session_id TEXT PRIMARY KEY,
|
|
306
|
+
binding_key TEXT NOT NULL REFERENCES project_bindings(binding_key) ON DELETE CASCADE,
|
|
307
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
308
|
+
state TEXT NOT NULL CHECK (state IN ('active','idle','unavailable','closed')),
|
|
309
|
+
last_message_id TEXT,
|
|
310
|
+
last_reconciled_at INTEGER,
|
|
311
|
+
next_reconcile_at INTEGER NOT NULL,
|
|
312
|
+
failure_count INTEGER NOT NULL DEFAULT 0 CHECK (failure_count >= 0),
|
|
313
|
+
lease_owner TEXT,
|
|
314
|
+
lease_expires_at INTEGER,
|
|
315
|
+
created_at INTEGER NOT NULL,
|
|
316
|
+
updated_at INTEGER NOT NULL
|
|
317
|
+
);
|
|
318
|
+
CREATE INDEX IF NOT EXISTS capture_checkpoints_due_idx
|
|
319
|
+
ON capture_checkpoints(state, next_reconcile_at);
|
|
320
|
+
${captureEventsTable()}
|
|
321
|
+
CREATE INDEX IF NOT EXISTS capture_events_state_idx ON capture_events(state, updated_at);
|
|
322
|
+
CREATE INDEX IF NOT EXISTS capture_events_session_idx ON capture_events(project_id, source_session_id);
|
|
323
|
+
CREATE INDEX IF NOT EXISTS capture_events_note_idx ON capture_events(project_id, note_id);
|
|
324
|
+
CREATE TABLE IF NOT EXISTS note_provenance (
|
|
325
|
+
id TEXT PRIMARY KEY,
|
|
326
|
+
project_id TEXT NOT NULL,
|
|
327
|
+
note_id TEXT NOT NULL,
|
|
328
|
+
source_type TEXT NOT NULL CHECK (source_type IN ('mcp-manual','opencode-capture','migration','legacy-import','admin')),
|
|
329
|
+
capture_event_id TEXT,
|
|
330
|
+
source_session_id TEXT,
|
|
331
|
+
source_message_id TEXT,
|
|
332
|
+
source_ordinal INTEGER,
|
|
333
|
+
source_tool_call_id TEXT,
|
|
334
|
+
redaction_version TEXT,
|
|
335
|
+
extractor_version TEXT,
|
|
336
|
+
confidence REAL CHECK (confidence IS NULL OR (confidence >= 0 AND confidence <= 1)),
|
|
337
|
+
created_at INTEGER NOT NULL,
|
|
338
|
+
UNIQUE(project_id, id),
|
|
339
|
+
FOREIGN KEY (project_id, note_id) REFERENCES notes(project_id, id) ON DELETE CASCADE
|
|
340
|
+
);
|
|
341
|
+
CREATE TABLE IF NOT EXISTS note_revisions (
|
|
342
|
+
project_id TEXT NOT NULL,
|
|
343
|
+
note_id TEXT NOT NULL,
|
|
344
|
+
revision INTEGER NOT NULL CHECK (revision >= 1),
|
|
345
|
+
kind TEXT NOT NULL,
|
|
346
|
+
title TEXT NOT NULL,
|
|
347
|
+
summary TEXT NOT NULL,
|
|
348
|
+
content TEXT NOT NULL,
|
|
349
|
+
size_class TEXT NOT NULL,
|
|
350
|
+
pinned INTEGER NOT NULL CHECK (pinned IN (0,1)),
|
|
351
|
+
status TEXT NOT NULL CHECK (status IN ('active','superseded','archived')),
|
|
352
|
+
supersedes_id TEXT,
|
|
353
|
+
subject_key TEXT,
|
|
354
|
+
content_hash TEXT NOT NULL,
|
|
355
|
+
provenance_id TEXT NOT NULL,
|
|
356
|
+
created_at INTEGER NOT NULL,
|
|
357
|
+
PRIMARY KEY(project_id, note_id, revision),
|
|
358
|
+
FOREIGN KEY (project_id, note_id) REFERENCES notes(project_id, id) ON DELETE CASCADE,
|
|
359
|
+
FOREIGN KEY (project_id, provenance_id) REFERENCES note_provenance(project_id, id)
|
|
360
|
+
);
|
|
361
|
+
CREATE TABLE IF NOT EXISTS index_outbox (
|
|
362
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
363
|
+
backend TEXT NOT NULL,
|
|
364
|
+
operation TEXT NOT NULL CHECK (operation IN ('upsert-note','delete-note','purge-project')),
|
|
365
|
+
project_id TEXT NOT NULL,
|
|
366
|
+
note_id TEXT,
|
|
367
|
+
revision INTEGER,
|
|
368
|
+
content_hash TEXT,
|
|
369
|
+
state TEXT NOT NULL DEFAULT 'pending' CHECK (state IN ('pending','leased','succeeded','dead')),
|
|
370
|
+
attempt_count INTEGER NOT NULL DEFAULT 0 CHECK (attempt_count >= 0),
|
|
371
|
+
available_at INTEGER NOT NULL,
|
|
372
|
+
lease_owner TEXT,
|
|
373
|
+
lease_expires_at INTEGER,
|
|
374
|
+
last_error_code TEXT,
|
|
375
|
+
created_at INTEGER NOT NULL,
|
|
376
|
+
completed_at INTEGER,
|
|
377
|
+
UNIQUE(backend, operation, project_id, note_id, revision)
|
|
378
|
+
);
|
|
379
|
+
CREATE INDEX IF NOT EXISTS index_outbox_due_idx
|
|
380
|
+
ON index_outbox(backend, project_id, state, available_at, id);
|
|
381
|
+
CREATE TABLE IF NOT EXISTS schema_state (version INTEGER PRIMARY KEY);
|
|
382
|
+
`;
|
|
383
|
+
var FTS_V9 = `
|
|
384
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS notes_fts USING fts5(
|
|
385
|
+
title, summary, content,
|
|
386
|
+
content='notes', content_rowid='rowid',
|
|
387
|
+
tokenize='unicode61'
|
|
388
|
+
);
|
|
389
|
+
CREATE TRIGGER IF NOT EXISTS notes_fts_ai AFTER INSERT ON notes BEGIN
|
|
390
|
+
INSERT INTO notes_fts(rowid, title, summary, content)
|
|
391
|
+
VALUES (new.rowid, new.title, new.summary, new.content);
|
|
392
|
+
END;
|
|
393
|
+
CREATE TRIGGER IF NOT EXISTS notes_fts_ad AFTER DELETE ON notes BEGIN
|
|
394
|
+
INSERT INTO notes_fts(notes_fts, rowid, title, summary, content)
|
|
395
|
+
VALUES ('delete', old.rowid, old.title, old.summary, old.content);
|
|
396
|
+
END;
|
|
397
|
+
CREATE TRIGGER IF NOT EXISTS notes_fts_au AFTER UPDATE OF title, summary, content ON notes BEGIN
|
|
398
|
+
INSERT INTO notes_fts(notes_fts, rowid, title, summary, content)
|
|
399
|
+
VALUES ('delete', old.rowid, old.title, old.summary, old.content);
|
|
400
|
+
INSERT INTO notes_fts(rowid, title, summary, content)
|
|
401
|
+
VALUES (new.rowid, new.title, new.summary, new.content);
|
|
402
|
+
END;
|
|
403
|
+
`;
|
|
404
|
+
var SCHEMA_V11_TABLES = `
|
|
405
|
+
CREATE TABLE projects (
|
|
406
|
+
id TEXT PRIMARY KEY,
|
|
407
|
+
name TEXT NOT NULL,
|
|
408
|
+
normalized_name TEXT NOT NULL UNIQUE,
|
|
409
|
+
created_at INTEGER NOT NULL,
|
|
410
|
+
updated_at INTEGER NOT NULL
|
|
411
|
+
);
|
|
412
|
+
CREATE TABLE notes (
|
|
413
|
+
id TEXT PRIMARY KEY,
|
|
414
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
415
|
+
kind TEXT NOT NULL CHECK (kind IN ('decision','fact','procedure','context','research','preference','task')),
|
|
416
|
+
title TEXT NOT NULL,
|
|
417
|
+
summary TEXT NOT NULL,
|
|
418
|
+
content TEXT NOT NULL,
|
|
419
|
+
size_class TEXT NOT NULL CHECK (size_class IN ('inline','indexed')),
|
|
420
|
+
pinned INTEGER NOT NULL DEFAULT 0 CHECK (pinned IN (0,1)),
|
|
421
|
+
status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('active','superseded','archived')),
|
|
422
|
+
supersedes_id TEXT,
|
|
423
|
+
current_revision INTEGER NOT NULL DEFAULT 1 CHECK (current_revision >= 1),
|
|
424
|
+
subject_key TEXT,
|
|
425
|
+
content_hash TEXT NOT NULL CHECK (length(content_hash) = 64),
|
|
426
|
+
created_at INTEGER NOT NULL,
|
|
427
|
+
updated_at INTEGER NOT NULL,
|
|
428
|
+
UNIQUE(project_id, id),
|
|
429
|
+
FOREIGN KEY (project_id, supersedes_id) REFERENCES notes(project_id, id) ON DELETE NO ACTION
|
|
430
|
+
);
|
|
431
|
+
CREATE INDEX notes_project_idx ON notes(project_id, status);
|
|
432
|
+
CREATE UNIQUE INDEX notes_active_subject_idx
|
|
433
|
+
ON notes(project_id, kind, subject_key)
|
|
434
|
+
WHERE status = 'active' AND subject_key IS NOT NULL;
|
|
435
|
+
CREATE TABLE note_edges (
|
|
436
|
+
id TEXT PRIMARY KEY,
|
|
437
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
438
|
+
source_id TEXT NOT NULL,
|
|
439
|
+
target_id TEXT NOT NULL,
|
|
440
|
+
predicate TEXT NOT NULL CHECK (predicate IN ('SUPPORTS','DERIVED_FROM','PART_OF','ABOUT','PRECEDES','SUPERSEDES')),
|
|
441
|
+
created_at INTEGER NOT NULL,
|
|
442
|
+
UNIQUE(project_id, source_id, target_id, predicate),
|
|
443
|
+
FOREIGN KEY (project_id, source_id) REFERENCES notes(project_id, id) ON DELETE CASCADE,
|
|
444
|
+
FOREIGN KEY (project_id, target_id) REFERENCES notes(project_id, id) ON DELETE CASCADE
|
|
445
|
+
);
|
|
446
|
+
CREATE INDEX note_edges_source_idx ON note_edges(project_id, source_id);
|
|
447
|
+
CREATE INDEX note_edges_target_idx ON note_edges(project_id, target_id);
|
|
448
|
+
CREATE TABLE project_bindings (
|
|
449
|
+
binding_key TEXT NOT NULL CHECK (length(binding_key) = 64),
|
|
450
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
451
|
+
source TEXT NOT NULL CHECK (source = 'opencode-v2'),
|
|
452
|
+
source_project_id TEXT NOT NULL,
|
|
453
|
+
workspace_id TEXT NOT NULL,
|
|
454
|
+
canonical_path_hash TEXT NOT NULL CHECK (length(canonical_path_hash) = 64),
|
|
455
|
+
created_at INTEGER NOT NULL,
|
|
456
|
+
updated_at INTEGER NOT NULL,
|
|
457
|
+
PRIMARY KEY(binding_key, project_id),
|
|
458
|
+
UNIQUE(source, source_project_id, workspace_id)
|
|
459
|
+
);
|
|
460
|
+
CREATE TABLE capture_checkpoints (
|
|
461
|
+
session_id TEXT NOT NULL,
|
|
462
|
+
binding_key TEXT NOT NULL,
|
|
463
|
+
project_id TEXT NOT NULL,
|
|
464
|
+
state TEXT NOT NULL CHECK (state IN ('active','idle','unavailable','closed')),
|
|
465
|
+
last_message_id TEXT,
|
|
466
|
+
last_reconciled_at INTEGER,
|
|
467
|
+
next_reconcile_at INTEGER NOT NULL,
|
|
468
|
+
failure_count INTEGER NOT NULL DEFAULT 0 CHECK (failure_count >= 0),
|
|
469
|
+
lease_owner TEXT,
|
|
470
|
+
lease_expires_at INTEGER,
|
|
471
|
+
created_at INTEGER NOT NULL,
|
|
472
|
+
updated_at INTEGER NOT NULL,
|
|
473
|
+
PRIMARY KEY(binding_key, session_id),
|
|
474
|
+
FOREIGN KEY (binding_key, project_id) REFERENCES project_bindings(binding_key, project_id) ON DELETE CASCADE
|
|
475
|
+
);
|
|
476
|
+
CREATE INDEX capture_checkpoints_due_idx
|
|
477
|
+
ON capture_checkpoints(state, next_reconcile_at);
|
|
478
|
+
${captureEventsTableV11()}
|
|
479
|
+
CREATE INDEX capture_events_state_idx ON capture_events(state, updated_at);
|
|
480
|
+
CREATE INDEX capture_events_session_idx ON capture_events(project_id, source_session_id);
|
|
481
|
+
CREATE INDEX capture_events_note_idx ON capture_events(project_id, note_id);
|
|
482
|
+
CREATE TABLE note_provenance (
|
|
483
|
+
id TEXT PRIMARY KEY,
|
|
484
|
+
project_id TEXT NOT NULL,
|
|
485
|
+
note_id TEXT NOT NULL,
|
|
486
|
+
source_type TEXT NOT NULL CHECK (source_type IN ('mcp-manual','opencode-capture','migration','legacy-import','admin')),
|
|
487
|
+
capture_event_id TEXT,
|
|
488
|
+
source_session_id TEXT,
|
|
489
|
+
source_message_id TEXT,
|
|
490
|
+
source_ordinal INTEGER,
|
|
491
|
+
source_tool_call_id TEXT,
|
|
492
|
+
redaction_version TEXT,
|
|
493
|
+
extractor_version TEXT,
|
|
494
|
+
confidence REAL CHECK (confidence IS NULL OR (confidence >= 0 AND confidence <= 1)),
|
|
495
|
+
created_at INTEGER NOT NULL,
|
|
496
|
+
UNIQUE(project_id, id),
|
|
497
|
+
FOREIGN KEY (project_id, note_id) REFERENCES notes(project_id, id) ON DELETE CASCADE
|
|
498
|
+
);
|
|
499
|
+
CREATE TABLE note_revisions (
|
|
500
|
+
project_id TEXT NOT NULL,
|
|
501
|
+
note_id TEXT NOT NULL,
|
|
502
|
+
revision INTEGER NOT NULL CHECK (revision >= 1),
|
|
503
|
+
kind TEXT NOT NULL CHECK (kind IN ('decision','fact','procedure','context','research','preference','task')),
|
|
504
|
+
title TEXT NOT NULL,
|
|
505
|
+
summary TEXT NOT NULL,
|
|
506
|
+
content TEXT NOT NULL,
|
|
507
|
+
size_class TEXT NOT NULL CHECK (size_class IN ('inline','indexed')),
|
|
508
|
+
pinned INTEGER NOT NULL CHECK (pinned IN (0,1)),
|
|
509
|
+
status TEXT NOT NULL CHECK (status IN ('active','superseded','archived')),
|
|
510
|
+
supersedes_id TEXT,
|
|
511
|
+
subject_key TEXT,
|
|
512
|
+
content_hash TEXT NOT NULL CHECK (length(content_hash) = 64),
|
|
513
|
+
provenance_id TEXT NOT NULL,
|
|
514
|
+
created_at INTEGER NOT NULL,
|
|
515
|
+
PRIMARY KEY(project_id, note_id, revision),
|
|
516
|
+
FOREIGN KEY (project_id, note_id) REFERENCES notes(project_id, id) ON DELETE CASCADE,
|
|
517
|
+
FOREIGN KEY (project_id, provenance_id) REFERENCES note_provenance(project_id, id),
|
|
518
|
+
FOREIGN KEY (project_id, supersedes_id) REFERENCES notes(project_id, id) ON DELETE NO ACTION
|
|
519
|
+
);
|
|
520
|
+
CREATE TABLE index_outbox (
|
|
521
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
522
|
+
backend TEXT NOT NULL,
|
|
523
|
+
operation_key TEXT NOT NULL CHECK (length(operation_key) = 64),
|
|
524
|
+
operation TEXT NOT NULL CHECK (operation IN ('upsert-note','delete-note','purge-project')),
|
|
525
|
+
project_id TEXT NOT NULL,
|
|
526
|
+
note_id TEXT,
|
|
527
|
+
revision INTEGER,
|
|
528
|
+
content_hash TEXT,
|
|
529
|
+
generation INTEGER NOT NULL DEFAULT 0 CHECK (generation >= 0),
|
|
530
|
+
lease_generation INTEGER NOT NULL DEFAULT 0 CHECK (lease_generation >= 0),
|
|
531
|
+
fence INTEGER NOT NULL DEFAULT 0 CHECK (fence >= 0),
|
|
532
|
+
state TEXT NOT NULL DEFAULT 'pending' CHECK (state IN ('pending','leased','succeeded','dead')),
|
|
533
|
+
attempt_count INTEGER NOT NULL DEFAULT 0 CHECK (attempt_count >= 0),
|
|
534
|
+
available_at INTEGER NOT NULL,
|
|
535
|
+
lease_owner TEXT,
|
|
536
|
+
lease_expires_at INTEGER,
|
|
537
|
+
heartbeat_at INTEGER CHECK (heartbeat_at IS NULL OR heartbeat_at >= 0),
|
|
538
|
+
last_error_code TEXT,
|
|
539
|
+
created_at INTEGER NOT NULL,
|
|
540
|
+
completed_at INTEGER,
|
|
541
|
+
CHECK (
|
|
542
|
+
(operation = 'upsert-note' AND note_id IS NOT NULL AND revision IS NOT NULL AND revision >= 1
|
|
543
|
+
AND content_hash IS NOT NULL AND length(content_hash) = 64)
|
|
544
|
+
OR (operation = 'delete-note' AND note_id IS NOT NULL AND revision IS NOT NULL AND revision >= 1
|
|
545
|
+
AND content_hash IS NULL)
|
|
546
|
+
OR (operation = 'purge-project' AND note_id IS NULL AND revision IS NULL AND content_hash IS NULL)
|
|
547
|
+
)
|
|
548
|
+
);
|
|
549
|
+
CREATE TRIGGER index_outbox_note_project_insert
|
|
550
|
+
BEFORE INSERT ON index_outbox
|
|
551
|
+
WHEN NEW.note_id IS NOT NULL
|
|
552
|
+
AND NEW.state IN ('pending','leased')
|
|
553
|
+
AND (NEW.operation = 'upsert-note'
|
|
554
|
+
OR EXISTS (SELECT 1 FROM notes WHERE id = NEW.note_id))
|
|
555
|
+
AND NOT EXISTS (
|
|
556
|
+
SELECT 1 FROM notes WHERE project_id = NEW.project_id AND id = NEW.note_id
|
|
557
|
+
)
|
|
558
|
+
BEGIN
|
|
559
|
+
SELECT RAISE(ABORT, 'foreign key index_outbox note project mismatch');
|
|
560
|
+
END;
|
|
561
|
+
CREATE TRIGGER index_outbox_note_project_update
|
|
562
|
+
BEFORE UPDATE OF project_id, note_id, operation ON index_outbox
|
|
563
|
+
WHEN NEW.note_id IS NOT NULL
|
|
564
|
+
AND NEW.state IN ('pending','leased')
|
|
565
|
+
AND (NEW.operation = 'upsert-note'
|
|
566
|
+
OR EXISTS (SELECT 1 FROM notes WHERE id = NEW.note_id))
|
|
567
|
+
AND NOT EXISTS (
|
|
568
|
+
SELECT 1 FROM notes WHERE project_id = NEW.project_id AND id = NEW.note_id
|
|
569
|
+
)
|
|
570
|
+
BEGIN
|
|
571
|
+
SELECT RAISE(ABORT, 'foreign key index_outbox note project mismatch');
|
|
572
|
+
END;
|
|
573
|
+
CREATE INDEX index_outbox_due_idx
|
|
574
|
+
ON index_outbox(backend, project_id, state, available_at, id);
|
|
575
|
+
CREATE UNIQUE INDEX index_outbox_active_operation_idx
|
|
576
|
+
ON index_outbox(operation_key)
|
|
577
|
+
WHERE state IN ('pending','leased');
|
|
578
|
+
CREATE UNIQUE INDEX index_outbox_active_upsert_idx
|
|
579
|
+
ON index_outbox(backend, project_id, note_id, revision, generation)
|
|
580
|
+
WHERE operation = 'upsert-note' AND state IN ('pending','leased');
|
|
581
|
+
CREATE UNIQUE INDEX index_outbox_active_delete_idx
|
|
582
|
+
ON index_outbox(backend, project_id, note_id, revision, generation)
|
|
583
|
+
WHERE operation = 'delete-note' AND state IN ('pending','leased');
|
|
584
|
+
CREATE UNIQUE INDEX index_outbox_active_purge_idx
|
|
585
|
+
ON index_outbox(backend, project_id, generation)
|
|
586
|
+
WHERE operation = 'purge-project' AND state IN ('pending','leased');
|
|
587
|
+
CREATE TABLE schema_state (version INTEGER PRIMARY KEY);
|
|
588
|
+
CREATE TABLE agz_meta (
|
|
589
|
+
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
590
|
+
database_id TEXT NOT NULL UNIQUE,
|
|
591
|
+
product_id TEXT NOT NULL CHECK (product_id = '${PRODUCT_ID}'),
|
|
592
|
+
schema_version INTEGER NOT NULL CHECK (schema_version = 11),
|
|
593
|
+
schema_fingerprint TEXT NOT NULL CHECK (length(schema_fingerprint) = 64),
|
|
594
|
+
hash_policy TEXT NOT NULL CHECK (hash_policy = '${HASH_POLICY}'),
|
|
595
|
+
created_at INTEGER NOT NULL
|
|
596
|
+
);
|
|
597
|
+
`;
|
|
598
|
+
function captureEventsTableV11(table = "capture_events") {
|
|
599
|
+
return `CREATE TABLE ${table} (
|
|
600
|
+
idempotency_key TEXT PRIMARY KEY CHECK (length(idempotency_key) = 64),
|
|
601
|
+
contract TEXT NOT NULL CHECK (contract = '${CAPTURE_SCHEMA}'),
|
|
602
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
603
|
+
binding_key TEXT NOT NULL,
|
|
604
|
+
event_kind TEXT NOT NULL CHECK (event_kind IN ('user-candidate','assistant-candidate','session-summary','tool-signal')),
|
|
605
|
+
source_session_id TEXT NOT NULL,
|
|
606
|
+
source_message_id TEXT,
|
|
607
|
+
source_ordinal INTEGER CHECK (source_ordinal IS NULL OR source_ordinal >= 0),
|
|
608
|
+
source_tool_call_id TEXT,
|
|
609
|
+
payload_json TEXT CHECK (payload_json IS NULL OR json_valid(payload_json)),
|
|
610
|
+
payload_hash TEXT CHECK (payload_hash IS NULL OR length(payload_hash) = 64),
|
|
611
|
+
redaction_version TEXT NOT NULL,
|
|
612
|
+
state TEXT NOT NULL CHECK (state IN ('pending','shadowed','review','materialized','duplicate','ignored','rejected','quarantined','failed','dead')),
|
|
613
|
+
attempt_count INTEGER NOT NULL DEFAULT 0 CHECK (attempt_count >= 0),
|
|
614
|
+
note_id TEXT,
|
|
615
|
+
last_error_code TEXT,
|
|
616
|
+
generation INTEGER NOT NULL DEFAULT 0 CHECK (generation >= 0),
|
|
617
|
+
created_at INTEGER NOT NULL,
|
|
618
|
+
updated_at INTEGER NOT NULL,
|
|
619
|
+
processed_at INTEGER,
|
|
620
|
+
UNIQUE(project_id, idempotency_key),
|
|
621
|
+
CHECK (
|
|
622
|
+
(event_kind = 'user-candidate' AND source_message_id IS NOT NULL
|
|
623
|
+
AND source_ordinal IS NULL AND source_tool_call_id IS NULL)
|
|
624
|
+
OR (event_kind = 'assistant-candidate' AND source_message_id IS NOT NULL
|
|
625
|
+
AND source_ordinal IS NOT NULL AND source_tool_call_id IS NULL)
|
|
626
|
+
OR (event_kind = 'session-summary' AND source_message_id IS NOT NULL
|
|
627
|
+
AND source_ordinal IS NULL AND source_tool_call_id IS NULL)
|
|
628
|
+
OR (event_kind = 'tool-signal' AND source_message_id IS NOT NULL
|
|
629
|
+
AND source_ordinal IS NULL AND source_tool_call_id IS NOT NULL)
|
|
630
|
+
),
|
|
631
|
+
FOREIGN KEY (binding_key, project_id) REFERENCES project_bindings(binding_key, project_id) ON DELETE CASCADE
|
|
632
|
+
);
|
|
633
|
+
CREATE TRIGGER ${table}_note_project_insert
|
|
634
|
+
BEFORE INSERT ON ${table}
|
|
635
|
+
WHEN NEW.note_id IS NOT NULL
|
|
636
|
+
AND EXISTS (SELECT 1 FROM notes WHERE id = NEW.note_id)
|
|
637
|
+
AND NOT EXISTS (SELECT 1 FROM notes WHERE project_id = NEW.project_id AND id = NEW.note_id)
|
|
638
|
+
BEGIN
|
|
639
|
+
SELECT RAISE(ABORT, 'foreign key capture event note project mismatch');
|
|
640
|
+
END;
|
|
641
|
+
CREATE TRIGGER ${table}_note_project_update
|
|
642
|
+
BEFORE UPDATE OF project_id, note_id ON ${table}
|
|
643
|
+
WHEN NEW.note_id IS NOT NULL
|
|
644
|
+
AND EXISTS (SELECT 1 FROM notes WHERE id = NEW.note_id)
|
|
645
|
+
AND NOT EXISTS (SELECT 1 FROM notes WHERE project_id = NEW.project_id AND id = NEW.note_id)
|
|
646
|
+
BEGIN
|
|
647
|
+
SELECT RAISE(ABORT, 'foreign key capture event note project mismatch');
|
|
648
|
+
END;`;
|
|
649
|
+
}
|
|
650
|
+
function schemaFingerprint(db) {
|
|
651
|
+
const rows = db.query(`SELECT type, name, tbl_name, sql
|
|
652
|
+
FROM sqlite_master
|
|
653
|
+
WHERE name NOT LIKE 'sqlite_%'
|
|
654
|
+
ORDER BY type, name`).all();
|
|
655
|
+
const fields = [];
|
|
656
|
+
for (const row of rows) {
|
|
657
|
+
fields.push(row.type, row.name, row.tbl_name, SQLITE_MANAGED_FTS_TABLES.has(row.name) ? null : row.sql);
|
|
658
|
+
}
|
|
659
|
+
return hashTuple("schema-fingerprint", 2, fields);
|
|
660
|
+
}
|
|
661
|
+
var cachedExpectedFingerprint;
|
|
662
|
+
function expectedSchemaFingerprint() {
|
|
663
|
+
if (cachedExpectedFingerprint)
|
|
664
|
+
return cachedExpectedFingerprint;
|
|
665
|
+
const db = new Database(":memory:");
|
|
666
|
+
try {
|
|
667
|
+
db.exec(SCHEMA_V11_TABLES);
|
|
668
|
+
db.exec(FTS_V9);
|
|
669
|
+
cachedExpectedFingerprint = schemaFingerprint(db);
|
|
670
|
+
return cachedExpectedFingerprint;
|
|
671
|
+
} finally {
|
|
672
|
+
db.close();
|
|
673
|
+
}
|
|
674
|
+
}
|
|
675
|
+
function createSchemaV11(db) {
|
|
676
|
+
db.exec(`PRAGMA application_id = ${APPLICATION_ID}`);
|
|
677
|
+
db.exec(SCHEMA_V11_TABLES);
|
|
678
|
+
db.exec(FTS_V9);
|
|
679
|
+
db.query("INSERT INTO schema_state(version) VALUES (11)").run();
|
|
680
|
+
const fingerprint = schemaFingerprint(db);
|
|
681
|
+
if (fingerprint !== expectedSchemaFingerprint()) {
|
|
682
|
+
throw new Error("schema_fingerprint_mismatch");
|
|
683
|
+
}
|
|
684
|
+
db.query(`INSERT INTO agz_meta
|
|
685
|
+
(id, database_id, product_id, schema_version, schema_fingerprint, hash_policy, created_at)
|
|
686
|
+
VALUES (1, ?, ?, 11, ?, ?, ?)`).run(randomUUID(), PRODUCT_ID, fingerprint, HASH_POLICY, Date.now());
|
|
687
|
+
}
|
|
688
|
+
function insertV11Identity(db, databaseID = randomUUID(), createdAt = Date.now()) {
|
|
689
|
+
const fingerprint = schemaFingerprint(db);
|
|
690
|
+
if (fingerprint !== expectedSchemaFingerprint()) {
|
|
691
|
+
throw new Error("schema_fingerprint_mismatch");
|
|
692
|
+
}
|
|
693
|
+
db.query(`INSERT INTO agz_meta
|
|
694
|
+
(id, database_id, product_id, schema_version, schema_fingerprint, hash_policy, created_at)
|
|
695
|
+
VALUES (1, ?, ?, 11, ?, ?, ?)`).run(databaseID, PRODUCT_ID, fingerprint, HASH_POLICY, createdAt);
|
|
696
|
+
}
|
|
697
|
+
function rebuildFts(db) {
|
|
698
|
+
db.query("INSERT INTO notes_fts(notes_fts) VALUES ('rebuild')").run();
|
|
699
|
+
}
|
|
700
|
+
function captureEventsTable(table = "capture_events") {
|
|
701
|
+
return `CREATE TABLE IF NOT EXISTS ${table} (
|
|
702
|
+
idempotency_key TEXT PRIMARY KEY,
|
|
703
|
+
contract TEXT NOT NULL CHECK (contract = '${CAPTURE_SCHEMA}'),
|
|
704
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
705
|
+
binding_key TEXT NOT NULL REFERENCES project_bindings(binding_key) ON DELETE CASCADE,
|
|
706
|
+
event_kind TEXT NOT NULL CHECK (event_kind IN ('user-candidate','assistant-candidate','session-summary','tool-signal')),
|
|
707
|
+
source_session_id TEXT NOT NULL,
|
|
708
|
+
source_message_id TEXT,
|
|
709
|
+
source_ordinal INTEGER CHECK (source_ordinal IS NULL OR source_ordinal >= 0),
|
|
710
|
+
source_tool_call_id TEXT,
|
|
711
|
+
payload_json TEXT CHECK (payload_json IS NULL OR json_valid(payload_json)),
|
|
712
|
+
payload_hash TEXT,
|
|
713
|
+
redaction_version TEXT NOT NULL,
|
|
714
|
+
state TEXT NOT NULL CHECK (state IN ('pending','shadowed','review','materialized','duplicate','ignored','rejected','quarantined','failed','dead')),
|
|
715
|
+
attempt_count INTEGER NOT NULL DEFAULT 0 CHECK (attempt_count >= 0),
|
|
716
|
+
note_id TEXT,
|
|
717
|
+
last_error_code TEXT,
|
|
718
|
+
generation INTEGER NOT NULL DEFAULT 0 CHECK (generation >= 0),
|
|
719
|
+
created_at INTEGER NOT NULL,
|
|
720
|
+
updated_at INTEGER NOT NULL,
|
|
721
|
+
processed_at INTEGER
|
|
722
|
+
);`;
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
// src/db/health.ts
|
|
726
|
+
function inspectDatabase(db) {
|
|
727
|
+
const integrity = db.query("PRAGMA integrity_check").get().integrity_check;
|
|
728
|
+
const foreignKeyViolations = db.query("PRAGMA foreign_key_check").all();
|
|
729
|
+
const schemaVersion = hasTable(db, "schema_state") ? db.query("SELECT MAX(version) AS version FROM schema_state").get().version ?? undefined : undefined;
|
|
730
|
+
const counts = {};
|
|
731
|
+
for (const table of [
|
|
732
|
+
"projects",
|
|
733
|
+
"notes",
|
|
734
|
+
"note_edges",
|
|
735
|
+
"notes_fts",
|
|
736
|
+
"project_bindings",
|
|
737
|
+
"capture_events",
|
|
738
|
+
"capture_checkpoints",
|
|
739
|
+
"note_provenance",
|
|
740
|
+
"note_revisions",
|
|
741
|
+
"index_outbox"
|
|
742
|
+
]) {
|
|
743
|
+
if (!hasTable(db, table))
|
|
744
|
+
continue;
|
|
745
|
+
counts[table] = db.query(`SELECT COUNT(*) AS count FROM ${table}`).get().count;
|
|
746
|
+
}
|
|
747
|
+
return { integrity, foreignKeyViolations, schemaVersion, counts };
|
|
748
|
+
}
|
|
749
|
+
function assertHealthyDatabase(db) {
|
|
750
|
+
const health = inspectDatabase(db);
|
|
751
|
+
if (health.integrity !== "ok") {
|
|
752
|
+
throw new Error(`database integrity check failed: ${health.integrity}`);
|
|
753
|
+
}
|
|
754
|
+
if (health.foreignKeyViolations.length > 0) {
|
|
755
|
+
throw new Error(`database foreign key check failed: ${health.foreignKeyViolations.length} violation(s)`);
|
|
756
|
+
}
|
|
757
|
+
if (health.schemaVersion === 11)
|
|
758
|
+
assertSchemaV11(db);
|
|
759
|
+
return health;
|
|
760
|
+
}
|
|
761
|
+
function assertSchemaV11(db) {
|
|
762
|
+
try {
|
|
763
|
+
if (!hasTable(db, "agz_meta"))
|
|
764
|
+
throw new Error("missing metadata table");
|
|
765
|
+
const applicationID = db.query("PRAGMA application_id").get().application_id;
|
|
766
|
+
if (applicationID !== APPLICATION_ID)
|
|
767
|
+
throw new Error("application id mismatch");
|
|
768
|
+
const states = db.query("SELECT version FROM schema_state").all();
|
|
769
|
+
if (states.length !== 1 || states[0]?.version !== 11)
|
|
770
|
+
throw new Error("schema state mismatch");
|
|
771
|
+
const metadata = db.query(`SELECT id, database_id, product_id, schema_version, schema_fingerprint, hash_policy, created_at
|
|
772
|
+
FROM agz_meta`).all();
|
|
773
|
+
if (metadata.length !== 1)
|
|
774
|
+
throw new Error("metadata cardinality mismatch");
|
|
775
|
+
const [meta] = metadata;
|
|
776
|
+
if (!meta || meta.id !== 1 || !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(meta.database_id) || meta.product_id !== PRODUCT_ID || meta.schema_version !== 11 || meta.hash_policy !== HASH_POLICY || !/^[0-9a-f]{64}$/.test(meta.schema_fingerprint) || !Number.isSafeInteger(meta.created_at) || meta.created_at < 0) {
|
|
777
|
+
throw new Error("metadata value mismatch");
|
|
778
|
+
}
|
|
779
|
+
const expected = expectedSchemaFingerprint();
|
|
780
|
+
if (meta.schema_fingerprint !== expected || schemaFingerprint(db) !== expected) {
|
|
781
|
+
throw new Error("schema fingerprint mismatch");
|
|
782
|
+
}
|
|
783
|
+
} catch (error) {
|
|
784
|
+
if (isSQLiteBusyError(error))
|
|
785
|
+
throw error;
|
|
786
|
+
if (error instanceof Error && error.message === "schema_fingerprint_mismatch")
|
|
787
|
+
throw error;
|
|
788
|
+
throw new Error("schema_fingerprint_mismatch");
|
|
789
|
+
}
|
|
790
|
+
}
|
|
791
|
+
function isSQLiteBusyError(error) {
|
|
792
|
+
if (error && typeof error === "object" && "code" in error) {
|
|
793
|
+
const code = String(error.code);
|
|
794
|
+
if (code === "SQLITE_BUSY")
|
|
795
|
+
return true;
|
|
796
|
+
}
|
|
797
|
+
return error instanceof Error && /\bdatabase is (?:busy|locked)\b/i.test(error.message);
|
|
798
|
+
}
|
|
799
|
+
function hasTable(db, table) {
|
|
800
|
+
const row = db.query("SELECT COUNT(*) AS count FROM sqlite_master WHERE type IN ('table','view') AND name = ?").get(table);
|
|
801
|
+
return row.count > 0;
|
|
802
|
+
}
|
|
803
|
+
|
|
804
|
+
// src/db/migrations/v011.ts
|
|
805
|
+
import { createHash as createHash3 } from "crypto";
|
|
806
|
+
|
|
807
|
+
// src/capture/identity.ts
|
|
808
|
+
function captureIdempotencyKey(input) {
|
|
809
|
+
let fields;
|
|
810
|
+
if (input.kind === "user") {
|
|
811
|
+
fields = ["user", input.bindingKey, input.sessionID, input.messageID];
|
|
812
|
+
} else if (input.kind === "assistant") {
|
|
813
|
+
if (!Number.isSafeInteger(input.ordinal) || input.ordinal < 0) {
|
|
814
|
+
throw new RangeError("assistant capture ordinal must be a non-negative safe integer");
|
|
815
|
+
}
|
|
816
|
+
fields = [
|
|
817
|
+
"assistant",
|
|
818
|
+
input.bindingKey,
|
|
819
|
+
input.sessionID,
|
|
820
|
+
input.assistantMessageID,
|
|
821
|
+
input.ordinal
|
|
822
|
+
];
|
|
823
|
+
} else if (input.kind === "tool") {
|
|
824
|
+
fields = [
|
|
825
|
+
"tool",
|
|
826
|
+
input.bindingKey,
|
|
827
|
+
input.sessionID,
|
|
828
|
+
input.assistantMessageID,
|
|
829
|
+
input.toolCallID,
|
|
830
|
+
input.terminalStatus
|
|
831
|
+
];
|
|
832
|
+
} else {
|
|
833
|
+
fields = ["summary", input.bindingKey, input.sessionID, input.checkpointMessageID];
|
|
834
|
+
}
|
|
835
|
+
return hashTuple("capture-identity", 2, fields);
|
|
836
|
+
}
|
|
837
|
+
|
|
838
|
+
// src/capture/redact.ts
|
|
839
|
+
var SECRET_ASSIGNMENT_KEY = String.raw`(?:[A-Za-z][A-Za-z0-9_-]*(?:password|passwd|secret|token|api[_-]?key|private[_-]?key)[A-Za-z0-9_-]*|password|passwd|secret|token|api[_-]?key|private[_-]?key)`;
|
|
840
|
+
var SECRET_ASSIGNMENT_VALUE = String.raw`(?:"(?![<$\[])[^"\r\n]{8,}"|'(?![<$\[])[^'\r\n]{8,}'|(?![<$\[])[^\s,;]{8,})`;
|
|
841
|
+
var RULES = [
|
|
842
|
+
{
|
|
843
|
+
name: "private-key",
|
|
844
|
+
pattern: /-----BEGIN (?:RSA |EC |OPENSSH |DSA |ENCRYPTED |PGP )?PRIVATE KEY(?: BLOCK)?-----[\s\S]*?-----END (?:RSA |EC |OPENSSH |DSA |ENCRYPTED |PGP )?PRIVATE KEY(?: BLOCK)?-----/gi
|
|
845
|
+
},
|
|
846
|
+
{
|
|
847
|
+
name: "credential-uri",
|
|
848
|
+
pattern: /[a-z][a-z0-9+.-]*:\/\/[^\s/:@]+:[^\s/@]+@[^\s]+/gi
|
|
849
|
+
},
|
|
850
|
+
{
|
|
851
|
+
name: "bearer",
|
|
852
|
+
pattern: /Bearer\s+[A-Za-z0-9._~+/=-]{12,}(?=$|[^A-Za-z0-9._~+/=-])/gi
|
|
853
|
+
},
|
|
854
|
+
{
|
|
855
|
+
name: "basic-auth",
|
|
856
|
+
pattern: /Basic\s+[A-Za-z0-9+/=]{12,}(?=$|[^A-Za-z0-9+/=])/gi
|
|
857
|
+
},
|
|
858
|
+
{
|
|
859
|
+
name: "github-token",
|
|
860
|
+
pattern: /(?:gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,})(?=$|[^A-Za-z0-9_])/g
|
|
861
|
+
},
|
|
862
|
+
{
|
|
863
|
+
name: "gitlab-token",
|
|
864
|
+
pattern: /glpat-[A-Za-z0-9_-]{16,}(?=$|[^A-Za-z0-9_-])/g
|
|
865
|
+
},
|
|
866
|
+
{
|
|
867
|
+
name: "aws-access-key",
|
|
868
|
+
pattern: /(?:AKIA|ASIA)[A-Z0-9]{16}(?=$|[^A-Z0-9])/g
|
|
869
|
+
},
|
|
870
|
+
{
|
|
871
|
+
name: "jwt",
|
|
872
|
+
pattern: /eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}(?=$|[^A-Za-z0-9_-])/g
|
|
873
|
+
},
|
|
874
|
+
{
|
|
875
|
+
name: "anthropic-token",
|
|
876
|
+
pattern: /sk-ant-[A-Za-z0-9_-]{20,}(?=$|[^A-Za-z0-9_-])/g
|
|
877
|
+
},
|
|
878
|
+
{
|
|
879
|
+
name: "openai-token",
|
|
880
|
+
pattern: /sk-(?!ant-)(?:proj-)?[A-Za-z0-9_-]{20,}(?=$|[^A-Za-z0-9_-])/g
|
|
881
|
+
},
|
|
882
|
+
{
|
|
883
|
+
name: "google-api-key",
|
|
884
|
+
pattern: /AIza[A-Za-z0-9_-]{30,}(?=$|[^A-Za-z0-9_-])/g
|
|
885
|
+
},
|
|
886
|
+
{
|
|
887
|
+
name: "slack-token",
|
|
888
|
+
pattern: /xox[baprs]-[A-Za-z0-9-]{32,}(?=$|[^A-Za-z0-9-])/g
|
|
889
|
+
},
|
|
890
|
+
{
|
|
891
|
+
name: "stripe-token",
|
|
892
|
+
pattern: /[sr]k_(?:live|test)_[A-Za-z0-9]{16,}(?=$|[^A-Za-z0-9])/g
|
|
893
|
+
},
|
|
894
|
+
{
|
|
895
|
+
name: "npm-token",
|
|
896
|
+
pattern: /npm_[A-Za-z0-9]{20,}(?=$|[^A-Za-z0-9])/g
|
|
897
|
+
},
|
|
898
|
+
{
|
|
899
|
+
name: "api-key-header",
|
|
900
|
+
pattern: new RegExp(String.raw`(?:x-api-key|api-key|api_key)\s*[:=]\s*${SECRET_ASSIGNMENT_VALUE}`, "gi")
|
|
901
|
+
},
|
|
902
|
+
{
|
|
903
|
+
name: "secret-assignment",
|
|
904
|
+
pattern: new RegExp(String.raw`${SECRET_ASSIGNMENT_KEY}\s*[:=]\s*${SECRET_ASSIGNMENT_VALUE}`, "gi")
|
|
905
|
+
}
|
|
906
|
+
];
|
|
907
|
+
function redactText(value, options = {}) {
|
|
908
|
+
const maxCharacters = normalizeLimit(options.maxCharacters);
|
|
909
|
+
const inputLength = value.length;
|
|
910
|
+
let text = value;
|
|
911
|
+
let replacements = 0;
|
|
912
|
+
let detectedSecret = false;
|
|
913
|
+
const classes = {};
|
|
914
|
+
for (const literal2 of options.denylist ?? []) {
|
|
915
|
+
if (typeof literal2 !== "string" || !literal2)
|
|
916
|
+
continue;
|
|
917
|
+
let count = 0;
|
|
918
|
+
let offset = 0;
|
|
919
|
+
while (offset <= text.length) {
|
|
920
|
+
const index = text.indexOf(literal2, offset);
|
|
921
|
+
if (index < 0)
|
|
922
|
+
break;
|
|
923
|
+
count++;
|
|
924
|
+
offset = index + literal2.length;
|
|
925
|
+
}
|
|
926
|
+
if (count === 0)
|
|
927
|
+
continue;
|
|
928
|
+
replacements += count;
|
|
929
|
+
detectedSecret = true;
|
|
930
|
+
classes.denylist = (classes.denylist ?? 0) + count;
|
|
931
|
+
text = text.replaceAll(literal2, "[REDACTED:denylist]");
|
|
932
|
+
}
|
|
933
|
+
for (const rule of RULES) {
|
|
934
|
+
rule.pattern.lastIndex = 0;
|
|
935
|
+
text = text.replace(rule.pattern, () => {
|
|
936
|
+
replacements++;
|
|
937
|
+
detectedSecret = true;
|
|
938
|
+
classes[rule.name] = (classes[rule.name] ?? 0) + 1;
|
|
939
|
+
return `[REDACTED:${rule.name}]`;
|
|
940
|
+
});
|
|
941
|
+
}
|
|
942
|
+
text = text.replace(/[A-Za-z0-9+/=_-]{32,}/g, (candidate) => {
|
|
943
|
+
if (!looksHighEntropy(candidate))
|
|
944
|
+
return candidate;
|
|
945
|
+
replacements++;
|
|
946
|
+
detectedSecret = true;
|
|
947
|
+
classes.entropy = (classes.entropy ?? 0) + 1;
|
|
948
|
+
return "[REDACTED:entropy]";
|
|
949
|
+
});
|
|
950
|
+
const truncated = Boolean(options.sourceTruncated) || inputLength > maxCharacters || text.length > maxCharacters;
|
|
951
|
+
if (text.length > maxCharacters)
|
|
952
|
+
text = truncateHead(text, maxCharacters);
|
|
953
|
+
return {
|
|
954
|
+
text,
|
|
955
|
+
replacements,
|
|
956
|
+
classes,
|
|
957
|
+
truncated,
|
|
958
|
+
quarantined: detectedSecret
|
|
959
|
+
};
|
|
960
|
+
}
|
|
961
|
+
function normalizeLimit(value) {
|
|
962
|
+
if (value === undefined || value === Number.POSITIVE_INFINITY)
|
|
963
|
+
return Number.MAX_SAFE_INTEGER;
|
|
964
|
+
if (!Number.isFinite(value))
|
|
965
|
+
return 0;
|
|
966
|
+
return Math.max(0, Math.floor(value));
|
|
967
|
+
}
|
|
968
|
+
function truncateHead(value, maxCharacters) {
|
|
969
|
+
if (maxCharacters <= 0)
|
|
970
|
+
return "";
|
|
971
|
+
if (value.length <= maxCharacters)
|
|
972
|
+
return value;
|
|
973
|
+
let offset = 0;
|
|
974
|
+
while (offset < value.length) {
|
|
975
|
+
const codePoint = value.codePointAt(offset);
|
|
976
|
+
const width = codePoint > 65535 ? 2 : 1;
|
|
977
|
+
if (offset + width > maxCharacters)
|
|
978
|
+
break;
|
|
979
|
+
offset += width;
|
|
980
|
+
}
|
|
981
|
+
return value.slice(0, offset);
|
|
982
|
+
}
|
|
983
|
+
function looksHighEntropy(value) {
|
|
984
|
+
if (!/[A-Za-z]/.test(value) || !/\d/.test(value))
|
|
985
|
+
return false;
|
|
986
|
+
const counts = new Map;
|
|
987
|
+
for (const character of value)
|
|
988
|
+
counts.set(character, (counts.get(character) ?? 0) + 1);
|
|
989
|
+
let entropy = 0;
|
|
990
|
+
for (const count of counts.values()) {
|
|
991
|
+
const probability = count / value.length;
|
|
992
|
+
entropy -= probability * Math.log2(probability);
|
|
993
|
+
}
|
|
994
|
+
return entropy >= 4.1;
|
|
995
|
+
}
|
|
996
|
+
|
|
997
|
+
// src/retrieval/derived.ts
|
|
998
|
+
function deriveDocument(source) {
|
|
999
|
+
const title = redactText(source.title);
|
|
1000
|
+
const summary = redactText(source.summary);
|
|
1001
|
+
const content = redactText(source.content);
|
|
1002
|
+
const contentHash = hashTuple("derived-note", 2, [source.kind, title.text, summary.text, content.text]);
|
|
1003
|
+
return {
|
|
1004
|
+
projectID: source.projectID,
|
|
1005
|
+
noteID: source.noteID,
|
|
1006
|
+
revision: source.revision,
|
|
1007
|
+
kind: source.kind,
|
|
1008
|
+
title: title.text,
|
|
1009
|
+
summary: summary.text,
|
|
1010
|
+
content: content.text,
|
|
1011
|
+
contentHash
|
|
1012
|
+
};
|
|
1013
|
+
}
|
|
1014
|
+
|
|
1015
|
+
// src/db/migrations/v011.ts
|
|
1016
|
+
var LEGACY_CAPTURE_SCHEMA = "agz-memory.capture/1";
|
|
1017
|
+
var CAPTURE_SCHEMAS = new Set([LEGACY_CAPTURE_SCHEMA, CAPTURE_SCHEMA]);
|
|
1018
|
+
var KINDS_SET = new Set(KINDS);
|
|
1019
|
+
var PREDICATES_SET = new Set(PREDICATES);
|
|
1020
|
+
function migrateV10ToV11(db) {
|
|
1021
|
+
const source = readSourceRows(db);
|
|
1022
|
+
const migrated = transformRows(source);
|
|
1023
|
+
replaceWithV11(db, migrated);
|
|
1024
|
+
console.warn(`[agz-memory] v10\u2192v11 migration complete: projects=${migrated.projects.length}, notes=${migrated.notes.length}, edges=${migrated.edges.length}, bindings=${migrated.bindings.length}, checkpoints=${migrated.checkpoints.length}, captures=${migrated.captures.length}, revisions=${migrated.revisions.length}, outbox=${migrated.outbox.length}`);
|
|
1025
|
+
}
|
|
1026
|
+
function assertV10SourceDatabase(db) {
|
|
1027
|
+
transformRows(readSourceRows(db));
|
|
1028
|
+
}
|
|
1029
|
+
function readSourceRows(db) {
|
|
1030
|
+
for (const table of [
|
|
1031
|
+
"projects",
|
|
1032
|
+
"notes",
|
|
1033
|
+
"note_edges",
|
|
1034
|
+
"project_bindings",
|
|
1035
|
+
"capture_checkpoints",
|
|
1036
|
+
"capture_events",
|
|
1037
|
+
"note_provenance",
|
|
1038
|
+
"note_revisions",
|
|
1039
|
+
"index_outbox",
|
|
1040
|
+
"schema_state"
|
|
1041
|
+
]) {
|
|
1042
|
+
requireTable(db, table);
|
|
1043
|
+
}
|
|
1044
|
+
if (!hasTable2(db, "notes_fts"))
|
|
1045
|
+
fail("notes_fts", "schema", "source_schema");
|
|
1046
|
+
validateSourceObjects(db);
|
|
1047
|
+
const states = selectRows(db, "schema_state", "SELECT version FROM schema_state");
|
|
1048
|
+
if (states.length !== 1 || states[0]?.version !== 10)
|
|
1049
|
+
fail("schema_state", "schema", "source_schema");
|
|
1050
|
+
const projects = selectRows(db, "projects", "SELECT id, name, normalized_name, created_at, updated_at FROM projects ORDER BY rowid");
|
|
1051
|
+
const notes = selectRows(db, "notes", `SELECT id, project_id, kind, title, summary, content, size_class, pinned, status,
|
|
1052
|
+
supersedes_id, current_revision, subject_key, content_hash, created_at, updated_at
|
|
1053
|
+
FROM notes ORDER BY rowid`);
|
|
1054
|
+
const edges = selectRows(db, "note_edges", "SELECT id, project_id, source_id, target_id, predicate, created_at FROM note_edges ORDER BY rowid");
|
|
1055
|
+
const bindings = selectRows(db, "project_bindings", `SELECT binding_key, project_id, source, source_project_id, workspace_id,
|
|
1056
|
+
canonical_path_hash, created_at, updated_at
|
|
1057
|
+
FROM project_bindings ORDER BY rowid`);
|
|
1058
|
+
const checkpoints = selectRows(db, "capture_checkpoints", `SELECT session_id, binding_key, project_id, state, last_message_id,
|
|
1059
|
+
last_reconciled_at, next_reconcile_at, failure_count, lease_owner,
|
|
1060
|
+
lease_expires_at, created_at, updated_at
|
|
1061
|
+
FROM capture_checkpoints ORDER BY rowid`);
|
|
1062
|
+
const captures = selectRows(db, "capture_events", `SELECT idempotency_key, contract, project_id, binding_key, event_kind,
|
|
1063
|
+
source_session_id, source_message_id, source_ordinal, source_tool_call_id,
|
|
1064
|
+
payload_json, payload_hash, redaction_version, state, attempt_count,
|
|
1065
|
+
note_id, last_error_code, generation, created_at, updated_at, processed_at
|
|
1066
|
+
FROM capture_events ORDER BY rowid`);
|
|
1067
|
+
const provenance = selectRows(db, "note_provenance", `SELECT id, project_id, note_id, source_type, capture_event_id,
|
|
1068
|
+
source_session_id, source_message_id, source_ordinal, source_tool_call_id,
|
|
1069
|
+
redaction_version, extractor_version, confidence, created_at
|
|
1070
|
+
FROM note_provenance ORDER BY rowid`);
|
|
1071
|
+
const revisions = selectRows(db, "note_revisions", `SELECT project_id, note_id, revision, kind, title, summary, content, size_class,
|
|
1072
|
+
pinned, status, supersedes_id, subject_key, content_hash, provenance_id, created_at
|
|
1073
|
+
FROM note_revisions ORDER BY rowid`);
|
|
1074
|
+
const outbox = selectRows(db, "index_outbox", `SELECT id, backend, operation, project_id, note_id, revision, content_hash,
|
|
1075
|
+
state, attempt_count, available_at, lease_owner, lease_expires_at,
|
|
1076
|
+
last_error_code, created_at, completed_at
|
|
1077
|
+
FROM index_outbox ORDER BY id`);
|
|
1078
|
+
return { projects, notes, edges, bindings, checkpoints, captures, provenance, revisions, outbox };
|
|
1079
|
+
}
|
|
1080
|
+
function validateSourceObjects(db) {
|
|
1081
|
+
const expected = new Set([
|
|
1082
|
+
"index\x00capture_checkpoints_due_idx",
|
|
1083
|
+
"index\x00capture_events_note_idx",
|
|
1084
|
+
"index\x00capture_events_session_idx",
|
|
1085
|
+
"index\x00capture_events_state_idx",
|
|
1086
|
+
"index\x00index_outbox_due_idx",
|
|
1087
|
+
"index\x00note_edges_source_idx",
|
|
1088
|
+
"index\x00note_edges_target_idx",
|
|
1089
|
+
"index\x00notes_active_subject_idx",
|
|
1090
|
+
"index\x00notes_project_idx",
|
|
1091
|
+
"table\x00capture_checkpoints",
|
|
1092
|
+
"table\x00capture_events",
|
|
1093
|
+
"table\x00index_outbox",
|
|
1094
|
+
"table\x00note_edges",
|
|
1095
|
+
"table\x00note_provenance",
|
|
1096
|
+
"table\x00note_revisions",
|
|
1097
|
+
"table\x00notes",
|
|
1098
|
+
"table\x00notes_fts",
|
|
1099
|
+
"table\x00notes_fts_config",
|
|
1100
|
+
"table\x00notes_fts_data",
|
|
1101
|
+
"table\x00notes_fts_docsize",
|
|
1102
|
+
"table\x00notes_fts_idx",
|
|
1103
|
+
"table\x00project_bindings",
|
|
1104
|
+
"table\x00projects",
|
|
1105
|
+
"table\x00schema_state",
|
|
1106
|
+
"trigger\x00notes_fts_ad",
|
|
1107
|
+
"trigger\x00notes_fts_ai",
|
|
1108
|
+
"trigger\x00notes_fts_au"
|
|
1109
|
+
]);
|
|
1110
|
+
const actual = db.query("SELECT type, name FROM sqlite_master WHERE name NOT LIKE 'sqlite_%'").all();
|
|
1111
|
+
if (actual.length !== expected.size || actual.some((object2) => !expected.has(`${object2.type}\x00${object2.name}`))) {
|
|
1112
|
+
fail("database", "schema", "source_schema");
|
|
1113
|
+
}
|
|
1114
|
+
}
|
|
1115
|
+
function transformRows(source) {
|
|
1116
|
+
const projectIDs = new Set;
|
|
1117
|
+
const normalizedProjectNames = new Set;
|
|
1118
|
+
for (const project of source.projects) {
|
|
1119
|
+
requireString(project.id, "projects", project.id, "project_id_invalid");
|
|
1120
|
+
requireString(project.name, "projects", project.id, "project_name_invalid");
|
|
1121
|
+
requireString(project.normalized_name, "projects", project.id, "project_name_invalid");
|
|
1122
|
+
requireInteger(project.created_at, "projects", project.id, "timestamp_invalid");
|
|
1123
|
+
requireInteger(project.updated_at, "projects", project.id, "timestamp_invalid");
|
|
1124
|
+
if (projectIDs.has(project.id))
|
|
1125
|
+
fail("projects", project.id, "duplicate_row");
|
|
1126
|
+
if (normalizedProjectNames.has(project.normalized_name)) {
|
|
1127
|
+
fail("projects", project.id, "duplicate_normalized_name");
|
|
1128
|
+
}
|
|
1129
|
+
projectIDs.add(project.id);
|
|
1130
|
+
normalizedProjectNames.add(project.normalized_name);
|
|
1131
|
+
}
|
|
1132
|
+
const noteMap = new Map;
|
|
1133
|
+
const noteIDs = new Set;
|
|
1134
|
+
for (const note of source.notes) {
|
|
1135
|
+
requireString(note.id, "notes", note.id, "note_id_invalid");
|
|
1136
|
+
requireString(note.project_id, "notes", note.id, "project_id_invalid");
|
|
1137
|
+
if (!projectIDs.has(note.project_id))
|
|
1138
|
+
fail("notes", note.id, "cross_project_reference");
|
|
1139
|
+
requireKind(note.kind, "notes", note.id);
|
|
1140
|
+
requireString(note.title, "notes", note.id, "note_value_invalid");
|
|
1141
|
+
requireString(note.summary, "notes", note.id, "note_value_invalid");
|
|
1142
|
+
requireString(note.content, "notes", note.id, "note_value_invalid");
|
|
1143
|
+
if (note.size_class !== "inline" && note.size_class !== "indexed") {
|
|
1144
|
+
fail("notes", note.id, "note_value_invalid");
|
|
1145
|
+
}
|
|
1146
|
+
if (note.pinned !== 0 && note.pinned !== 1)
|
|
1147
|
+
fail("notes", note.id, "note_value_invalid");
|
|
1148
|
+
if (!new Set(["active", "superseded", "archived"]).has(note.status)) {
|
|
1149
|
+
fail("notes", note.id, "note_value_invalid");
|
|
1150
|
+
}
|
|
1151
|
+
requireNullableString(note.supersedes_id, "notes", note.id, "note_value_invalid");
|
|
1152
|
+
requireIntegerAtLeast(note.current_revision, 1, "notes", note.id, "revision_invalid");
|
|
1153
|
+
requireNullableString(note.subject_key, "notes", note.id, "note_value_invalid");
|
|
1154
|
+
requireString(note.content_hash, "notes", note.id, "hash_invalid");
|
|
1155
|
+
const legacyHash = legacyNoteContentHash(note.kind, note.title, note.summary, note.content);
|
|
1156
|
+
const v2Hash = noteContentHash(note.kind, note.title, note.summary, note.content);
|
|
1157
|
+
if (note.content_hash !== legacyHash && note.content_hash !== v2Hash) {
|
|
1158
|
+
fail("notes", note.id, "hash_invalid");
|
|
1159
|
+
}
|
|
1160
|
+
requireInteger(note.created_at, "notes", note.id, "timestamp_invalid");
|
|
1161
|
+
requireInteger(note.updated_at, "notes", note.id, "timestamp_invalid");
|
|
1162
|
+
if (noteMap.has(noteKey(note.project_id, note.id)))
|
|
1163
|
+
fail("notes", note.id, "duplicate_row");
|
|
1164
|
+
noteMap.set(noteKey(note.project_id, note.id), note);
|
|
1165
|
+
noteIDs.add(note.id);
|
|
1166
|
+
}
|
|
1167
|
+
const activeSubjects = new Set;
|
|
1168
|
+
for (const note of source.notes) {
|
|
1169
|
+
if (note.supersedes_id !== null) {
|
|
1170
|
+
const superseded = noteMap.get(noteKey(note.project_id, note.supersedes_id));
|
|
1171
|
+
if (!superseded || note.supersedes_id === note.id) {
|
|
1172
|
+
fail("notes", note.id, "cross_project_reference");
|
|
1173
|
+
}
|
|
1174
|
+
}
|
|
1175
|
+
if (note.status === "active" && note.subject_key !== null) {
|
|
1176
|
+
const subject = hashTuple("active-subject", 2, [note.project_id, note.kind, note.subject_key]);
|
|
1177
|
+
if (activeSubjects.has(subject))
|
|
1178
|
+
fail("notes", note.id, "duplicate_active_subject");
|
|
1179
|
+
activeSubjects.add(subject);
|
|
1180
|
+
}
|
|
1181
|
+
}
|
|
1182
|
+
const edges = source.edges.map((edge) => {
|
|
1183
|
+
requireString(edge.id, "note_edges", edge.id, "edge_id_invalid");
|
|
1184
|
+
requireString(edge.project_id, "note_edges", edge.id, "project_id_invalid");
|
|
1185
|
+
requireString(edge.source_id, "note_edges", edge.id, "edge_value_invalid");
|
|
1186
|
+
requireString(edge.target_id, "note_edges", edge.id, "edge_value_invalid");
|
|
1187
|
+
const sourceNote = noteMap.get(noteKey(edge.project_id, edge.source_id));
|
|
1188
|
+
const targetNote = noteMap.get(noteKey(edge.project_id, edge.target_id));
|
|
1189
|
+
if (!projectIDs.has(edge.project_id) || !sourceNote || !targetNote) {
|
|
1190
|
+
fail("note_edges", edge.id, "cross_project_reference");
|
|
1191
|
+
}
|
|
1192
|
+
if (!PREDICATES_SET.has(edge.predicate))
|
|
1193
|
+
fail("note_edges", edge.id, "edge_value_invalid");
|
|
1194
|
+
requireInteger(edge.created_at, "note_edges", edge.id, "timestamp_invalid");
|
|
1195
|
+
return edge;
|
|
1196
|
+
});
|
|
1197
|
+
const bindingMap = new Map;
|
|
1198
|
+
const bindingProjects = new Map;
|
|
1199
|
+
const bindingKeys = new Set;
|
|
1200
|
+
const bindings = source.bindings.map((binding) => {
|
|
1201
|
+
requireHash(binding.binding_key, "project_bindings", binding.binding_key, "binding_id_invalid");
|
|
1202
|
+
requireString(binding.project_id, "project_bindings", binding.binding_key, "project_id_invalid");
|
|
1203
|
+
if (!projectIDs.has(binding.project_id)) {
|
|
1204
|
+
fail("project_bindings", binding.binding_key, "cross_project_reference");
|
|
1205
|
+
}
|
|
1206
|
+
if (binding.source !== "opencode-v2")
|
|
1207
|
+
fail("project_bindings", binding.binding_key, "binding_value_invalid");
|
|
1208
|
+
requireString(binding.source_project_id, "project_bindings", binding.binding_key, "binding_value_invalid");
|
|
1209
|
+
requireString(binding.workspace_id, "project_bindings", binding.binding_key, "binding_value_invalid");
|
|
1210
|
+
requireHash(binding.canonical_path_hash, "project_bindings", binding.binding_key, "hash_invalid");
|
|
1211
|
+
requireInteger(binding.created_at, "project_bindings", binding.binding_key, "timestamp_invalid");
|
|
1212
|
+
requireInteger(binding.updated_at, "project_bindings", binding.binding_key, "timestamp_invalid");
|
|
1213
|
+
const nextKey = hashTuple("project-binding", 2, [
|
|
1214
|
+
binding.source,
|
|
1215
|
+
binding.source_project_id,
|
|
1216
|
+
binding.workspace_id,
|
|
1217
|
+
binding.canonical_path_hash
|
|
1218
|
+
]);
|
|
1219
|
+
if (bindingKeys.has(nextKey))
|
|
1220
|
+
fail("project_bindings", binding.binding_key, "identity_collision");
|
|
1221
|
+
bindingKeys.add(nextKey);
|
|
1222
|
+
bindingMap.set(binding.binding_key, nextKey);
|
|
1223
|
+
bindingProjects.set(binding.binding_key, binding.project_id);
|
|
1224
|
+
return { ...binding, binding_key: nextKey };
|
|
1225
|
+
});
|
|
1226
|
+
const checkpointIdentities = new Set;
|
|
1227
|
+
const checkpoints = source.checkpoints.map((checkpoint) => {
|
|
1228
|
+
requireNonEmptyString(checkpoint.session_id, "capture_checkpoints", checkpoint.session_id, "checkpoint_id_invalid");
|
|
1229
|
+
requireHash(checkpoint.binding_key, "capture_checkpoints", checkpoint.session_id, "checkpoint_value_invalid");
|
|
1230
|
+
requireString(checkpoint.project_id, "capture_checkpoints", checkpoint.session_id, "project_id_invalid");
|
|
1231
|
+
const nextBinding = bindingMap.get(checkpoint.binding_key);
|
|
1232
|
+
if (!nextBinding || checkpoint.project_id !== bindingProjects.get(checkpoint.binding_key)) {
|
|
1233
|
+
fail("capture_checkpoints", checkpoint.session_id, "cross_project_reference");
|
|
1234
|
+
}
|
|
1235
|
+
if (checkpoint.state !== "active" && checkpoint.state !== "idle" && checkpoint.state !== "unavailable" && checkpoint.state !== "closed") {
|
|
1236
|
+
fail("capture_checkpoints", checkpoint.session_id, "checkpoint_value_invalid");
|
|
1237
|
+
}
|
|
1238
|
+
requireNullableString(checkpoint.last_message_id, "capture_checkpoints", checkpoint.session_id, "checkpoint_value_invalid");
|
|
1239
|
+
requireNullableInteger(checkpoint.last_reconciled_at, "capture_checkpoints", checkpoint.session_id, "timestamp_invalid");
|
|
1240
|
+
requireInteger(checkpoint.next_reconcile_at, "capture_checkpoints", checkpoint.session_id, "timestamp_invalid");
|
|
1241
|
+
requireIntegerAtLeast(checkpoint.failure_count, 0, "capture_checkpoints", checkpoint.session_id, "checkpoint_value_invalid");
|
|
1242
|
+
requireNullableString(checkpoint.lease_owner, "capture_checkpoints", checkpoint.session_id, "checkpoint_value_invalid");
|
|
1243
|
+
requireNullableInteger(checkpoint.lease_expires_at, "capture_checkpoints", checkpoint.session_id, "timestamp_invalid");
|
|
1244
|
+
requireInteger(checkpoint.created_at, "capture_checkpoints", checkpoint.session_id, "timestamp_invalid");
|
|
1245
|
+
requireInteger(checkpoint.updated_at, "capture_checkpoints", checkpoint.session_id, "timestamp_invalid");
|
|
1246
|
+
const identity = hashTuple("checkpoint-identity", 2, [nextBinding, checkpoint.session_id]);
|
|
1247
|
+
if (checkpointIdentities.has(identity)) {
|
|
1248
|
+
fail("capture_checkpoints", checkpoint.session_id, "identity_collision");
|
|
1249
|
+
}
|
|
1250
|
+
checkpointIdentities.add(identity);
|
|
1251
|
+
return { ...checkpoint, binding_key: nextBinding, last_message_id: null };
|
|
1252
|
+
});
|
|
1253
|
+
const captureMap = new Map;
|
|
1254
|
+
const captureProjects = new Map;
|
|
1255
|
+
const captureKeys = new Set;
|
|
1256
|
+
const captures = source.captures.map((capture) => {
|
|
1257
|
+
requireHash(capture.idempotency_key, "capture_events", capture.idempotency_key, "capture_id_invalid");
|
|
1258
|
+
requireString(capture.project_id, "capture_events", capture.idempotency_key, "project_id_invalid");
|
|
1259
|
+
requireHash(capture.binding_key, "capture_events", capture.idempotency_key, "binding_id_invalid");
|
|
1260
|
+
const nextBinding = bindingMap.get(capture.binding_key);
|
|
1261
|
+
const bindingProjectID = bindingProjects.get(capture.binding_key);
|
|
1262
|
+
if (!nextBinding || !bindingProjectID || capture.project_id !== bindingProjectID) {
|
|
1263
|
+
fail("capture_events", capture.idempotency_key, "cross_project_reference");
|
|
1264
|
+
}
|
|
1265
|
+
const migrated = migrateCapture(capture, nextBinding, noteMap, noteIDs);
|
|
1266
|
+
if (captureKeys.has(migrated.idempotency_key)) {
|
|
1267
|
+
fail("capture_events", capture.idempotency_key, "identity_collision");
|
|
1268
|
+
}
|
|
1269
|
+
captureKeys.add(migrated.idempotency_key);
|
|
1270
|
+
captureMap.set(capture.idempotency_key, migrated.idempotency_key);
|
|
1271
|
+
captureProjects.set(capture.idempotency_key, capture.project_id);
|
|
1272
|
+
return migrated;
|
|
1273
|
+
});
|
|
1274
|
+
const provenanceKeys = new Set;
|
|
1275
|
+
const provenance = source.provenance.map((row) => {
|
|
1276
|
+
requireString(row.id, "note_provenance", row.id, "provenance_id_invalid");
|
|
1277
|
+
requireString(row.project_id, "note_provenance", row.id, "project_id_invalid");
|
|
1278
|
+
requireString(row.note_id, "note_provenance", row.id, "note_id_invalid");
|
|
1279
|
+
if (!noteMap.has(noteKey(row.project_id, row.note_id))) {
|
|
1280
|
+
fail("note_provenance", row.id, "cross_project_reference");
|
|
1281
|
+
}
|
|
1282
|
+
if (!new Set(["mcp-manual", "opencode-capture", "migration", "legacy-import", "admin"]).has(row.source_type)) {
|
|
1283
|
+
fail("note_provenance", row.id, "provenance_value_invalid");
|
|
1284
|
+
}
|
|
1285
|
+
requireNullableString(row.capture_event_id, "note_provenance", row.id, "provenance_value_invalid");
|
|
1286
|
+
if (row.capture_event_id !== null) {
|
|
1287
|
+
if (!captureMap.has(row.capture_event_id))
|
|
1288
|
+
fail("note_provenance", row.id, "capture_reference_invalid");
|
|
1289
|
+
if (captureProjects.get(row.capture_event_id) !== row.project_id) {
|
|
1290
|
+
fail("note_provenance", row.id, "cross_project_reference");
|
|
1291
|
+
}
|
|
1292
|
+
}
|
|
1293
|
+
requireNullableString(row.source_session_id, "note_provenance", row.id, "provenance_value_invalid");
|
|
1294
|
+
requireNullableString(row.source_message_id, "note_provenance", row.id, "provenance_value_invalid");
|
|
1295
|
+
requireNullableInteger(row.source_ordinal, "note_provenance", row.id, "provenance_value_invalid");
|
|
1296
|
+
requireNullableString(row.source_tool_call_id, "note_provenance", row.id, "provenance_value_invalid");
|
|
1297
|
+
requireNullableString(row.redaction_version, "note_provenance", row.id, "provenance_value_invalid");
|
|
1298
|
+
requireNullableString(row.extractor_version, "note_provenance", row.id, "provenance_value_invalid");
|
|
1299
|
+
if (row.confidence !== null && (!Number.isFinite(row.confidence) || row.confidence < 0 || row.confidence > 1)) {
|
|
1300
|
+
fail("note_provenance", row.id, "provenance_value_invalid");
|
|
1301
|
+
}
|
|
1302
|
+
requireInteger(row.created_at, "note_provenance", row.id, "timestamp_invalid");
|
|
1303
|
+
const key = noteKey(row.project_id, row.id);
|
|
1304
|
+
if (provenanceKeys.has(key))
|
|
1305
|
+
fail("note_provenance", row.id, "duplicate_row");
|
|
1306
|
+
provenanceKeys.add(key);
|
|
1307
|
+
return {
|
|
1308
|
+
...row,
|
|
1309
|
+
capture_event_id: row.capture_event_id === null ? null : captureMap.get(row.capture_event_id)
|
|
1310
|
+
};
|
|
1311
|
+
});
|
|
1312
|
+
const provenanceMap = new Map(provenance.map((row) => [noteKey(row.project_id, row.id), row]));
|
|
1313
|
+
const revisionMap = new Map;
|
|
1314
|
+
const revisions = source.revisions.map((revision) => {
|
|
1315
|
+
requireString(revision.project_id, "note_revisions", revision.note_id, "project_id_invalid");
|
|
1316
|
+
requireString(revision.note_id, "note_revisions", revision.note_id, "note_id_invalid");
|
|
1317
|
+
const note = noteMap.get(noteKey(revision.project_id, revision.note_id));
|
|
1318
|
+
const provenanceRow = provenanceMap.get(noteKey(revision.project_id, revision.provenance_id));
|
|
1319
|
+
if (!note || !provenanceRow)
|
|
1320
|
+
fail("note_revisions", revision.note_id, "cross_project_reference");
|
|
1321
|
+
requireIntegerAtLeast(revision.revision, 1, "note_revisions", revision.note_id, "revision_invalid");
|
|
1322
|
+
requireKind(revision.kind, "note_revisions", revision.note_id);
|
|
1323
|
+
requireString(revision.title, "note_revisions", revision.note_id, "revision_value_invalid");
|
|
1324
|
+
requireString(revision.summary, "note_revisions", revision.note_id, "revision_value_invalid");
|
|
1325
|
+
requireString(revision.content, "note_revisions", revision.note_id, "revision_value_invalid");
|
|
1326
|
+
if (revision.size_class !== "inline" && revision.size_class !== "indexed") {
|
|
1327
|
+
fail("note_revisions", revision.note_id, "revision_value_invalid");
|
|
1328
|
+
}
|
|
1329
|
+
if (revision.pinned !== 0 && revision.pinned !== 1)
|
|
1330
|
+
fail("note_revisions", revision.note_id, "revision_value_invalid");
|
|
1331
|
+
if (!new Set(["active", "superseded", "archived"]).has(revision.status)) {
|
|
1332
|
+
fail("note_revisions", revision.note_id, "revision_value_invalid");
|
|
1333
|
+
}
|
|
1334
|
+
requireNullableString(revision.supersedes_id, "note_revisions", revision.note_id, "revision_value_invalid");
|
|
1335
|
+
if (revision.supersedes_id !== null) {
|
|
1336
|
+
if (revision.supersedes_id === revision.note_id) {
|
|
1337
|
+
fail("note_revisions", revision.note_id, "revision_value_invalid");
|
|
1338
|
+
}
|
|
1339
|
+
if (!noteMap.has(noteKey(revision.project_id, revision.supersedes_id))) {
|
|
1340
|
+
fail("note_revisions", revision.note_id, "cross_project_reference");
|
|
1341
|
+
}
|
|
1342
|
+
}
|
|
1343
|
+
requireNullableString(revision.subject_key, "note_revisions", revision.note_id, "revision_value_invalid");
|
|
1344
|
+
requireString(revision.content_hash, "note_revisions", revision.note_id, "hash_invalid");
|
|
1345
|
+
const legacyHash = legacyNoteContentHash(revision.kind, revision.title, revision.summary, revision.content);
|
|
1346
|
+
const v2Hash = noteContentHash(revision.kind, revision.title, revision.summary, revision.content);
|
|
1347
|
+
if (revision.content_hash !== legacyHash && revision.content_hash !== v2Hash) {
|
|
1348
|
+
fail("note_revisions", revision.note_id, "hash_invalid");
|
|
1349
|
+
}
|
|
1350
|
+
requireInteger(revision.created_at, "note_revisions", revision.note_id, "timestamp_invalid");
|
|
1351
|
+
const key = revisionKey(revision.project_id, revision.note_id, revision.revision);
|
|
1352
|
+
if (revisionMap.has(key))
|
|
1353
|
+
fail("note_revisions", revision.note_id, "duplicate_row");
|
|
1354
|
+
const migrated = {
|
|
1355
|
+
...revision,
|
|
1356
|
+
content_hash: noteContentHash(revision.kind, revision.title, revision.summary, revision.content)
|
|
1357
|
+
};
|
|
1358
|
+
revisionMap.set(key, migrated);
|
|
1359
|
+
return migrated;
|
|
1360
|
+
});
|
|
1361
|
+
for (const note of source.notes) {
|
|
1362
|
+
const current = revisionMap.get(revisionKey(note.project_id, note.id, note.current_revision));
|
|
1363
|
+
if (!current)
|
|
1364
|
+
fail("note_revisions", note.id, "revision_invalid");
|
|
1365
|
+
for (let revision = 1;revision <= note.current_revision; revision++) {
|
|
1366
|
+
if (!revisionMap.has(revisionKey(note.project_id, note.id, revision))) {
|
|
1367
|
+
fail("note_revisions", note.id, "revision_invalid");
|
|
1368
|
+
}
|
|
1369
|
+
}
|
|
1370
|
+
if (current.kind !== note.kind || current.title !== note.title || current.summary !== note.summary || current.content !== note.content || current.size_class !== note.size_class || current.pinned !== note.pinned || current.status !== note.status || current.supersedes_id !== note.supersedes_id || current.subject_key !== note.subject_key || current.content_hash !== noteContentHash(note.kind, note.title, note.summary, note.content)) {
|
|
1371
|
+
fail("note_revisions", note.id, "revision_invalid");
|
|
1372
|
+
}
|
|
1373
|
+
}
|
|
1374
|
+
const activeOperations = new Set;
|
|
1375
|
+
const outbox = source.outbox.map((row) => {
|
|
1376
|
+
requireIntegerAtLeast(row.id, 1, "index_outbox", String(row.id), "outbox_value_invalid");
|
|
1377
|
+
requireNonEmptyString(row.backend, "index_outbox", String(row.id), "outbox_value_invalid");
|
|
1378
|
+
requireString(row.operation, "index_outbox", String(row.id), "outbox_operation_invalid");
|
|
1379
|
+
requireString(row.project_id, "index_outbox", String(row.id), "project_id_invalid");
|
|
1380
|
+
requireNullableString(row.note_id, "index_outbox", String(row.id), "outbox_operation_invalid");
|
|
1381
|
+
requireNullableInteger(row.revision, "index_outbox", String(row.id), "outbox_operation_invalid");
|
|
1382
|
+
requireNullableString(row.content_hash, "index_outbox", String(row.id), "outbox_operation_invalid");
|
|
1383
|
+
if (row.state !== "pending" && row.state !== "leased" && row.state !== "succeeded" && row.state !== "dead") {
|
|
1384
|
+
fail("index_outbox", String(row.id), "outbox_value_invalid");
|
|
1385
|
+
}
|
|
1386
|
+
requireIntegerAtLeast(row.attempt_count, 0, "index_outbox", String(row.id), "outbox_value_invalid");
|
|
1387
|
+
requireInteger(row.available_at, "index_outbox", String(row.id), "timestamp_invalid");
|
|
1388
|
+
requireNullableString(row.lease_owner, "index_outbox", String(row.id), "outbox_value_invalid");
|
|
1389
|
+
requireNullableInteger(row.lease_expires_at, "index_outbox", String(row.id), "timestamp_invalid");
|
|
1390
|
+
requireNullableString(row.last_error_code, "index_outbox", String(row.id), "outbox_value_invalid");
|
|
1391
|
+
requireInteger(row.created_at, "index_outbox", String(row.id), "timestamp_invalid");
|
|
1392
|
+
requireNullableInteger(row.completed_at, "index_outbox", String(row.id), "timestamp_invalid");
|
|
1393
|
+
let contentHash = null;
|
|
1394
|
+
let state = row.state;
|
|
1395
|
+
let leaseOwner = row.lease_owner;
|
|
1396
|
+
let leaseExpiresAt = row.lease_expires_at;
|
|
1397
|
+
let completedAt = row.completed_at;
|
|
1398
|
+
if (row.operation === "upsert-note") {
|
|
1399
|
+
if (row.note_id === null || row.revision === null || row.content_hash === null) {
|
|
1400
|
+
fail("index_outbox", String(row.id), "outbox_operation_invalid");
|
|
1401
|
+
}
|
|
1402
|
+
requireNonEmptyString(row.note_id, "index_outbox", String(row.id), "outbox_operation_invalid");
|
|
1403
|
+
requireIntegerAtLeast(row.revision, 1, "index_outbox", String(row.id), "outbox_operation_invalid");
|
|
1404
|
+
requireHash(row.content_hash, "index_outbox", String(row.id), "hash_invalid");
|
|
1405
|
+
const revision = revisionMap.get(revisionKey(row.project_id, row.note_id, row.revision));
|
|
1406
|
+
if (!revision) {
|
|
1407
|
+
contentHash = row.content_hash;
|
|
1408
|
+
if (state === "pending" || state === "leased") {
|
|
1409
|
+
state = "succeeded";
|
|
1410
|
+
leaseOwner = null;
|
|
1411
|
+
leaseExpiresAt = null;
|
|
1412
|
+
completedAt = row.completed_at ?? row.created_at;
|
|
1413
|
+
}
|
|
1414
|
+
} else {
|
|
1415
|
+
const document = deriveDocument({
|
|
1416
|
+
projectID: revision.project_id,
|
|
1417
|
+
noteID: revision.note_id,
|
|
1418
|
+
revision: revision.revision,
|
|
1419
|
+
kind: revision.kind,
|
|
1420
|
+
title: revision.title,
|
|
1421
|
+
summary: revision.summary,
|
|
1422
|
+
content: revision.content
|
|
1423
|
+
});
|
|
1424
|
+
if (!document)
|
|
1425
|
+
fail("index_outbox", String(row.id), "derived_identity_unavailable");
|
|
1426
|
+
const legacyContentHash = legacyNoteContentHash(document.kind, document.title, document.summary, document.content);
|
|
1427
|
+
if (row.content_hash !== document.contentHash && row.content_hash !== legacyContentHash) {
|
|
1428
|
+
fail("index_outbox", String(row.id), "hash_invalid");
|
|
1429
|
+
}
|
|
1430
|
+
contentHash = document.contentHash;
|
|
1431
|
+
}
|
|
1432
|
+
} else if (row.operation === "delete-note") {
|
|
1433
|
+
if (row.note_id === null || row.revision === null) {
|
|
1434
|
+
fail("index_outbox", String(row.id), "outbox_operation_invalid");
|
|
1435
|
+
}
|
|
1436
|
+
requireNonEmptyString(row.note_id, "index_outbox", String(row.id), "outbox_operation_invalid");
|
|
1437
|
+
requireIntegerAtLeast(row.revision, 1, "index_outbox", String(row.id), "outbox_operation_invalid");
|
|
1438
|
+
if (row.content_hash !== null) {
|
|
1439
|
+
requireHash(row.content_hash, "index_outbox", String(row.id), "hash_invalid");
|
|
1440
|
+
}
|
|
1441
|
+
} else if (row.operation === "purge-project") {
|
|
1442
|
+
if (row.note_id !== null || row.revision !== null || row.content_hash !== null) {
|
|
1443
|
+
fail("index_outbox", String(row.id), "outbox_operation_invalid");
|
|
1444
|
+
}
|
|
1445
|
+
} else {
|
|
1446
|
+
fail("index_outbox", String(row.id), "outbox_operation_invalid");
|
|
1447
|
+
}
|
|
1448
|
+
if (state === "leased")
|
|
1449
|
+
state = "pending";
|
|
1450
|
+
leaseOwner = null;
|
|
1451
|
+
leaseExpiresAt = null;
|
|
1452
|
+
let generation = 0;
|
|
1453
|
+
if (state === "pending" || state === "leased") {
|
|
1454
|
+
let activeKey = hashTuple("outbox-active", 2, [
|
|
1455
|
+
row.backend,
|
|
1456
|
+
row.operation,
|
|
1457
|
+
row.project_id,
|
|
1458
|
+
row.note_id,
|
|
1459
|
+
row.revision,
|
|
1460
|
+
generation
|
|
1461
|
+
]);
|
|
1462
|
+
while (activeOperations.has(activeKey)) {
|
|
1463
|
+
generation++;
|
|
1464
|
+
activeKey = hashTuple("outbox-active", 2, [
|
|
1465
|
+
row.backend,
|
|
1466
|
+
row.operation,
|
|
1467
|
+
row.project_id,
|
|
1468
|
+
row.note_id,
|
|
1469
|
+
row.revision,
|
|
1470
|
+
generation
|
|
1471
|
+
]);
|
|
1472
|
+
}
|
|
1473
|
+
activeOperations.add(activeKey);
|
|
1474
|
+
}
|
|
1475
|
+
const operationKey = hashTuple("outbox-operation", 2, [
|
|
1476
|
+
row.backend,
|
|
1477
|
+
row.operation,
|
|
1478
|
+
row.project_id,
|
|
1479
|
+
row.note_id,
|
|
1480
|
+
row.revision,
|
|
1481
|
+
contentHash,
|
|
1482
|
+
generation
|
|
1483
|
+
]);
|
|
1484
|
+
return {
|
|
1485
|
+
...row,
|
|
1486
|
+
operation_key: operationKey,
|
|
1487
|
+
content_hash: contentHash,
|
|
1488
|
+
state,
|
|
1489
|
+
lease_owner: leaseOwner,
|
|
1490
|
+
lease_expires_at: leaseExpiresAt,
|
|
1491
|
+
completed_at: completedAt,
|
|
1492
|
+
generation,
|
|
1493
|
+
lease_generation: 0,
|
|
1494
|
+
fence: 0,
|
|
1495
|
+
heartbeat_at: null
|
|
1496
|
+
};
|
|
1497
|
+
});
|
|
1498
|
+
const migratedNotes = source.notes.map((note) => ({
|
|
1499
|
+
...note,
|
|
1500
|
+
content_hash: noteContentHash(note.kind, note.title, note.summary, note.content)
|
|
1501
|
+
}));
|
|
1502
|
+
return {
|
|
1503
|
+
projects: source.projects,
|
|
1504
|
+
notes: migratedNotes,
|
|
1505
|
+
edges,
|
|
1506
|
+
bindings,
|
|
1507
|
+
checkpoints,
|
|
1508
|
+
captures,
|
|
1509
|
+
provenance,
|
|
1510
|
+
revisions,
|
|
1511
|
+
outbox,
|
|
1512
|
+
bindingMap,
|
|
1513
|
+
captureMap
|
|
1514
|
+
};
|
|
1515
|
+
}
|
|
1516
|
+
function migrateCapture(row, bindingKey, notes, noteIDs) {
|
|
1517
|
+
requireHash(row.idempotency_key, "capture_events", row.idempotency_key, "capture_id_invalid");
|
|
1518
|
+
if (!CAPTURE_SCHEMAS.has(row.contract))
|
|
1519
|
+
fail("capture_events", row.idempotency_key, "capture_contract_invalid");
|
|
1520
|
+
requireString(row.project_id, "capture_events", row.idempotency_key, "project_id_invalid");
|
|
1521
|
+
requireString(row.event_kind, "capture_events", row.idempotency_key, "capture_identity_invalid");
|
|
1522
|
+
requireNonEmptyString(row.source_session_id, "capture_events", row.idempotency_key, "capture_identity_invalid");
|
|
1523
|
+
requireNullableString(row.source_message_id, "capture_events", row.idempotency_key, "capture_identity_invalid");
|
|
1524
|
+
requireNullableInteger(row.source_ordinal, "capture_events", row.idempotency_key, "capture_identity_invalid");
|
|
1525
|
+
requireNullableString(row.source_tool_call_id, "capture_events", row.idempotency_key, "capture_identity_invalid");
|
|
1526
|
+
requireString(row.redaction_version, "capture_events", row.idempotency_key, "capture_value_invalid");
|
|
1527
|
+
if (!new Set(["pending", "shadowed", "review", "materialized", "duplicate", "ignored", "rejected", "quarantined", "failed", "dead"]).has(row.state)) {
|
|
1528
|
+
fail("capture_events", row.idempotency_key, "capture_value_invalid");
|
|
1529
|
+
}
|
|
1530
|
+
requireIntegerAtLeast(row.attempt_count, 0, "capture_events", row.idempotency_key, "capture_value_invalid");
|
|
1531
|
+
if (row.note_id !== null && notes.get(noteKey(row.project_id, row.note_id)) === undefined) {
|
|
1532
|
+
if (noteIDs.has(row.note_id)) {
|
|
1533
|
+
fail("capture_events", row.idempotency_key, "cross_project_reference");
|
|
1534
|
+
}
|
|
1535
|
+
}
|
|
1536
|
+
requireNullableString(row.note_id, "capture_events", row.idempotency_key, "capture_value_invalid");
|
|
1537
|
+
requireNullableString(row.last_error_code, "capture_events", row.idempotency_key, "capture_value_invalid");
|
|
1538
|
+
requireIntegerAtLeast(row.generation, 0, "capture_events", row.idempotency_key, "capture_value_invalid");
|
|
1539
|
+
requireInteger(row.created_at, "capture_events", row.idempotency_key, "timestamp_invalid");
|
|
1540
|
+
requireInteger(row.updated_at, "capture_events", row.idempotency_key, "timestamp_invalid");
|
|
1541
|
+
requireNullableInteger(row.processed_at, "capture_events", row.idempotency_key, "timestamp_invalid");
|
|
1542
|
+
requireNullableString(row.payload_json, "capture_events", row.idempotency_key, "payload_value_invalid");
|
|
1543
|
+
requireNullableString(row.payload_hash, "capture_events", row.idempotency_key, "payload_value_invalid");
|
|
1544
|
+
if (row.payload_json === null) {
|
|
1545
|
+
if (row.payload_hash !== null)
|
|
1546
|
+
fail("capture_events", row.idempotency_key, "payload_identity_unavailable");
|
|
1547
|
+
let key;
|
|
1548
|
+
try {
|
|
1549
|
+
assertLegacyCaptureKey(row);
|
|
1550
|
+
key = captureKeyFromColumns(row, bindingKey);
|
|
1551
|
+
} catch {
|
|
1552
|
+
fail("capture_events", row.idempotency_key, "capture_identity_invalid");
|
|
1553
|
+
}
|
|
1554
|
+
return { ...row, idempotency_key: key, contract: CAPTURE_SCHEMA, binding_key: bindingKey };
|
|
1555
|
+
}
|
|
1556
|
+
if (row.payload_hash === null)
|
|
1557
|
+
fail("capture_events", row.idempotency_key, "payload_identity_unavailable");
|
|
1558
|
+
const legacyPayloadHash = sha256(row.payload_json);
|
|
1559
|
+
const v2PayloadHash = hashTuple("capture-payload", 2, [row.payload_json]);
|
|
1560
|
+
if (row.payload_hash !== legacyPayloadHash && row.payload_hash !== v2PayloadHash) {
|
|
1561
|
+
fail("capture_events", row.idempotency_key, "payload_identity_mismatch");
|
|
1562
|
+
}
|
|
1563
|
+
let raw;
|
|
1564
|
+
try {
|
|
1565
|
+
const value = JSON.parse(row.payload_json);
|
|
1566
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
1567
|
+
throw new Error("not an object");
|
|
1568
|
+
raw = value;
|
|
1569
|
+
} catch {
|
|
1570
|
+
fail("capture_events", row.idempotency_key, "payload_invalid");
|
|
1571
|
+
}
|
|
1572
|
+
if (raw.schema !== row.contract || raw.idempotencyKey !== row.idempotency_key) {
|
|
1573
|
+
fail("capture_events", row.idempotency_key, "payload_identity_mismatch");
|
|
1574
|
+
}
|
|
1575
|
+
if (raw.projectID !== row.project_id || raw.bindingKey !== row.binding_key || raw.kind !== row.event_kind) {
|
|
1576
|
+
fail("capture_events", row.idempotency_key, "cross_project_reference");
|
|
1577
|
+
}
|
|
1578
|
+
const source = raw.source;
|
|
1579
|
+
if (!source || typeof source !== "object" || Array.isArray(source)) {
|
|
1580
|
+
fail("capture_events", row.idempotency_key, "capture_identity_invalid");
|
|
1581
|
+
}
|
|
1582
|
+
let nextKey;
|
|
1583
|
+
try {
|
|
1584
|
+
nextKey = captureKeyFromPayload(row.event_kind, bindingKey, source, raw.signal);
|
|
1585
|
+
} catch {
|
|
1586
|
+
fail("capture_events", row.idempotency_key, "capture_identity_invalid");
|
|
1587
|
+
}
|
|
1588
|
+
const candidate = {
|
|
1589
|
+
...raw,
|
|
1590
|
+
schema: CAPTURE_SCHEMA,
|
|
1591
|
+
idempotencyKey: nextKey,
|
|
1592
|
+
projectID: row.project_id,
|
|
1593
|
+
bindingKey
|
|
1594
|
+
};
|
|
1595
|
+
let parsed;
|
|
1596
|
+
try {
|
|
1597
|
+
parsed = parseCaptureEvent(candidate);
|
|
1598
|
+
} catch {
|
|
1599
|
+
fail("capture_events", row.idempotency_key, "capture_identity_invalid");
|
|
1600
|
+
}
|
|
1601
|
+
if (parsed.kind !== row.event_kind || parsed.source.sessionID !== row.source_session_id || (parsed.source.messageID ?? null) !== row.source_message_id || (parsed.source.ordinal ?? null) !== row.source_ordinal || (parsed.source.toolCallID ?? null) !== row.source_tool_call_id) {
|
|
1602
|
+
fail("capture_events", row.idempotency_key, "capture_identity_mismatch");
|
|
1603
|
+
}
|
|
1604
|
+
const payload = JSON.stringify(parsed);
|
|
1605
|
+
return {
|
|
1606
|
+
...row,
|
|
1607
|
+
idempotency_key: nextKey,
|
|
1608
|
+
contract: CAPTURE_SCHEMA,
|
|
1609
|
+
binding_key: bindingKey,
|
|
1610
|
+
payload_json: payload,
|
|
1611
|
+
payload_hash: capturePayloadHash(parsed)
|
|
1612
|
+
};
|
|
1613
|
+
}
|
|
1614
|
+
function captureKeyFromColumns(row, bindingKey) {
|
|
1615
|
+
if (row.event_kind === "user-candidate") {
|
|
1616
|
+
requireSourceMessage(row.source_message_id, "capture_events", row.idempotency_key);
|
|
1617
|
+
if (row.source_ordinal !== null || row.source_tool_call_id !== null) {
|
|
1618
|
+
fail("capture_events", row.idempotency_key, "capture_identity_invalid");
|
|
1619
|
+
}
|
|
1620
|
+
return captureIdempotencyKey({
|
|
1621
|
+
kind: "user",
|
|
1622
|
+
bindingKey,
|
|
1623
|
+
sessionID: row.source_session_id,
|
|
1624
|
+
messageID: row.source_message_id
|
|
1625
|
+
});
|
|
1626
|
+
}
|
|
1627
|
+
if (row.event_kind === "assistant-candidate") {
|
|
1628
|
+
requireSourceMessage(row.source_message_id, "capture_events", row.idempotency_key);
|
|
1629
|
+
if (row.source_ordinal === null || row.source_tool_call_id !== null) {
|
|
1630
|
+
fail("capture_events", row.idempotency_key, "capture_identity_invalid");
|
|
1631
|
+
}
|
|
1632
|
+
return captureIdempotencyKey({
|
|
1633
|
+
kind: "assistant",
|
|
1634
|
+
bindingKey,
|
|
1635
|
+
sessionID: row.source_session_id,
|
|
1636
|
+
assistantMessageID: row.source_message_id,
|
|
1637
|
+
ordinal: row.source_ordinal
|
|
1638
|
+
});
|
|
1639
|
+
}
|
|
1640
|
+
if (row.event_kind === "session-summary") {
|
|
1641
|
+
requireSourceMessage(row.source_message_id, "capture_events", row.idempotency_key);
|
|
1642
|
+
if (row.source_ordinal !== null || row.source_tool_call_id !== null) {
|
|
1643
|
+
fail("capture_events", row.idempotency_key, "capture_identity_invalid");
|
|
1644
|
+
}
|
|
1645
|
+
return captureIdempotencyKey({
|
|
1646
|
+
kind: "summary",
|
|
1647
|
+
bindingKey,
|
|
1648
|
+
sessionID: row.source_session_id,
|
|
1649
|
+
checkpointMessageID: row.source_message_id
|
|
1650
|
+
});
|
|
1651
|
+
}
|
|
1652
|
+
if (row.event_kind === "tool-signal") {
|
|
1653
|
+
requireSourceMessage(row.source_message_id, "capture_events", row.idempotency_key);
|
|
1654
|
+
if (row.source_ordinal !== null || !row.source_tool_call_id) {
|
|
1655
|
+
fail("capture_events", row.idempotency_key, "capture_identity_invalid");
|
|
1656
|
+
}
|
|
1657
|
+
const status = legacyToolStatus(row);
|
|
1658
|
+
return captureIdempotencyKey({
|
|
1659
|
+
kind: "tool",
|
|
1660
|
+
bindingKey,
|
|
1661
|
+
sessionID: row.source_session_id,
|
|
1662
|
+
assistantMessageID: row.source_message_id,
|
|
1663
|
+
toolCallID: row.source_tool_call_id,
|
|
1664
|
+
terminalStatus: status
|
|
1665
|
+
});
|
|
1666
|
+
}
|
|
1667
|
+
fail("capture_events", row.idempotency_key, "capture_identity_invalid");
|
|
1668
|
+
}
|
|
1669
|
+
function assertLegacyCaptureKey(row) {
|
|
1670
|
+
let expected;
|
|
1671
|
+
if (row.event_kind === "user-candidate") {
|
|
1672
|
+
requireSourceMessage(row.source_message_id, "capture_events", row.idempotency_key);
|
|
1673
|
+
expected = legacyCaptureKey([
|
|
1674
|
+
"user",
|
|
1675
|
+
row.binding_key,
|
|
1676
|
+
row.source_session_id,
|
|
1677
|
+
row.source_message_id
|
|
1678
|
+
]);
|
|
1679
|
+
} else if (row.event_kind === "assistant-candidate") {
|
|
1680
|
+
requireSourceMessage(row.source_message_id, "capture_events", row.idempotency_key);
|
|
1681
|
+
if (row.source_ordinal === null)
|
|
1682
|
+
fail("capture_events", row.idempotency_key, "capture_identity_invalid");
|
|
1683
|
+
expected = legacyCaptureKey([
|
|
1684
|
+
"assistant",
|
|
1685
|
+
row.binding_key,
|
|
1686
|
+
row.source_session_id,
|
|
1687
|
+
row.source_message_id,
|
|
1688
|
+
String(row.source_ordinal)
|
|
1689
|
+
]);
|
|
1690
|
+
} else if (row.event_kind === "session-summary") {
|
|
1691
|
+
requireSourceMessage(row.source_message_id, "capture_events", row.idempotency_key);
|
|
1692
|
+
expected = legacyCaptureKey([
|
|
1693
|
+
"summary",
|
|
1694
|
+
row.binding_key,
|
|
1695
|
+
row.source_session_id,
|
|
1696
|
+
row.source_message_id
|
|
1697
|
+
]);
|
|
1698
|
+
} else if (row.event_kind === "tool-signal") {
|
|
1699
|
+
legacyToolStatus(row);
|
|
1700
|
+
return;
|
|
1701
|
+
} else {
|
|
1702
|
+
fail("capture_events", row.idempotency_key, "capture_identity_invalid");
|
|
1703
|
+
}
|
|
1704
|
+
if (row.idempotency_key !== expected) {
|
|
1705
|
+
fail("capture_events", row.idempotency_key, "capture_identity_mismatch");
|
|
1706
|
+
}
|
|
1707
|
+
}
|
|
1708
|
+
function legacyToolStatus(row) {
|
|
1709
|
+
requireSourceMessage(row.source_message_id, "capture_events", row.idempotency_key);
|
|
1710
|
+
if (!row.source_tool_call_id)
|
|
1711
|
+
fail("capture_events", row.idempotency_key, "capture_identity_invalid");
|
|
1712
|
+
const completed = legacyCaptureKey([
|
|
1713
|
+
"tool",
|
|
1714
|
+
row.binding_key,
|
|
1715
|
+
row.source_session_id,
|
|
1716
|
+
row.source_message_id,
|
|
1717
|
+
row.source_tool_call_id,
|
|
1718
|
+
"completed"
|
|
1719
|
+
]);
|
|
1720
|
+
const errored = legacyCaptureKey([
|
|
1721
|
+
"tool",
|
|
1722
|
+
row.binding_key,
|
|
1723
|
+
row.source_session_id,
|
|
1724
|
+
row.source_message_id,
|
|
1725
|
+
row.source_tool_call_id,
|
|
1726
|
+
"error"
|
|
1727
|
+
]);
|
|
1728
|
+
if (row.idempotency_key === completed)
|
|
1729
|
+
return "completed";
|
|
1730
|
+
if (row.idempotency_key === errored)
|
|
1731
|
+
return "error";
|
|
1732
|
+
fail("capture_events", row.idempotency_key, "capture_identity_mismatch");
|
|
1733
|
+
}
|
|
1734
|
+
function legacyCaptureKey(fields) {
|
|
1735
|
+
return sha256(["capture/1", ...fields].join("\x00"));
|
|
1736
|
+
}
|
|
1737
|
+
function captureKeyFromPayload(eventKind, bindingKey, source, signal) {
|
|
1738
|
+
const sessionID = source.sessionID;
|
|
1739
|
+
const messageID = source.messageID;
|
|
1740
|
+
if (typeof sessionID !== "string" || !sessionID)
|
|
1741
|
+
throw new Error("invalid source");
|
|
1742
|
+
if (eventKind === "user-candidate") {
|
|
1743
|
+
if (typeof messageID !== "string" || !messageID || hasOwn(source, "ordinal") || hasOwn(source, "toolCallID")) {
|
|
1744
|
+
throw new Error("invalid source");
|
|
1745
|
+
}
|
|
1746
|
+
return captureIdempotencyKey({ kind: "user", bindingKey, sessionID, messageID });
|
|
1747
|
+
}
|
|
1748
|
+
if (eventKind === "assistant-candidate") {
|
|
1749
|
+
if (typeof messageID !== "string" || !messageID || typeof source.ordinal !== "number" || hasOwn(source, "toolCallID")) {
|
|
1750
|
+
throw new Error("invalid source");
|
|
1751
|
+
}
|
|
1752
|
+
return captureIdempotencyKey({
|
|
1753
|
+
kind: "assistant",
|
|
1754
|
+
bindingKey,
|
|
1755
|
+
sessionID,
|
|
1756
|
+
assistantMessageID: messageID,
|
|
1757
|
+
ordinal: source.ordinal
|
|
1758
|
+
});
|
|
1759
|
+
}
|
|
1760
|
+
if (eventKind === "session-summary") {
|
|
1761
|
+
if (typeof messageID !== "string" || !messageID || hasOwn(source, "ordinal") || hasOwn(source, "toolCallID")) {
|
|
1762
|
+
throw new Error("invalid source");
|
|
1763
|
+
}
|
|
1764
|
+
return captureIdempotencyKey({ kind: "summary", bindingKey, sessionID, checkpointMessageID: messageID });
|
|
1765
|
+
}
|
|
1766
|
+
if (eventKind === "tool-signal") {
|
|
1767
|
+
const status = signal && typeof signal === "object" ? signal.status : undefined;
|
|
1768
|
+
const toolCallID = source.toolCallID;
|
|
1769
|
+
if (typeof messageID !== "string" || !messageID || typeof toolCallID !== "string" || !toolCallID || hasOwn(source, "ordinal") || status !== "completed" && status !== "error") {
|
|
1770
|
+
throw new Error("invalid source");
|
|
1771
|
+
}
|
|
1772
|
+
return captureIdempotencyKey({
|
|
1773
|
+
kind: "tool",
|
|
1774
|
+
bindingKey,
|
|
1775
|
+
sessionID,
|
|
1776
|
+
assistantMessageID: messageID,
|
|
1777
|
+
toolCallID,
|
|
1778
|
+
terminalStatus: status
|
|
1779
|
+
});
|
|
1780
|
+
}
|
|
1781
|
+
throw new Error("invalid event kind");
|
|
1782
|
+
}
|
|
1783
|
+
function replaceWithV11(db, rows) {
|
|
1784
|
+
db.exec(`
|
|
1785
|
+
DROP TRIGGER IF EXISTS notes_fts_ai;
|
|
1786
|
+
DROP TRIGGER IF EXISTS notes_fts_ad;
|
|
1787
|
+
DROP TRIGGER IF EXISTS notes_fts_au;
|
|
1788
|
+
DROP TABLE IF EXISTS notes_fts;
|
|
1789
|
+
DROP TABLE IF EXISTS index_outbox;
|
|
1790
|
+
DROP TABLE IF EXISTS note_revisions;
|
|
1791
|
+
DROP TABLE IF EXISTS note_provenance;
|
|
1792
|
+
DROP TABLE IF EXISTS capture_events;
|
|
1793
|
+
DROP TABLE IF EXISTS capture_checkpoints;
|
|
1794
|
+
DROP TABLE IF EXISTS project_bindings;
|
|
1795
|
+
DROP TABLE IF EXISTS note_edges;
|
|
1796
|
+
DROP TABLE IF EXISTS notes;
|
|
1797
|
+
DROP TABLE IF EXISTS projects;
|
|
1798
|
+
DROP TABLE IF EXISTS schema_state;
|
|
1799
|
+
DROP TABLE IF EXISTS agz_meta;
|
|
1800
|
+
`);
|
|
1801
|
+
db.exec(SCHEMA_V11_TABLES);
|
|
1802
|
+
db.exec(FTS_V9);
|
|
1803
|
+
for (const row of rows.projects) {
|
|
1804
|
+
insertRow("projects", row.id, () => {
|
|
1805
|
+
db.query("INSERT INTO projects (id, name, normalized_name, created_at, updated_at) VALUES (?, ?, ?, ?, ?)").run(row.id, row.name, row.normalized_name, row.created_at, row.updated_at);
|
|
1806
|
+
});
|
|
1807
|
+
}
|
|
1808
|
+
for (const row of rows.notes) {
|
|
1809
|
+
insertRow("notes", row.id, () => {
|
|
1810
|
+
db.query(`INSERT INTO notes
|
|
1811
|
+
(id, project_id, kind, title, summary, content, size_class, pinned, status,
|
|
1812
|
+
supersedes_id, current_revision, subject_key, content_hash, created_at, updated_at)
|
|
1813
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(row.id, row.project_id, row.kind, row.title, row.summary, row.content, row.size_class, row.pinned, row.status, row.supersedes_id, row.current_revision, row.subject_key, row.content_hash, row.created_at, row.updated_at);
|
|
1814
|
+
});
|
|
1815
|
+
}
|
|
1816
|
+
for (const row of rows.bindings) {
|
|
1817
|
+
insertRow("project_bindings", row.binding_key, () => {
|
|
1818
|
+
db.query(`INSERT INTO project_bindings
|
|
1819
|
+
(binding_key, project_id, source, source_project_id, workspace_id,
|
|
1820
|
+
canonical_path_hash, created_at, updated_at)
|
|
1821
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`).run(row.binding_key, row.project_id, row.source, row.source_project_id, row.workspace_id, row.canonical_path_hash, row.created_at, row.updated_at);
|
|
1822
|
+
});
|
|
1823
|
+
}
|
|
1824
|
+
for (const row of rows.checkpoints) {
|
|
1825
|
+
insertRow("capture_checkpoints", row.session_id, () => {
|
|
1826
|
+
db.query(`INSERT INTO capture_checkpoints
|
|
1827
|
+
(session_id, binding_key, project_id, state, last_message_id,
|
|
1828
|
+
last_reconciled_at, next_reconcile_at, failure_count, lease_owner,
|
|
1829
|
+
lease_expires_at, created_at, updated_at)
|
|
1830
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(row.session_id, row.binding_key, row.project_id, row.state, row.last_message_id, row.last_reconciled_at, row.next_reconcile_at, row.failure_count, row.lease_owner, row.lease_expires_at, row.created_at, row.updated_at);
|
|
1831
|
+
});
|
|
1832
|
+
}
|
|
1833
|
+
for (const row of rows.captures) {
|
|
1834
|
+
insertRow("capture_events", row.idempotency_key, () => {
|
|
1835
|
+
db.query(`INSERT INTO capture_events
|
|
1836
|
+
(idempotency_key, contract, project_id, binding_key, event_kind,
|
|
1837
|
+
source_session_id, source_message_id, source_ordinal, source_tool_call_id,
|
|
1838
|
+
payload_json, payload_hash, redaction_version, state, attempt_count,
|
|
1839
|
+
note_id, last_error_code, generation, created_at, updated_at, processed_at)
|
|
1840
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(row.idempotency_key, row.contract, row.project_id, row.binding_key, row.event_kind, row.source_session_id, row.source_message_id, row.source_ordinal, row.source_tool_call_id, row.payload_json, row.payload_hash, row.redaction_version, row.state, row.attempt_count, row.note_id, row.last_error_code, row.generation, row.created_at, row.updated_at, row.processed_at);
|
|
1841
|
+
});
|
|
1842
|
+
}
|
|
1843
|
+
for (const row of rows.provenance) {
|
|
1844
|
+
insertRow("note_provenance", row.id, () => {
|
|
1845
|
+
db.query(`INSERT INTO note_provenance
|
|
1846
|
+
(id, project_id, note_id, source_type, capture_event_id, source_session_id,
|
|
1847
|
+
source_message_id, source_ordinal, source_tool_call_id, redaction_version,
|
|
1848
|
+
extractor_version, confidence, created_at)
|
|
1849
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(row.id, row.project_id, row.note_id, row.source_type, row.capture_event_id, row.source_session_id, row.source_message_id, row.source_ordinal, row.source_tool_call_id, row.redaction_version, row.extractor_version, row.confidence, row.created_at);
|
|
1850
|
+
});
|
|
1851
|
+
}
|
|
1852
|
+
for (const row of rows.revisions) {
|
|
1853
|
+
insertRow("note_revisions", row.note_id, () => {
|
|
1854
|
+
db.query(`INSERT INTO note_revisions
|
|
1855
|
+
(project_id, note_id, revision, kind, title, summary, content, size_class,
|
|
1856
|
+
pinned, status, supersedes_id, subject_key, content_hash, provenance_id, created_at)
|
|
1857
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(row.project_id, row.note_id, row.revision, row.kind, row.title, row.summary, row.content, row.size_class, row.pinned, row.status, row.supersedes_id, row.subject_key, row.content_hash, row.provenance_id, row.created_at);
|
|
1858
|
+
});
|
|
1859
|
+
}
|
|
1860
|
+
for (const row of rows.edges) {
|
|
1861
|
+
insertRow("note_edges", row.id, () => {
|
|
1862
|
+
db.query("INSERT INTO note_edges (id, project_id, source_id, target_id, predicate, created_at) VALUES (?, ?, ?, ?, ?, ?)").run(row.id, row.project_id, row.source_id, row.target_id, row.predicate, row.created_at);
|
|
1863
|
+
});
|
|
1864
|
+
}
|
|
1865
|
+
for (const row of rows.outbox) {
|
|
1866
|
+
insertRow("index_outbox", String(row.id), () => {
|
|
1867
|
+
db.query(`INSERT INTO index_outbox
|
|
1868
|
+
(id, backend, operation_key, operation, project_id, note_id, revision,
|
|
1869
|
+
content_hash, generation, lease_generation, fence, state, attempt_count,
|
|
1870
|
+
available_at, lease_owner, lease_expires_at, heartbeat_at, last_error_code,
|
|
1871
|
+
created_at, completed_at)
|
|
1872
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(row.id, row.backend, row.operation_key, row.operation, row.project_id, row.note_id, row.revision, row.content_hash, row.generation, row.lease_generation, row.fence, row.state, row.attempt_count, row.available_at, row.lease_owner, row.lease_expires_at, row.heartbeat_at, row.last_error_code, row.created_at, row.completed_at);
|
|
1873
|
+
});
|
|
1874
|
+
}
|
|
1875
|
+
rebuildFts(db);
|
|
1876
|
+
db.query("INSERT INTO schema_state(version) VALUES (11)").run();
|
|
1877
|
+
insertV11Identity(db);
|
|
1878
|
+
}
|
|
1879
|
+
function revisionKey(projectID, noteID, revision) {
|
|
1880
|
+
return `${projectID.length}:${projectID}:${noteID.length}:${noteID}:${revision}`;
|
|
1881
|
+
}
|
|
1882
|
+
function noteKey(projectID, noteID) {
|
|
1883
|
+
return `${projectID.length}:${projectID}:${noteID.length}:${noteID}`;
|
|
1884
|
+
}
|
|
1885
|
+
function requireTable(db, table) {
|
|
1886
|
+
if (!hasTable2(db, table))
|
|
1887
|
+
fail(table, "schema", "source_schema");
|
|
1888
|
+
}
|
|
1889
|
+
function hasTable2(db, table) {
|
|
1890
|
+
const row = db.query("SELECT COUNT(*) AS count FROM sqlite_master WHERE type IN ('table','view') AND name = ?").get(table);
|
|
1891
|
+
return row.count > 0;
|
|
1892
|
+
}
|
|
1893
|
+
function selectRows(db, table, sql) {
|
|
1894
|
+
try {
|
|
1895
|
+
return db.query(sql).all();
|
|
1896
|
+
} catch {
|
|
1897
|
+
fail(table, "schema", "source_schema");
|
|
1898
|
+
}
|
|
1899
|
+
}
|
|
1900
|
+
function insertRow(table, identity, insert) {
|
|
1901
|
+
try {
|
|
1902
|
+
insert();
|
|
1903
|
+
} catch {
|
|
1904
|
+
fail(table, identity, "row_constraint");
|
|
1905
|
+
}
|
|
1906
|
+
}
|
|
1907
|
+
function requireString(value, table, identity, code) {
|
|
1908
|
+
if (typeof value !== "string")
|
|
1909
|
+
fail(table, identity, code);
|
|
1910
|
+
}
|
|
1911
|
+
function requireNonEmptyString(value, table, identity, code) {
|
|
1912
|
+
requireString(value, table, identity, code);
|
|
1913
|
+
if (value.length === 0)
|
|
1914
|
+
fail(table, identity, code);
|
|
1915
|
+
}
|
|
1916
|
+
function requireNullableString(value, table, identity, code) {
|
|
1917
|
+
if (value !== null && typeof value !== "string")
|
|
1918
|
+
fail(table, identity, code);
|
|
1919
|
+
}
|
|
1920
|
+
function requireInteger(value, table, identity, code) {
|
|
1921
|
+
if (!Number.isSafeInteger(value))
|
|
1922
|
+
fail(table, identity, code);
|
|
1923
|
+
}
|
|
1924
|
+
function requireNullableInteger(value, table, identity, code) {
|
|
1925
|
+
if (value !== null && !Number.isSafeInteger(value))
|
|
1926
|
+
fail(table, identity, code);
|
|
1927
|
+
}
|
|
1928
|
+
function requireIntegerAtLeast(value, minimum, table, identity, code) {
|
|
1929
|
+
if (typeof value !== "number" || !Number.isSafeInteger(value) || value < minimum) {
|
|
1930
|
+
fail(table, identity, code);
|
|
1931
|
+
}
|
|
1932
|
+
}
|
|
1933
|
+
function requireHash(value, table, identity, code) {
|
|
1934
|
+
if (typeof value !== "string" || !/^[0-9a-f]{64}$/i.test(value))
|
|
1935
|
+
fail(table, identity, code);
|
|
1936
|
+
}
|
|
1937
|
+
function requireKind(value, table, identity) {
|
|
1938
|
+
if (typeof value !== "string" || !KINDS_SET.has(value))
|
|
1939
|
+
fail(table, identity, "kind_invalid");
|
|
1940
|
+
}
|
|
1941
|
+
function requireSourceMessage(value, table, identity) {
|
|
1942
|
+
if (typeof value !== "string" || !value)
|
|
1943
|
+
fail(table, identity, "capture_identity_invalid");
|
|
1944
|
+
}
|
|
1945
|
+
function hasOwn(value, key) {
|
|
1946
|
+
return Object.prototype.hasOwnProperty.call(value, key);
|
|
1947
|
+
}
|
|
1948
|
+
function legacyNoteContentHash(kind, title, summary, content) {
|
|
1949
|
+
return createHash3("sha256").update(`${kind}\x00${title}\x00${summary}\x00${content}`, "utf8").digest("hex");
|
|
1950
|
+
}
|
|
1951
|
+
function sha256(value) {
|
|
1952
|
+
return createHash3("sha256").update(value, "utf8").digest("hex");
|
|
1953
|
+
}
|
|
1954
|
+
function capturePayloadHash(event) {
|
|
1955
|
+
const canonical = structuredClone(event);
|
|
1956
|
+
canonical.source.observedAt = 0;
|
|
1957
|
+
return hashTuple("capture-payload", 2, [JSON.stringify(canonical)]);
|
|
1958
|
+
}
|
|
1959
|
+
function fail(table, identity, code) {
|
|
1960
|
+
const safeIdentity = typeof identity === "string" && /^[A-Za-z0-9._:-]{1,120}$/.test(identity) ? identity : "redacted";
|
|
1961
|
+
throw new Error(`schema_v11_migration_${code} table=${table} row=${safeIdentity}`);
|
|
1962
|
+
}
|
|
1963
|
+
|
|
1964
|
+
// src/db/legacy-health.ts
|
|
1965
|
+
var V2_TABLES = [
|
|
1966
|
+
"memory_items",
|
|
1967
|
+
"memory_versions",
|
|
1968
|
+
"memory_identities",
|
|
1969
|
+
"document_sources",
|
|
1970
|
+
"document_chunks",
|
|
1971
|
+
"memories",
|
|
1972
|
+
"memory_links",
|
|
1973
|
+
"memory_edges",
|
|
1974
|
+
"memory_associations"
|
|
1975
|
+
];
|
|
1976
|
+
var PRE_V8_OBJECTS = new Set([
|
|
1977
|
+
"index:note_edges_source_idx",
|
|
1978
|
+
"index:note_edges_target_idx",
|
|
1979
|
+
"index:notes_project_idx",
|
|
1980
|
+
"table:note_edges",
|
|
1981
|
+
"table:notes",
|
|
1982
|
+
"table:notes_fts",
|
|
1983
|
+
"table:notes_fts_config",
|
|
1984
|
+
"table:notes_fts_content",
|
|
1985
|
+
"table:notes_fts_data",
|
|
1986
|
+
"table:notes_fts_docsize",
|
|
1987
|
+
"table:notes_fts_idx",
|
|
1988
|
+
"table:projects",
|
|
1989
|
+
"table:schema_state",
|
|
1990
|
+
...V2_TABLES.map((table) => `table:${table}`)
|
|
1991
|
+
]);
|
|
1992
|
+
var PRE_V8_COLUMNS = {
|
|
1993
|
+
schema_state: ["version"],
|
|
1994
|
+
projects: ["id", "name", "normalized_name", "created_at", "updated_at"],
|
|
1995
|
+
notes: [
|
|
1996
|
+
"id",
|
|
1997
|
+
"project_id",
|
|
1998
|
+
"kind",
|
|
1999
|
+
"title",
|
|
2000
|
+
"summary",
|
|
2001
|
+
"content",
|
|
2002
|
+
"size_class",
|
|
2003
|
+
"status",
|
|
2004
|
+
"supersedes_id",
|
|
2005
|
+
"created_at",
|
|
2006
|
+
"updated_at"
|
|
2007
|
+
],
|
|
2008
|
+
note_edges: ["id", "project_id", "source_id", "target_id", "predicate", "created_at"],
|
|
2009
|
+
notes_fts: ["id", "title", "summary", "content"]
|
|
2010
|
+
};
|
|
2011
|
+
var V2_COLUMNS = {
|
|
2012
|
+
memory_items: [
|
|
2013
|
+
"id",
|
|
2014
|
+
"identity_id",
|
|
2015
|
+
"subject_key",
|
|
2016
|
+
"kind",
|
|
2017
|
+
"lifecycle_state",
|
|
2018
|
+
"current_version_id",
|
|
2019
|
+
"created_at",
|
|
2020
|
+
"updated_at"
|
|
2021
|
+
],
|
|
2022
|
+
memory_versions: ["id", "summary", "content"],
|
|
2023
|
+
memory_identities: ["id", "project_id"]
|
|
2024
|
+
};
|
|
2025
|
+
var V8_OBJECTS = new Set([
|
|
2026
|
+
"table:note_edges",
|
|
2027
|
+
"table:notes",
|
|
2028
|
+
"table:notes_fts",
|
|
2029
|
+
"table:notes_fts_config",
|
|
2030
|
+
"table:notes_fts_content",
|
|
2031
|
+
"table:notes_fts_data",
|
|
2032
|
+
"table:notes_fts_docsize",
|
|
2033
|
+
"table:notes_fts_idx",
|
|
2034
|
+
"table:projects",
|
|
2035
|
+
"table:schema_state"
|
|
2036
|
+
]);
|
|
2037
|
+
var V9_V10_OBJECTS = new Set([
|
|
2038
|
+
"index:capture_checkpoints_due_idx",
|
|
2039
|
+
"index:capture_events_note_idx",
|
|
2040
|
+
"index:capture_events_session_idx",
|
|
2041
|
+
"index:capture_events_state_idx",
|
|
2042
|
+
"index:index_outbox_due_idx",
|
|
2043
|
+
"index:note_edges_source_idx",
|
|
2044
|
+
"index:note_edges_target_idx",
|
|
2045
|
+
"index:notes_active_subject_idx",
|
|
2046
|
+
"index:notes_project_idx",
|
|
2047
|
+
"table:capture_checkpoints",
|
|
2048
|
+
"table:capture_events",
|
|
2049
|
+
"table:index_outbox",
|
|
2050
|
+
"table:note_edges",
|
|
2051
|
+
"table:note_provenance",
|
|
2052
|
+
"table:note_revisions",
|
|
2053
|
+
"table:notes",
|
|
2054
|
+
"table:notes_fts",
|
|
2055
|
+
"table:notes_fts_config",
|
|
2056
|
+
"table:notes_fts_data",
|
|
2057
|
+
"table:notes_fts_docsize",
|
|
2058
|
+
"table:notes_fts_idx",
|
|
2059
|
+
"table:project_bindings",
|
|
2060
|
+
"table:projects",
|
|
2061
|
+
"table:schema_state",
|
|
2062
|
+
"trigger:notes_fts_ad",
|
|
2063
|
+
"trigger:notes_fts_ai",
|
|
2064
|
+
"trigger:notes_fts_au"
|
|
2065
|
+
]);
|
|
2066
|
+
var V9_V10_COLUMNS = {
|
|
2067
|
+
schema_state: ["version"],
|
|
2068
|
+
projects: ["id", "name", "normalized_name", "created_at", "updated_at"],
|
|
2069
|
+
notes: [
|
|
2070
|
+
"id",
|
|
2071
|
+
"project_id",
|
|
2072
|
+
"kind",
|
|
2073
|
+
"title",
|
|
2074
|
+
"summary",
|
|
2075
|
+
"content",
|
|
2076
|
+
"size_class",
|
|
2077
|
+
"pinned",
|
|
2078
|
+
"status",
|
|
2079
|
+
"supersedes_id",
|
|
2080
|
+
"current_revision",
|
|
2081
|
+
"subject_key",
|
|
2082
|
+
"content_hash",
|
|
2083
|
+
"created_at",
|
|
2084
|
+
"updated_at"
|
|
2085
|
+
],
|
|
2086
|
+
note_edges: ["id", "project_id", "source_id", "target_id", "predicate", "created_at"],
|
|
2087
|
+
project_bindings: [
|
|
2088
|
+
"binding_key",
|
|
2089
|
+
"project_id",
|
|
2090
|
+
"source",
|
|
2091
|
+
"source_project_id",
|
|
2092
|
+
"workspace_id",
|
|
2093
|
+
"canonical_path_hash",
|
|
2094
|
+
"created_at",
|
|
2095
|
+
"updated_at"
|
|
2096
|
+
],
|
|
2097
|
+
capture_checkpoints: [
|
|
2098
|
+
"session_id",
|
|
2099
|
+
"binding_key",
|
|
2100
|
+
"project_id",
|
|
2101
|
+
"state",
|
|
2102
|
+
"last_message_id",
|
|
2103
|
+
"last_reconciled_at",
|
|
2104
|
+
"next_reconcile_at",
|
|
2105
|
+
"failure_count",
|
|
2106
|
+
"lease_owner",
|
|
2107
|
+
"lease_expires_at",
|
|
2108
|
+
"created_at",
|
|
2109
|
+
"updated_at"
|
|
2110
|
+
],
|
|
2111
|
+
capture_events: [
|
|
2112
|
+
"idempotency_key",
|
|
2113
|
+
"contract",
|
|
2114
|
+
"project_id",
|
|
2115
|
+
"binding_key",
|
|
2116
|
+
"event_kind",
|
|
2117
|
+
"source_session_id",
|
|
2118
|
+
"source_message_id",
|
|
2119
|
+
"source_ordinal",
|
|
2120
|
+
"source_tool_call_id",
|
|
2121
|
+
"payload_json",
|
|
2122
|
+
"payload_hash",
|
|
2123
|
+
"redaction_version",
|
|
2124
|
+
"state",
|
|
2125
|
+
"attempt_count",
|
|
2126
|
+
"note_id",
|
|
2127
|
+
"last_error_code",
|
|
2128
|
+
"generation",
|
|
2129
|
+
"created_at",
|
|
2130
|
+
"updated_at",
|
|
2131
|
+
"processed_at"
|
|
2132
|
+
],
|
|
2133
|
+
note_provenance: [
|
|
2134
|
+
"id",
|
|
2135
|
+
"project_id",
|
|
2136
|
+
"note_id",
|
|
2137
|
+
"source_type",
|
|
2138
|
+
"capture_event_id",
|
|
2139
|
+
"source_session_id",
|
|
2140
|
+
"source_message_id",
|
|
2141
|
+
"source_ordinal",
|
|
2142
|
+
"source_tool_call_id",
|
|
2143
|
+
"redaction_version",
|
|
2144
|
+
"extractor_version",
|
|
2145
|
+
"confidence",
|
|
2146
|
+
"created_at"
|
|
2147
|
+
],
|
|
2148
|
+
note_revisions: [
|
|
2149
|
+
"project_id",
|
|
2150
|
+
"note_id",
|
|
2151
|
+
"revision",
|
|
2152
|
+
"kind",
|
|
2153
|
+
"title",
|
|
2154
|
+
"summary",
|
|
2155
|
+
"content",
|
|
2156
|
+
"size_class",
|
|
2157
|
+
"pinned",
|
|
2158
|
+
"status",
|
|
2159
|
+
"supersedes_id",
|
|
2160
|
+
"subject_key",
|
|
2161
|
+
"content_hash",
|
|
2162
|
+
"provenance_id",
|
|
2163
|
+
"created_at"
|
|
2164
|
+
],
|
|
2165
|
+
index_outbox: [
|
|
2166
|
+
"id",
|
|
2167
|
+
"backend",
|
|
2168
|
+
"operation",
|
|
2169
|
+
"project_id",
|
|
2170
|
+
"note_id",
|
|
2171
|
+
"revision",
|
|
2172
|
+
"content_hash",
|
|
2173
|
+
"state",
|
|
2174
|
+
"attempt_count",
|
|
2175
|
+
"available_at",
|
|
2176
|
+
"lease_owner",
|
|
2177
|
+
"lease_expires_at",
|
|
2178
|
+
"last_error_code",
|
|
2179
|
+
"created_at",
|
|
2180
|
+
"completed_at"
|
|
2181
|
+
]
|
|
2182
|
+
};
|
|
2183
|
+
function assertLegacySchemaIdentity(db, version) {
|
|
2184
|
+
if (version < 2 || version > 10)
|
|
2185
|
+
throw new Error("unrecognized_database");
|
|
2186
|
+
const applicationID = db.query("PRAGMA application_id").get().application_id;
|
|
2187
|
+
if (applicationID !== 0 || tableExists(db, "agz_meta"))
|
|
2188
|
+
throw new Error("unrecognized_database");
|
|
2189
|
+
if (version === 2 && tableExists(db, "memory_items")) {
|
|
2190
|
+
assertV2Identity(db);
|
|
2191
|
+
assertHealthyDatabase(db);
|
|
2192
|
+
return;
|
|
2193
|
+
}
|
|
2194
|
+
if (version < 8) {
|
|
2195
|
+
assertPreV8Identity(db, version);
|
|
2196
|
+
assertHealthyDatabase(db);
|
|
2197
|
+
return;
|
|
2198
|
+
}
|
|
2199
|
+
const states = db.query("SELECT version FROM schema_state").all();
|
|
2200
|
+
if (states.length !== 1 || states[0]?.version !== version)
|
|
2201
|
+
throw new Error("unrecognized_database");
|
|
2202
|
+
const rows = db.query("SELECT type, name FROM sqlite_master WHERE name NOT LIKE 'sqlite_%'").all();
|
|
2203
|
+
const actual = new Set(rows.map((row) => `${row.type}:${row.name}`));
|
|
2204
|
+
const allowed = version === 8 ? V8_OBJECTS : V9_V10_OBJECTS;
|
|
2205
|
+
const required = version === 8 ? V8_OBJECTS : new Set([...V9_V10_OBJECTS].filter((name) => !name.startsWith("index:capture_events_")));
|
|
2206
|
+
if (actual.size > allowed.size || [...actual].some((name) => !allowed.has(name)) || [...required].some((name) => !actual.has(name))) {
|
|
2207
|
+
throw new Error("unrecognized_database");
|
|
2208
|
+
}
|
|
2209
|
+
const columns = version === 8 ? {
|
|
2210
|
+
projects: V9_V10_COLUMNS.projects,
|
|
2211
|
+
notes: V9_V10_COLUMNS.notes.filter((name) => !["current_revision", "subject_key", "content_hash"].includes(name)),
|
|
2212
|
+
note_edges: V9_V10_COLUMNS.note_edges,
|
|
2213
|
+
schema_state: V9_V10_COLUMNS.schema_state
|
|
2214
|
+
} : V9_V10_COLUMNS;
|
|
2215
|
+
for (const [table, expected] of Object.entries(columns)) {
|
|
2216
|
+
const actualColumns = db.query(`PRAGMA table_info(${table})`).all().map((row) => row.name);
|
|
2217
|
+
if (JSON.stringify(actualColumns) !== JSON.stringify(expected)) {
|
|
2218
|
+
throw new Error("unrecognized_database");
|
|
2219
|
+
}
|
|
2220
|
+
}
|
|
2221
|
+
assertHealthyDatabase(db);
|
|
2222
|
+
if (version === 10)
|
|
2223
|
+
assertV10SourceDatabase(db);
|
|
2224
|
+
}
|
|
2225
|
+
function assertV2Identity(db) {
|
|
2226
|
+
const rows = db.query("SELECT type, name FROM sqlite_master WHERE name NOT LIKE 'sqlite_%'").all();
|
|
2227
|
+
const allowed = new Set(V2_TABLES.map((table) => `table:${table}`));
|
|
2228
|
+
const actual = new Set(rows.map((row) => `${row.type}:${row.name}`));
|
|
2229
|
+
if ([...actual].some((name) => !allowed.has(name)))
|
|
2230
|
+
throw new Error("unrecognized_database");
|
|
2231
|
+
for (const [table, expected] of Object.entries(V2_COLUMNS)) {
|
|
2232
|
+
assertColumns(db, table, expected);
|
|
2233
|
+
}
|
|
38
2234
|
}
|
|
39
|
-
function
|
|
40
|
-
|
|
2235
|
+
function assertPreV8Identity(db, version) {
|
|
2236
|
+
const rows = db.query("SELECT type, name FROM sqlite_master WHERE name NOT LIKE 'sqlite_%'").all();
|
|
2237
|
+
const actual = new Set(rows.map((row) => `${row.type}:${row.name}`));
|
|
2238
|
+
const required = ["table:notes", "table:note_edges", "table:schema_state"];
|
|
2239
|
+
if ([...actual].some((name) => !PRE_V8_OBJECTS.has(name)) || required.some((name) => !actual.has(name)) || version === 7 && !actual.has("table:projects")) {
|
|
2240
|
+
throw new Error("unrecognized_database");
|
|
2241
|
+
}
|
|
2242
|
+
const states = db.query("SELECT version FROM schema_state").all();
|
|
2243
|
+
if (states.length !== 1 || states[0]?.version !== version)
|
|
2244
|
+
throw new Error("unrecognized_database");
|
|
2245
|
+
const noteColumns = columns(db, "notes");
|
|
2246
|
+
const expectedNotes = [...PRE_V8_COLUMNS.notes];
|
|
2247
|
+
const expectedPinnedNotes = [...expectedNotes.slice(0, 7), "pinned", ...expectedNotes.slice(7)];
|
|
2248
|
+
if (JSON.stringify(noteColumns) !== JSON.stringify(expectedNotes) && JSON.stringify(noteColumns) !== JSON.stringify(expectedPinnedNotes)) {
|
|
2249
|
+
throw new Error("unrecognized_database");
|
|
2250
|
+
}
|
|
2251
|
+
assertColumns(db, "note_edges", PRE_V8_COLUMNS.note_edges);
|
|
2252
|
+
assertColumns(db, "schema_state", PRE_V8_COLUMNS.schema_state);
|
|
2253
|
+
if (tableExists(db, "projects"))
|
|
2254
|
+
assertColumns(db, "projects", PRE_V8_COLUMNS.projects);
|
|
2255
|
+
if (tableExists(db, "notes_fts"))
|
|
2256
|
+
assertColumns(db, "notes_fts", PRE_V8_COLUMNS.notes_fts);
|
|
2257
|
+
}
|
|
2258
|
+
function assertColumns(db, table, expected) {
|
|
2259
|
+
if (JSON.stringify(columns(db, table)) !== JSON.stringify(expected)) {
|
|
2260
|
+
throw new Error("unrecognized_database");
|
|
2261
|
+
}
|
|
2262
|
+
}
|
|
2263
|
+
function columns(db, table) {
|
|
2264
|
+
return db.query(`PRAGMA table_info(${table})`).all().map((row) => row.name);
|
|
2265
|
+
}
|
|
2266
|
+
function tableExists(db, table) {
|
|
2267
|
+
return db.query("SELECT COUNT(*) AS count FROM sqlite_master WHERE type = 'table' AND name = ?").get(table).count > 0;
|
|
41
2268
|
}
|
|
42
2269
|
|
|
43
|
-
// src/
|
|
44
|
-
|
|
45
|
-
var KINDS = ["decision", "fact", "procedure", "context", "research", "preference", "task"];
|
|
46
|
-
var PREDICATES = ["SUPPORTS", "DERIVED_FROM", "PART_OF", "ABOUT", "PRECEDES", "SUPERSEDES"];
|
|
47
|
-
|
|
48
|
-
// src/db/backup.ts
|
|
49
|
-
import { createHash as createHash2, randomUUID } from "crypto";
|
|
50
|
-
import { Database } from "bun:sqlite";
|
|
2270
|
+
// src/db/maintenance.ts
|
|
2271
|
+
import { randomUUID as randomUUID2 } from "crypto";
|
|
51
2272
|
import {
|
|
52
2273
|
chmodSync,
|
|
53
2274
|
closeSync,
|
|
54
|
-
|
|
2275
|
+
constants,
|
|
55
2276
|
existsSync,
|
|
56
2277
|
fsyncSync,
|
|
57
2278
|
lstatSync,
|
|
58
2279
|
mkdirSync,
|
|
59
2280
|
openSync,
|
|
60
2281
|
readFileSync,
|
|
2282
|
+
readdirSync,
|
|
61
2283
|
renameSync,
|
|
62
|
-
|
|
2284
|
+
rmdirSync,
|
|
2285
|
+
unlinkSync,
|
|
63
2286
|
writeFileSync
|
|
64
2287
|
} from "fs";
|
|
65
|
-
import {
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
const
|
|
70
|
-
const
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
"
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
if (
|
|
2288
|
+
import { hostname, platform } from "os";
|
|
2289
|
+
import { basename, dirname, join as join2, parse, resolve } from "path";
|
|
2290
|
+
function acquireDatabaseLease(databasePath) {
|
|
2291
|
+
const canonicalPath = canonicalDatabasePath(databasePath);
|
|
2292
|
+
const gatePath = maintenanceGatePath(canonicalPath);
|
|
2293
|
+
const leaseDirectory = databaseLeaseDirectory(canonicalPath);
|
|
2294
|
+
assertNoSymbolicLinks(dirname(canonicalPath));
|
|
2295
|
+
assertNoSymbolicLinks(canonicalPath, true);
|
|
2296
|
+
assertProtocolPath(gatePath, "maintenance gate", true);
|
|
2297
|
+
if (existsSync(gatePath))
|
|
2298
|
+
throw new Error("maintenance_gate_active");
|
|
2299
|
+
ensurePrivateDirectory(leaseDirectory, "database lease registry");
|
|
2300
|
+
const owner = ownerRecord();
|
|
2301
|
+
const temporaryPath = join2(leaseDirectory, `.${owner.ownerID}.tmp`);
|
|
2302
|
+
const leasePath = join2(leaseDirectory, `${owner.ownerID}.json`);
|
|
2303
|
+
try {
|
|
2304
|
+
writeExclusiveRecord(temporaryPath, owner);
|
|
2305
|
+
renameSync(temporaryPath, leasePath);
|
|
2306
|
+
fsyncPath(leaseDirectory);
|
|
2307
|
+
assertProtocolPath(gatePath, "maintenance gate", true);
|
|
2308
|
+
if (existsSync(gatePath)) {
|
|
2309
|
+
removeOwnedRecord(leasePath, owner.ownerID);
|
|
2310
|
+
throw new Error("maintenance_gate_active");
|
|
2311
|
+
}
|
|
2312
|
+
} catch (error) {
|
|
2313
|
+
removeFreshRecord(temporaryPath);
|
|
2314
|
+
removeOwnedRecord(leasePath, owner.ownerID);
|
|
2315
|
+
throw error;
|
|
2316
|
+
}
|
|
2317
|
+
let released = false;
|
|
2318
|
+
return {
|
|
2319
|
+
databasePath: canonicalPath,
|
|
2320
|
+
release: () => {
|
|
2321
|
+
if (released)
|
|
2322
|
+
return;
|
|
2323
|
+
removeOwnedRecord(leasePath, owner.ownerID);
|
|
2324
|
+
released = true;
|
|
2325
|
+
}
|
|
2326
|
+
};
|
|
2327
|
+
}
|
|
2328
|
+
function acquireMaintenanceGate(databasePath, recovery) {
|
|
2329
|
+
const canonicalPath = canonicalDatabasePath(databasePath);
|
|
2330
|
+
const gatePath = maintenanceGatePath(canonicalPath);
|
|
2331
|
+
const leaseDirectory = databaseLeaseDirectory(canonicalPath);
|
|
2332
|
+
assertNoSymbolicLinks(dirname(canonicalPath));
|
|
2333
|
+
assertNoSymbolicLinks(canonicalPath, true);
|
|
2334
|
+
assertProtocolPath(gatePath, "maintenance gate", true);
|
|
2335
|
+
assertProtocolPath(leaseDirectory, "database lease registry", true);
|
|
2336
|
+
const owner = ownerRecord();
|
|
2337
|
+
let created = false;
|
|
2338
|
+
let gateStat;
|
|
2339
|
+
try {
|
|
2340
|
+
mkdirSync(gatePath, { mode: 448 });
|
|
2341
|
+
created = true;
|
|
2342
|
+
gateStat = lstatSync(gatePath);
|
|
2343
|
+
} catch (error) {
|
|
2344
|
+
if (!isAlreadyExists(error))
|
|
2345
|
+
throw error;
|
|
2346
|
+
gateStat = claimStaleGate(gatePath, owner, recovery);
|
|
2347
|
+
}
|
|
2348
|
+
const ownerPath = join2(gatePath, "owner.json");
|
|
2349
|
+
try {
|
|
2350
|
+
if (created) {
|
|
2351
|
+
writeExclusiveRecord(ownerPath, owner);
|
|
2352
|
+
fsyncPath(gatePath);
|
|
2353
|
+
}
|
|
2354
|
+
rejectActiveLeases(leaseDirectory);
|
|
2355
|
+
} catch (error) {
|
|
2356
|
+
removeFreshGate(gatePath, owner.ownerID, gateStat.dev, gateStat.ino);
|
|
2357
|
+
throw error;
|
|
2358
|
+
}
|
|
2359
|
+
let released = false;
|
|
2360
|
+
let retained = false;
|
|
2361
|
+
const assertOwned = () => {
|
|
2362
|
+
if (released)
|
|
2363
|
+
throw new Error("maintenance_gate_not_owned");
|
|
2364
|
+
const record = readOwnerRecord(ownerPath, "maintenance gate owner");
|
|
2365
|
+
if (record.ownerID !== owner.ownerID)
|
|
2366
|
+
throw new Error("maintenance_gate_not_owned");
|
|
2367
|
+
};
|
|
2368
|
+
return {
|
|
2369
|
+
databasePath: canonicalPath,
|
|
2370
|
+
assertOwned,
|
|
2371
|
+
retain: () => {
|
|
2372
|
+
assertOwned();
|
|
2373
|
+
replaceOwnerRecord(ownerPath, owner.ownerID, {
|
|
2374
|
+
...owner,
|
|
2375
|
+
state: "recovery-required"
|
|
2376
|
+
});
|
|
2377
|
+
retained = true;
|
|
2378
|
+
},
|
|
2379
|
+
release: () => {
|
|
2380
|
+
if (released || retained)
|
|
2381
|
+
return;
|
|
2382
|
+
assertOwned();
|
|
2383
|
+
removeOwnedGate(gatePath, owner.ownerID);
|
|
2384
|
+
released = true;
|
|
2385
|
+
}
|
|
2386
|
+
};
|
|
2387
|
+
}
|
|
2388
|
+
function recoverStaleMaintenanceGate(databasePath, validate) {
|
|
2389
|
+
const canonicalPath = canonicalDatabasePath(databasePath);
|
|
2390
|
+
if (!existsSync(maintenanceGatePath(canonicalPath)))
|
|
2391
|
+
return false;
|
|
2392
|
+
const gate = acquireMaintenanceGate(canonicalPath);
|
|
2393
|
+
try {
|
|
2394
|
+
validate();
|
|
2395
|
+
} catch (error) {
|
|
2396
|
+
gate.retain();
|
|
2397
|
+
throw error;
|
|
2398
|
+
} finally {
|
|
2399
|
+
gate.release();
|
|
2400
|
+
}
|
|
2401
|
+
return true;
|
|
2402
|
+
}
|
|
2403
|
+
function assertMaintenanceGateFor(gate, databasePath) {
|
|
2404
|
+
if (gate.databasePath !== canonicalDatabasePath(databasePath)) {
|
|
2405
|
+
throw new Error("maintenance_gate_target_mismatch");
|
|
2406
|
+
}
|
|
2407
|
+
gate.assertOwned();
|
|
2408
|
+
}
|
|
2409
|
+
function canonicalDatabasePath(path) {
|
|
2410
|
+
const absolute = resolve(path);
|
|
2411
|
+
const parent = dirname(absolute);
|
|
2412
|
+
assertNoSymbolicLinks(parent);
|
|
2413
|
+
return join2(resolve(parent), basename(absolute));
|
|
2414
|
+
}
|
|
2415
|
+
function ensureDatabaseParent(path) {
|
|
2416
|
+
const parent = dirname(resolve(path));
|
|
2417
|
+
assertNoSymbolicLinks(parent);
|
|
2418
|
+
const existed = existsSync(parent);
|
|
2419
|
+
mkdirSync(parent, { recursive: true, mode: 448 });
|
|
2420
|
+
assertNoSymbolicLinks(parent);
|
|
2421
|
+
if (!existed)
|
|
2422
|
+
chmodSync(parent, 448);
|
|
2423
|
+
const stat = lstatSync(parent);
|
|
2424
|
+
if (!stat.isDirectory() || stat.isSymbolicLink()) {
|
|
2425
|
+
throw new Error("database parent must be a directory");
|
|
2426
|
+
}
|
|
2427
|
+
if (typeof process.getuid === "function" && stat.uid !== process.getuid()) {
|
|
2428
|
+
throw new Error("database parent must be owned by the current user");
|
|
2429
|
+
}
|
|
2430
|
+
if (platform() !== "win32" && (stat.mode & 18) !== 0) {
|
|
2431
|
+
throw new Error("database parent must not be group or world writable");
|
|
2432
|
+
}
|
|
2433
|
+
}
|
|
2434
|
+
function assertNoSymbolicLinks(path, allowMissingLeaf = false) {
|
|
2435
|
+
const absolute = resolve(path);
|
|
2436
|
+
const root = parse(absolute).root;
|
|
2437
|
+
const parts = absolute.slice(root.length).split(/[\\/]+/).filter(Boolean);
|
|
2438
|
+
let current = root;
|
|
2439
|
+
for (let index = 0;index < parts.length; index++) {
|
|
2440
|
+
current = join2(current, parts[index]);
|
|
2441
|
+
try {
|
|
2442
|
+
const stat = lstatSync(current);
|
|
2443
|
+
if (stat.isSymbolicLink())
|
|
2444
|
+
throw new Error(`symbolic links are not allowed: ${current}`);
|
|
2445
|
+
} catch (error) {
|
|
2446
|
+
if (isMissing(error) && (allowMissingLeaf ? index === parts.length - 1 : true))
|
|
2447
|
+
return;
|
|
2448
|
+
throw error;
|
|
2449
|
+
}
|
|
2450
|
+
}
|
|
2451
|
+
}
|
|
2452
|
+
function maintenanceGatePath(databasePath) {
|
|
2453
|
+
return `${databasePath}.maintenance`;
|
|
2454
|
+
}
|
|
2455
|
+
function databaseLeaseDirectory(databasePath) {
|
|
2456
|
+
return `${databasePath}.leases`;
|
|
2457
|
+
}
|
|
2458
|
+
function ensurePrivateDirectory(path, label) {
|
|
2459
|
+
assertProtocolPath(path, label, true);
|
|
2460
|
+
if (!existsSync(path)) {
|
|
2461
|
+
try {
|
|
2462
|
+
mkdirSync(path, { recursive: false, mode: 448 });
|
|
2463
|
+
} catch (error) {
|
|
2464
|
+
if (!isAlreadyExists(error))
|
|
2465
|
+
throw error;
|
|
2466
|
+
}
|
|
2467
|
+
}
|
|
2468
|
+
const stat = lstatSync(path);
|
|
2469
|
+
if (!stat.isDirectory() || stat.isSymbolicLink())
|
|
2470
|
+
throw new Error(`${label} must be a directory`);
|
|
2471
|
+
if (typeof process.getuid === "function" && stat.uid !== process.getuid()) {
|
|
2472
|
+
throw new Error(`${label} must be owned by the current user`);
|
|
2473
|
+
}
|
|
2474
|
+
chmodSync(path, 448);
|
|
2475
|
+
}
|
|
2476
|
+
function assertProtocolPath(path, label, allowMissing) {
|
|
2477
|
+
assertNoSymbolicLinks(path, allowMissing);
|
|
2478
|
+
if (!existsSync(path))
|
|
2479
|
+
return;
|
|
2480
|
+
const stat = lstatSync(path);
|
|
2481
|
+
if (stat.isSymbolicLink())
|
|
2482
|
+
throw new Error(`${label} must not be a symbolic link`);
|
|
2483
|
+
}
|
|
2484
|
+
function writeExclusiveRecord(path, owner) {
|
|
2485
|
+
writeFileSync(path, `${JSON.stringify(owner)}
|
|
2486
|
+
`, {
|
|
2487
|
+
encoding: "utf8",
|
|
2488
|
+
flag: "wx",
|
|
2489
|
+
mode: 384
|
|
2490
|
+
});
|
|
2491
|
+
fsyncPath(path);
|
|
2492
|
+
}
|
|
2493
|
+
function rejectActiveLeases(leaseDirectory) {
|
|
2494
|
+
if (!existsSync(leaseDirectory))
|
|
2495
|
+
return;
|
|
2496
|
+
const directoryStat = lstatSync(leaseDirectory);
|
|
2497
|
+
if (!directoryStat.isDirectory() || directoryStat.isSymbolicLink()) {
|
|
2498
|
+
throw new Error("database lease registry must be a directory");
|
|
2499
|
+
}
|
|
2500
|
+
for (const entry of readdirSync(leaseDirectory)) {
|
|
2501
|
+
if (!entry.endsWith(".json"))
|
|
86
2502
|
continue;
|
|
87
|
-
|
|
2503
|
+
const leasePath = join2(leaseDirectory, entry);
|
|
2504
|
+
const before = lstatSync(leasePath);
|
|
2505
|
+
if (!before.isFile() || before.isSymbolicLink())
|
|
2506
|
+
throw new Error("database lease must be a regular file");
|
|
2507
|
+
const lease = readOwnerRecord(leasePath, "database lease");
|
|
2508
|
+
if (ownerIsActive(lease))
|
|
2509
|
+
throw new Error("active_database_handles");
|
|
2510
|
+
const after = lstatSync(leasePath);
|
|
2511
|
+
if (before.dev !== after.dev || before.ino !== after.ino) {
|
|
2512
|
+
throw new Error("database lease changed during stale-owner verification");
|
|
2513
|
+
}
|
|
2514
|
+
unlinkSync(leasePath);
|
|
88
2515
|
}
|
|
89
|
-
|
|
2516
|
+
fsyncPath(leaseDirectory);
|
|
90
2517
|
}
|
|
91
|
-
function
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
2518
|
+
function ownerStatus(owner) {
|
|
2519
|
+
if (owner.state === "recovery-required")
|
|
2520
|
+
return "recovery-required";
|
|
2521
|
+
if (owner.hostname !== hostname())
|
|
2522
|
+
return "unverifiable";
|
|
2523
|
+
try {
|
|
2524
|
+
process.kill(owner.pid, 0);
|
|
2525
|
+
} catch (error) {
|
|
2526
|
+
if (error && typeof error === "object" && "code" in error && error.code === "ESRCH") {
|
|
2527
|
+
return "stale";
|
|
2528
|
+
}
|
|
2529
|
+
return "unverifiable";
|
|
95
2530
|
}
|
|
96
|
-
|
|
97
|
-
|
|
2531
|
+
const currentStart = processStartMarker(owner.pid);
|
|
2532
|
+
if (owner.processStart === null || currentStart === null)
|
|
2533
|
+
return "unverifiable";
|
|
2534
|
+
return owner.processStart === currentStart ? "active" : "stale";
|
|
2535
|
+
}
|
|
2536
|
+
function ownerIsActive(owner) {
|
|
2537
|
+
return ownerStatus(owner) !== "stale";
|
|
2538
|
+
}
|
|
2539
|
+
function ownerRecord() {
|
|
2540
|
+
return {
|
|
2541
|
+
ownerID: randomUUID2(),
|
|
2542
|
+
pid: process.pid,
|
|
2543
|
+
processStart: processStartMarker(process.pid),
|
|
2544
|
+
hostname: hostname(),
|
|
2545
|
+
createdAt: Date.now(),
|
|
2546
|
+
state: "active"
|
|
2547
|
+
};
|
|
2548
|
+
}
|
|
2549
|
+
function processStartMarker(pid) {
|
|
2550
|
+
if (platform() !== "linux")
|
|
2551
|
+
return null;
|
|
2552
|
+
try {
|
|
2553
|
+
const value = readFileSync(`/proc/${pid}/stat`, "utf8");
|
|
2554
|
+
const close = value.lastIndexOf(")");
|
|
2555
|
+
const fields = value.slice(close + 1).trim().split(/\s+/);
|
|
2556
|
+
const startTime = fields[19];
|
|
2557
|
+
return startTime ? `linux:${startTime}` : null;
|
|
2558
|
+
} catch {
|
|
2559
|
+
return null;
|
|
98
2560
|
}
|
|
99
|
-
return health;
|
|
100
2561
|
}
|
|
101
|
-
function
|
|
102
|
-
const
|
|
103
|
-
|
|
2562
|
+
function readOwnerRecord(path, label) {
|
|
2563
|
+
const stat = lstatSync(path);
|
|
2564
|
+
if (!stat.isFile() || stat.isSymbolicLink())
|
|
2565
|
+
throw new Error(`${label} must be a regular file`);
|
|
2566
|
+
const parsed = JSON.parse(readFileSync(path, "utf8"));
|
|
2567
|
+
if (typeof parsed.ownerID !== "string" || !parsed.ownerID || !Number.isSafeInteger(parsed.pid) || parsed.pid <= 0 || parsed.processStart !== null && (typeof parsed.processStart !== "string" || !parsed.processStart) || typeof parsed.hostname !== "string" || !parsed.hostname || !Number.isSafeInteger(parsed.createdAt) || parsed.createdAt < 0 || parsed.state !== undefined && parsed.state !== "active" && parsed.state !== "recovery-required") {
|
|
2568
|
+
throw new Error(`${label} is invalid`);
|
|
2569
|
+
}
|
|
2570
|
+
return { ...parsed, state: parsed.state ?? "active" };
|
|
2571
|
+
}
|
|
2572
|
+
function claimStaleGate(gatePath, owner, recovery) {
|
|
2573
|
+
const gate = lstatSync(gatePath);
|
|
2574
|
+
if (!gate.isDirectory() || gate.isSymbolicLink()) {
|
|
2575
|
+
throw new Error("maintenance gate must be a directory");
|
|
2576
|
+
}
|
|
2577
|
+
const ownerPath = join2(gatePath, "owner.json");
|
|
2578
|
+
const claimPath = join2(gatePath, "takeover.json");
|
|
2579
|
+
acquireTakeoverClaim(claimPath, owner);
|
|
2580
|
+
try {
|
|
2581
|
+
const before = lstatSync(ownerPath);
|
|
2582
|
+
const previous = readOwnerRecord(ownerPath, "maintenance gate owner");
|
|
2583
|
+
const status = ownerStatus(previous);
|
|
2584
|
+
if (status === "active")
|
|
2585
|
+
throw new Error("maintenance_gate_active");
|
|
2586
|
+
if (status === "unverifiable")
|
|
2587
|
+
throw new Error("maintenance_gate_unverifiable");
|
|
2588
|
+
if (status === "recovery-required") {
|
|
2589
|
+
if (recovery?.ownerID !== previous.ownerID || recovery.confirmation !== "RECOVER_RETAINED_MAINTENANCE_GATE") {
|
|
2590
|
+
throw new Error("maintenance_gate_recovery_required");
|
|
2591
|
+
}
|
|
2592
|
+
}
|
|
2593
|
+
const currentGate = lstatSync(gatePath);
|
|
2594
|
+
const currentOwner = lstatSync(ownerPath);
|
|
2595
|
+
if (currentGate.dev !== gate.dev || currentGate.ino !== gate.ino || currentOwner.dev !== before.dev || currentOwner.ino !== before.ino || readOwnerRecord(ownerPath, "maintenance gate owner").ownerID !== previous.ownerID) {
|
|
2596
|
+
throw new Error("maintenance gate changed during stale-owner verification");
|
|
2597
|
+
}
|
|
2598
|
+
renameSync(claimPath, ownerPath);
|
|
2599
|
+
fsyncPath(gatePath);
|
|
2600
|
+
return { dev: gate.dev, ino: gate.ino };
|
|
2601
|
+
} catch (error) {
|
|
2602
|
+
removeOwnedRecord(claimPath, owner.ownerID);
|
|
2603
|
+
throw error;
|
|
2604
|
+
}
|
|
2605
|
+
}
|
|
2606
|
+
function acquireTakeoverClaim(path, owner) {
|
|
2607
|
+
for (let attempt = 0;attempt < 2; attempt++) {
|
|
2608
|
+
try {
|
|
2609
|
+
writeExclusiveRecord(path, owner);
|
|
2610
|
+
return;
|
|
2611
|
+
} catch (error) {
|
|
2612
|
+
if (!isAlreadyExists(error))
|
|
2613
|
+
throw error;
|
|
2614
|
+
const before = lstatSync(path);
|
|
2615
|
+
const existing = readOwnerRecord(path, "maintenance takeover owner");
|
|
2616
|
+
const status = ownerStatus(existing);
|
|
2617
|
+
if (status === "active")
|
|
2618
|
+
throw new Error("maintenance_gate_active");
|
|
2619
|
+
if (status !== "stale")
|
|
2620
|
+
throw new Error("maintenance_gate_unverifiable");
|
|
2621
|
+
const after = lstatSync(path);
|
|
2622
|
+
if (before.dev !== after.dev || before.ino !== after.ino) {
|
|
2623
|
+
throw new Error("maintenance takeover changed during stale-owner verification");
|
|
2624
|
+
}
|
|
2625
|
+
unlinkSync(path);
|
|
2626
|
+
fsyncPath(dirname(path));
|
|
2627
|
+
}
|
|
2628
|
+
}
|
|
2629
|
+
throw new Error("maintenance_gate_active");
|
|
2630
|
+
}
|
|
2631
|
+
function replaceOwnerRecord(ownerPath, ownerID, replacement) {
|
|
2632
|
+
const temporary = `${ownerPath}.${ownerID}.tmp`;
|
|
2633
|
+
try {
|
|
2634
|
+
writeExclusiveRecord(temporary, replacement);
|
|
2635
|
+
if (readOwnerRecord(ownerPath, "maintenance gate owner").ownerID !== ownerID) {
|
|
2636
|
+
throw new Error("maintenance_gate_not_owned");
|
|
2637
|
+
}
|
|
2638
|
+
renameSync(temporary, ownerPath);
|
|
2639
|
+
fsyncPath(dirname(ownerPath));
|
|
2640
|
+
} catch (error) {
|
|
2641
|
+
removeFreshRecord(temporary);
|
|
2642
|
+
throw error;
|
|
2643
|
+
}
|
|
2644
|
+
}
|
|
2645
|
+
function removeOwnedRecord(path, ownerID) {
|
|
2646
|
+
try {
|
|
2647
|
+
if (readOwnerRecord(path, "owner record").ownerID !== ownerID)
|
|
2648
|
+
return;
|
|
2649
|
+
unlinkSync(path);
|
|
2650
|
+
} catch (error) {
|
|
2651
|
+
if (!isMissing(error))
|
|
2652
|
+
throw error;
|
|
2653
|
+
}
|
|
2654
|
+
}
|
|
2655
|
+
function removeOwnedGate(gatePath, ownerID) {
|
|
2656
|
+
const ownerPath = join2(gatePath, "owner.json");
|
|
2657
|
+
try {
|
|
2658
|
+
if (readOwnerRecord(ownerPath, "maintenance gate owner").ownerID !== ownerID)
|
|
2659
|
+
return;
|
|
2660
|
+
unlinkSync(ownerPath);
|
|
2661
|
+
rmdirSync(gatePath);
|
|
2662
|
+
} catch (error) {
|
|
2663
|
+
if (!isMissing(error))
|
|
2664
|
+
throw error;
|
|
2665
|
+
}
|
|
2666
|
+
}
|
|
2667
|
+
function removeFreshRecord(path) {
|
|
2668
|
+
try {
|
|
2669
|
+
const stat = lstatSync(path);
|
|
2670
|
+
if (!stat.isFile() || stat.isSymbolicLink())
|
|
2671
|
+
return;
|
|
2672
|
+
if (typeof process.getuid === "function" && stat.uid !== process.getuid())
|
|
2673
|
+
return;
|
|
2674
|
+
unlinkSync(path);
|
|
2675
|
+
} catch (error) {
|
|
2676
|
+
if (!isMissing(error))
|
|
2677
|
+
throw error;
|
|
2678
|
+
}
|
|
2679
|
+
}
|
|
2680
|
+
function removeFreshGate(gatePath, ownerID, device, inode) {
|
|
2681
|
+
try {
|
|
2682
|
+
const gate = lstatSync(gatePath);
|
|
2683
|
+
if (!gate.isDirectory() || gate.isSymbolicLink() || gate.dev !== device || gate.ino !== inode)
|
|
2684
|
+
return;
|
|
2685
|
+
const ownerPath = join2(gatePath, "owner.json");
|
|
2686
|
+
try {
|
|
2687
|
+
const record = readOwnerRecord(ownerPath, "maintenance gate owner");
|
|
2688
|
+
if (record.ownerID !== ownerID)
|
|
2689
|
+
return;
|
|
2690
|
+
unlinkSync(ownerPath);
|
|
2691
|
+
} catch (error) {
|
|
2692
|
+
if (!isMissing(error))
|
|
2693
|
+
removeFreshRecord(ownerPath);
|
|
2694
|
+
}
|
|
2695
|
+
rmdirSync(gatePath);
|
|
2696
|
+
} catch (error) {
|
|
2697
|
+
if (!isMissing(error))
|
|
2698
|
+
throw error;
|
|
2699
|
+
}
|
|
2700
|
+
}
|
|
2701
|
+
function fsyncPath(path) {
|
|
2702
|
+
const stat = lstatSync(path);
|
|
2703
|
+
if (platform() === "win32" && stat.isDirectory())
|
|
2704
|
+
return;
|
|
2705
|
+
const descriptor = openSync(path, platform() === "win32" ? constants.O_RDWR : constants.O_RDONLY);
|
|
2706
|
+
try {
|
|
2707
|
+
fsyncSync(descriptor);
|
|
2708
|
+
} finally {
|
|
2709
|
+
closeSync(descriptor);
|
|
2710
|
+
}
|
|
2711
|
+
}
|
|
2712
|
+
function isMissing(error) {
|
|
2713
|
+
return Boolean(error && typeof error === "object" && "code" in error && error.code === "ENOENT");
|
|
2714
|
+
}
|
|
2715
|
+
function isAlreadyExists(error) {
|
|
2716
|
+
return Boolean(error && typeof error === "object" && "code" in error && error.code === "EEXIST");
|
|
104
2717
|
}
|
|
105
2718
|
|
|
106
2719
|
// src/db/backup.ts
|
|
107
2720
|
var BACKUP_FORMAT = "agz-memory-backup/1";
|
|
108
2721
|
function createVerifiedBackup(db, databasePath, sourceSchema, targetSchema, productVersion) {
|
|
109
2722
|
const sourceHealth = assertHealthyDatabase(db);
|
|
2723
|
+
try {
|
|
2724
|
+
assertManifestSourceSchema(db, sourceHealth.schemaVersion, sourceSchema);
|
|
2725
|
+
} catch {
|
|
2726
|
+
throw new Error(`backup source schema v${sourceSchema} does not match database schema v${sourceHealth.schemaVersion ?? "unknown"}`);
|
|
2727
|
+
}
|
|
110
2728
|
const checkpoint = db.query("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
111
2729
|
if (checkpoint.busy !== 0)
|
|
112
2730
|
throw new Error("database WAL checkpoint is busy");
|
|
113
2731
|
const backupDirectory = `${databasePath}.backup`;
|
|
114
|
-
|
|
115
|
-
|
|
2732
|
+
assertNoSymbolicLinks(databasePath);
|
|
2733
|
+
assertNoSymbolicLinks(backupDirectory, true);
|
|
2734
|
+
mkdirSync2(backupDirectory, { recursive: true, mode: 448 });
|
|
2735
|
+
const backupRootStat = lstatSync2(backupDirectory);
|
|
2736
|
+
if (!backupRootStat.isDirectory() || backupRootStat.isSymbolicLink()) {
|
|
2737
|
+
throw new Error("backup root must be a directory and not a symbolic link");
|
|
2738
|
+
}
|
|
2739
|
+
chmodSync2(backupDirectory, 448);
|
|
116
2740
|
const stamp = new Date().toISOString().replace(/[:.]/g, "-");
|
|
117
|
-
const stem = `schema-v${sourceSchema}-${stamp}-${
|
|
118
|
-
const finalDatabasePath =
|
|
119
|
-
const finalManifestPath =
|
|
2741
|
+
const stem = `schema-v${sourceSchema}-${stamp}-${randomUUID3()}`;
|
|
2742
|
+
const finalDatabasePath = join3(backupDirectory, `${stem}.sqlite`);
|
|
2743
|
+
const finalManifestPath = join3(backupDirectory, `${stem}.manifest.json`);
|
|
120
2744
|
const temporaryDatabasePath = `${finalDatabasePath}.tmp`;
|
|
121
2745
|
const temporaryManifestPath = `${finalManifestPath}.tmp`;
|
|
122
2746
|
try {
|
|
123
2747
|
db.exec(`VACUUM INTO '${escapeSql(temporaryDatabasePath)}'`);
|
|
124
|
-
|
|
125
|
-
const verification = new
|
|
2748
|
+
chmodSync2(temporaryDatabasePath, 384);
|
|
2749
|
+
const verification = new Database2(temporaryDatabasePath, { readonly: true });
|
|
126
2750
|
let backupHealth;
|
|
127
2751
|
let sqliteVersion;
|
|
128
2752
|
try {
|
|
129
2753
|
backupHealth = assertHealthyDatabase(verification);
|
|
2754
|
+
assertManifestSourceSchema(verification, backupHealth.schemaVersion, sourceSchema);
|
|
130
2755
|
sqliteVersion = verification.query("SELECT sqlite_version() AS version").get().version;
|
|
131
2756
|
} finally {
|
|
132
2757
|
verification.close();
|
|
@@ -134,7 +2759,7 @@ function createVerifiedBackup(db, databasePath, sourceSchema, targetSchema, prod
|
|
|
134
2759
|
if (JSON.stringify(backupHealth.counts) !== JSON.stringify(sourceHealth.counts)) {
|
|
135
2760
|
throw new Error("backup row counts differ from source database");
|
|
136
2761
|
}
|
|
137
|
-
const
|
|
2762
|
+
const digest = hashRegularFile(temporaryDatabasePath);
|
|
138
2763
|
const manifest = {
|
|
139
2764
|
format: BACKUP_FORMAT,
|
|
140
2765
|
productVersion,
|
|
@@ -142,22 +2767,22 @@ function createVerifiedBackup(db, databasePath, sourceSchema, targetSchema, prod
|
|
|
142
2767
|
targetSchema,
|
|
143
2768
|
createdAt: new Date().toISOString(),
|
|
144
2769
|
sqliteVersion,
|
|
145
|
-
databaseFile:
|
|
146
|
-
sha256:
|
|
147
|
-
size:
|
|
2770
|
+
databaseFile: basename2(finalDatabasePath),
|
|
2771
|
+
sha256: digest.sha256,
|
|
2772
|
+
size: digest.size,
|
|
148
2773
|
counts: backupHealth.counts,
|
|
149
2774
|
integrity: "ok",
|
|
150
2775
|
foreignKeyViolations: 0
|
|
151
2776
|
};
|
|
152
|
-
|
|
2777
|
+
writeFileSync2(temporaryManifestPath, `${JSON.stringify(manifest, null, 2)}
|
|
153
2778
|
`, {
|
|
154
2779
|
mode: 384
|
|
155
2780
|
});
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
2781
|
+
fsyncPath2(temporaryDatabasePath);
|
|
2782
|
+
fsyncPath2(temporaryManifestPath);
|
|
2783
|
+
renameSync2(temporaryDatabasePath, finalDatabasePath);
|
|
2784
|
+
renameSync2(temporaryManifestPath, finalManifestPath);
|
|
2785
|
+
fsyncPath2(backupDirectory);
|
|
161
2786
|
return { databasePath: finalDatabasePath, manifestPath: finalManifestPath, manifest };
|
|
162
2787
|
} catch (error) {
|
|
163
2788
|
rmSync(temporaryDatabasePath, { force: true });
|
|
@@ -166,16 +2791,20 @@ function createVerifiedBackup(db, databasePath, sourceSchema, targetSchema, prod
|
|
|
166
2791
|
}
|
|
167
2792
|
}
|
|
168
2793
|
function verifyBackupManifest(manifestPath) {
|
|
169
|
-
|
|
170
|
-
const
|
|
2794
|
+
assertNoSymbolicLinks(manifestPath);
|
|
2795
|
+
const resolvedManifestPath = resolve2(manifestPath);
|
|
2796
|
+
const manifestStat = lstatSync2(resolvedManifestPath);
|
|
171
2797
|
if (!manifestStat.isFile() || manifestStat.isSymbolicLink()) {
|
|
172
2798
|
throw new Error("backup manifest must be a regular file");
|
|
173
2799
|
}
|
|
174
|
-
const manifest = JSON.parse(
|
|
2800
|
+
const manifest = JSON.parse(readFileSync2(resolvedManifestPath, "utf8"));
|
|
175
2801
|
if (manifest.format !== BACKUP_FORMAT) {
|
|
176
2802
|
throw new Error("unsupported backup manifest format");
|
|
177
2803
|
}
|
|
178
|
-
if (
|
|
2804
|
+
if (!Number.isSafeInteger(manifest.sourceSchema) || manifest.sourceSchema < 2 || manifest.sourceSchema > 11) {
|
|
2805
|
+
throw new Error("backup manifest source schema is invalid");
|
|
2806
|
+
}
|
|
2807
|
+
if (typeof manifest.databaseFile !== "string" || !manifest.databaseFile || basename2(manifest.databaseFile) !== manifest.databaseFile) {
|
|
179
2808
|
throw new Error("backup databaseFile must be a basename");
|
|
180
2809
|
}
|
|
181
2810
|
if (typeof manifest.sha256 !== "string" || !/^[0-9a-f]{64}$/.test(manifest.sha256)) {
|
|
@@ -184,82 +2813,91 @@ function verifyBackupManifest(manifestPath) {
|
|
|
184
2813
|
if (!Number.isSafeInteger(manifest.size) || manifest.size < 0) {
|
|
185
2814
|
throw new Error("backup manifest size is invalid");
|
|
186
2815
|
}
|
|
187
|
-
const manifestDirectory =
|
|
188
|
-
const databasePath =
|
|
189
|
-
if (
|
|
2816
|
+
const manifestDirectory = dirname2(resolvedManifestPath);
|
|
2817
|
+
const databasePath = resolve2(manifestDirectory, manifest.databaseFile);
|
|
2818
|
+
if (dirname2(databasePath) !== manifestDirectory) {
|
|
190
2819
|
throw new Error("backup database file must stay inside the manifest directory");
|
|
191
2820
|
}
|
|
192
|
-
if (!
|
|
2821
|
+
if (!existsSync2(databasePath))
|
|
193
2822
|
throw new Error("backup database file is missing");
|
|
194
|
-
const databaseStat =
|
|
2823
|
+
const databaseStat = lstatSync2(databasePath);
|
|
195
2824
|
if (!databaseStat.isFile() || databaseStat.isSymbolicLink()) {
|
|
196
2825
|
throw new Error("backup database must be a regular file");
|
|
197
2826
|
}
|
|
198
|
-
const
|
|
199
|
-
|
|
200
|
-
if (hash !== manifest.sha256 || databaseStat.size !== manifest.size) {
|
|
2827
|
+
const digest = hashRegularFile(databasePath);
|
|
2828
|
+
if (digest.sha256 !== manifest.sha256 || digest.size !== manifest.size || databaseStat.size !== manifest.size) {
|
|
201
2829
|
throw new Error("backup hash or size mismatch");
|
|
202
2830
|
}
|
|
203
|
-
const db = new
|
|
2831
|
+
const db = new Database2(databasePath, { readonly: true });
|
|
204
2832
|
try {
|
|
205
2833
|
const health = assertHealthyDatabase(db);
|
|
206
2834
|
if (JSON.stringify(health.counts) !== JSON.stringify(manifest.counts)) {
|
|
207
2835
|
throw new Error("backup manifest row counts do not match");
|
|
208
2836
|
}
|
|
2837
|
+
assertManifestSourceSchema(db, health.schemaVersion, manifest.sourceSchema);
|
|
209
2838
|
} finally {
|
|
210
2839
|
db.close();
|
|
211
2840
|
}
|
|
212
2841
|
return { databasePath, manifestPath: resolvedManifestPath, manifest };
|
|
213
2842
|
}
|
|
214
|
-
function restoreVerifiedBackup(manifestPath, targetPath, confirmation) {
|
|
2843
|
+
function restoreVerifiedBackup(manifestPath, targetPath, confirmation, existingGate, expectedSha256) {
|
|
215
2844
|
if (confirmation !== "RESTORE_DATABASE_FROM_VERIFIED_BACKUP") {
|
|
216
2845
|
throw new Error("invalid restore confirmation");
|
|
217
2846
|
}
|
|
2847
|
+
ensureDatabaseParent(targetPath);
|
|
2848
|
+
const gate = existingGate ?? acquireMaintenanceGate(targetPath);
|
|
2849
|
+
const releaseGate = existingGate === undefined;
|
|
2850
|
+
try {
|
|
2851
|
+
return restoreUnderGate(manifestPath, targetPath, gate, expectedSha256);
|
|
2852
|
+
} finally {
|
|
2853
|
+
if (releaseGate)
|
|
2854
|
+
gate.release();
|
|
2855
|
+
}
|
|
2856
|
+
}
|
|
2857
|
+
function restoreUnderGate(manifestPath, targetPath, gate, expectedSha256) {
|
|
2858
|
+
assertMaintenanceGateFor(gate, targetPath);
|
|
218
2859
|
const verified = verifyBackupManifest(manifestPath);
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
2860
|
+
if (expectedSha256 !== undefined && verified.manifest.sha256 !== expectedSha256) {
|
|
2861
|
+
throw new Error("restore manifest hash mismatch");
|
|
2862
|
+
}
|
|
2863
|
+
mkdirSync2(dirname2(targetPath), { recursive: true, mode: 448 });
|
|
2864
|
+
const temporary = `${targetPath}.restore-${randomUUID3()}.tmp`;
|
|
2865
|
+
const preserved = `${targetPath}.failed-restore-source-${Date.now()}-${randomUUID3()}`;
|
|
222
2866
|
const movedSidecars = [];
|
|
223
2867
|
let hasPreservedSource = false;
|
|
224
2868
|
let preservedSourceHealthy = false;
|
|
225
2869
|
let targetInstalled = false;
|
|
226
2870
|
try {
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
fsyncPath(temporary);
|
|
230
|
-
if (existsSync(targetPath)) {
|
|
2871
|
+
copyVerifiedSource(verified, temporary);
|
|
2872
|
+
if (existsSync2(targetPath)) {
|
|
231
2873
|
preservedSourceHealthy = checkpointSource(targetPath);
|
|
232
|
-
|
|
233
|
-
chmodSync(preserved, 384);
|
|
234
|
-
fsyncPath(preserved);
|
|
235
|
-
if (preservedSourceHealthy)
|
|
236
|
-
verifyDatabaseFile(preserved);
|
|
2874
|
+
copyRegularFile(targetPath, preserved);
|
|
237
2875
|
for (const suffix of ["-wal", "-shm"]) {
|
|
238
2876
|
const source = `${targetPath}${suffix}`;
|
|
239
|
-
if (!
|
|
2877
|
+
if (!existsSync2(source))
|
|
240
2878
|
continue;
|
|
241
2879
|
const preservedSidecar = `${preserved}${suffix}`;
|
|
242
|
-
|
|
243
|
-
chmodSync(preservedSidecar, 384);
|
|
244
|
-
fsyncPath(preservedSidecar);
|
|
2880
|
+
copyRegularFile(source, preservedSidecar);
|
|
245
2881
|
}
|
|
246
|
-
|
|
2882
|
+
if (preservedSourceHealthy)
|
|
2883
|
+
verifyDatabaseFile(preserved);
|
|
2884
|
+
fsyncPath2(dirname2(targetPath));
|
|
247
2885
|
hasPreservedSource = true;
|
|
248
2886
|
}
|
|
249
2887
|
for (const suffix of ["-wal", "-shm"]) {
|
|
250
2888
|
const source = `${targetPath}${suffix}`;
|
|
251
|
-
if (!
|
|
2889
|
+
if (!existsSync2(source))
|
|
252
2890
|
continue;
|
|
253
|
-
const quarantine = `${source}.quarantine-${
|
|
254
|
-
|
|
2891
|
+
const quarantine = `${source}.quarantine-${randomUUID3()}`;
|
|
2892
|
+
quarantineRegularFile(source, quarantine);
|
|
255
2893
|
movedSidecars.push({ source, quarantine });
|
|
256
2894
|
}
|
|
257
|
-
|
|
2895
|
+
renameSync2(temporary, targetPath);
|
|
258
2896
|
targetInstalled = true;
|
|
259
|
-
|
|
260
|
-
|
|
2897
|
+
fsyncPath2(dirname2(targetPath));
|
|
2898
|
+
verifyInstalledBackup(targetPath, verified.manifest);
|
|
261
2899
|
for (const { quarantine } of movedSidecars) {
|
|
262
|
-
|
|
2900
|
+
unlinkRegularFile(quarantine);
|
|
263
2901
|
}
|
|
264
2902
|
} catch (error) {
|
|
265
2903
|
const rollbackErrors = [];
|
|
@@ -267,62 +2905,231 @@ function restoreVerifiedBackup(manifestPath, targetPath, confirmation) {
|
|
|
267
2905
|
if (targetInstalled) {
|
|
268
2906
|
try {
|
|
269
2907
|
for (const suffix of ["-wal", "-shm"]) {
|
|
270
|
-
|
|
2908
|
+
unlinkRegularFile(`${targetPath}${suffix}`, true);
|
|
271
2909
|
}
|
|
272
2910
|
if (hasPreservedSource) {
|
|
273
|
-
const rollback = `${targetPath}.rollback-${
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
rmSync(targetPath, { force: true });
|
|
278
|
-
renameSync(rollback, targetPath);
|
|
2911
|
+
const rollback = `${targetPath}.rollback-${randomUUID3()}.tmp`;
|
|
2912
|
+
copyRegularFile(preserved, rollback);
|
|
2913
|
+
unlinkRegularFile(targetPath);
|
|
2914
|
+
renameSync2(rollback, targetPath);
|
|
279
2915
|
} else {
|
|
280
|
-
|
|
2916
|
+
unlinkRegularFile(targetPath);
|
|
281
2917
|
}
|
|
282
2918
|
} catch (rollbackError) {
|
|
283
2919
|
rollbackErrors.push(rollbackError);
|
|
284
2920
|
}
|
|
285
2921
|
}
|
|
286
2922
|
for (const { source, quarantine } of movedSidecars.reverse()) {
|
|
287
|
-
if (!
|
|
2923
|
+
if (!existsSync2(quarantine))
|
|
288
2924
|
continue;
|
|
289
2925
|
try {
|
|
290
|
-
|
|
291
|
-
|
|
2926
|
+
if (existsSync2(source))
|
|
2927
|
+
throw new Error("restore sidecar path was replaced during rollback");
|
|
2928
|
+
renameSync2(quarantine, source);
|
|
292
2929
|
} catch (rollbackError) {
|
|
293
2930
|
rollbackErrors.push(rollbackError);
|
|
294
2931
|
}
|
|
295
2932
|
}
|
|
296
2933
|
if (targetInstalled && hasPreservedSource && preservedSourceHealthy && rollbackErrors.length === 0) {
|
|
297
2934
|
try {
|
|
298
|
-
|
|
2935
|
+
fsyncPath2(dirname2(targetPath));
|
|
299
2936
|
verifyDatabaseFile(targetPath);
|
|
300
2937
|
} catch (rollbackError) {
|
|
301
2938
|
rollbackErrors.push(rollbackError);
|
|
302
2939
|
}
|
|
303
2940
|
}
|
|
304
2941
|
if (rollbackErrors.length > 0) {
|
|
2942
|
+
gate.retain();
|
|
305
2943
|
throw new AggregateError([error, ...rollbackErrors], "restore failed and rollback was incomplete");
|
|
306
2944
|
}
|
|
307
2945
|
throw error;
|
|
308
2946
|
}
|
|
309
|
-
return preserved;
|
|
2947
|
+
return preserved;
|
|
2948
|
+
}
|
|
2949
|
+
function copyVerifiedSource(verified, target) {
|
|
2950
|
+
assertNoSymbolicLinks(verified.databasePath);
|
|
2951
|
+
const source = openSync2(verified.databasePath, constants2.O_RDONLY | constants2.O_NOFOLLOW);
|
|
2952
|
+
let destination;
|
|
2953
|
+
try {
|
|
2954
|
+
const before = fstatSync(source);
|
|
2955
|
+
if (!before.isFile())
|
|
2956
|
+
throw new Error("backup database must be a regular file");
|
|
2957
|
+
destination = openSync2(target, constants2.O_WRONLY | constants2.O_CREAT | constants2.O_EXCL | constants2.O_NOFOLLOW, 384);
|
|
2958
|
+
const hash = createHash4("sha256");
|
|
2959
|
+
const buffer = Buffer.allocUnsafe(64 * 1024);
|
|
2960
|
+
let size = 0;
|
|
2961
|
+
while (true) {
|
|
2962
|
+
const read = readSync(source, buffer, 0, buffer.byteLength, null);
|
|
2963
|
+
if (read === 0)
|
|
2964
|
+
break;
|
|
2965
|
+
hash.update(buffer.subarray(0, read));
|
|
2966
|
+
let written = 0;
|
|
2967
|
+
while (written < read) {
|
|
2968
|
+
written += writeSync(destination, buffer, written, read - written);
|
|
2969
|
+
}
|
|
2970
|
+
size += read;
|
|
2971
|
+
}
|
|
2972
|
+
const after = fstatSync(source);
|
|
2973
|
+
if (before.dev !== after.dev || before.ino !== after.ino || before.size !== after.size) {
|
|
2974
|
+
throw new Error("backup database changed while it was copied");
|
|
2975
|
+
}
|
|
2976
|
+
if (size !== verified.manifest.size || hash.digest("hex") !== verified.manifest.sha256) {
|
|
2977
|
+
throw new Error("backup hash or size mismatch during restore copy");
|
|
2978
|
+
}
|
|
2979
|
+
fsyncSync2(destination);
|
|
2980
|
+
} finally {
|
|
2981
|
+
if (destination !== undefined)
|
|
2982
|
+
closeSync2(destination);
|
|
2983
|
+
closeSync2(source);
|
|
2984
|
+
}
|
|
2985
|
+
chmodSync2(target, 384);
|
|
2986
|
+
verifyDatabaseMatchesManifest(target, verified.manifest);
|
|
2987
|
+
}
|
|
2988
|
+
function verifyDatabaseMatchesManifest(path, manifest) {
|
|
2989
|
+
const db = new Database2(path, { readonly: true });
|
|
2990
|
+
try {
|
|
2991
|
+
const health = assertHealthyDatabase(db);
|
|
2992
|
+
assertManifestSourceSchema(db, health.schemaVersion, manifest.sourceSchema);
|
|
2993
|
+
if (JSON.stringify(health.counts) !== JSON.stringify(manifest.counts)) {
|
|
2994
|
+
throw new Error("backup manifest row counts do not match");
|
|
2995
|
+
}
|
|
2996
|
+
} finally {
|
|
2997
|
+
db.close();
|
|
2998
|
+
}
|
|
2999
|
+
}
|
|
3000
|
+
function verifyInstalledBackup(path, manifest) {
|
|
3001
|
+
const digest = hashRegularFile(path);
|
|
3002
|
+
if (digest.size !== manifest.size) {
|
|
3003
|
+
throw new Error("installed backup size does not match manifest");
|
|
3004
|
+
}
|
|
3005
|
+
if (digest.sha256 !== manifest.sha256) {
|
|
3006
|
+
throw new Error("installed backup hash does not match manifest");
|
|
3007
|
+
}
|
|
3008
|
+
verifyDatabaseMatchesManifest(path, manifest);
|
|
3009
|
+
}
|
|
3010
|
+
function assertManifestSourceSchema(db, actualSchema, sourceSchema) {
|
|
3011
|
+
if (!Number.isSafeInteger(sourceSchema) || sourceSchema < 2 || sourceSchema > 11) {
|
|
3012
|
+
throw new Error("backup manifest source schema does not match database");
|
|
3013
|
+
}
|
|
3014
|
+
if (actualSchema === sourceSchema) {
|
|
3015
|
+
if (sourceSchema >= 2 && sourceSchema <= 10) {
|
|
3016
|
+
try {
|
|
3017
|
+
assertLegacySchemaIdentity(db, sourceSchema);
|
|
3018
|
+
} catch {
|
|
3019
|
+
throw new Error("backup manifest source schema does not match database");
|
|
3020
|
+
}
|
|
3021
|
+
}
|
|
3022
|
+
return;
|
|
3023
|
+
}
|
|
3024
|
+
if (sourceSchema === 2 && actualSchema === undefined && hasTable3(db, "memory_items")) {
|
|
3025
|
+
try {
|
|
3026
|
+
assertLegacySchemaIdentity(db, sourceSchema);
|
|
3027
|
+
return;
|
|
3028
|
+
} catch {
|
|
3029
|
+
throw new Error("backup manifest source schema does not match database");
|
|
3030
|
+
}
|
|
3031
|
+
}
|
|
3032
|
+
throw new Error("backup manifest source schema does not match database");
|
|
3033
|
+
}
|
|
3034
|
+
function hasTable3(db, table) {
|
|
3035
|
+
return db.query("SELECT COUNT(*) AS count FROM sqlite_master WHERE type = 'table' AND name = ?").get(table).count > 0;
|
|
3036
|
+
}
|
|
3037
|
+
function hashRegularFile(path) {
|
|
3038
|
+
const descriptor = openSync2(path, constants2.O_RDONLY | constants2.O_NOFOLLOW);
|
|
3039
|
+
try {
|
|
3040
|
+
const before = fstatSync(descriptor);
|
|
3041
|
+
if (!before.isFile())
|
|
3042
|
+
throw new Error(`${path} must be a regular file`);
|
|
3043
|
+
const hash = createHash4("sha256");
|
|
3044
|
+
const buffer = Buffer.allocUnsafe(64 * 1024);
|
|
3045
|
+
let size = 0;
|
|
3046
|
+
while (true) {
|
|
3047
|
+
const read = readSync(descriptor, buffer, 0, buffer.byteLength, null);
|
|
3048
|
+
if (read === 0)
|
|
3049
|
+
break;
|
|
3050
|
+
hash.update(buffer.subarray(0, read));
|
|
3051
|
+
size += read;
|
|
3052
|
+
}
|
|
3053
|
+
const after = fstatSync(descriptor);
|
|
3054
|
+
if (before.dev !== after.dev || before.ino !== after.ino || before.size !== after.size || size !== after.size) {
|
|
3055
|
+
throw new Error(`${path} changed while it was hashed`);
|
|
3056
|
+
}
|
|
3057
|
+
return { sha256: hash.digest("hex"), size };
|
|
3058
|
+
} finally {
|
|
3059
|
+
closeSync2(descriptor);
|
|
3060
|
+
}
|
|
3061
|
+
}
|
|
3062
|
+
function copyRegularFile(sourcePath, targetPath) {
|
|
3063
|
+
assertNoSymbolicLinks(sourcePath);
|
|
3064
|
+
const source = openSync2(sourcePath, constants2.O_RDONLY | constants2.O_NOFOLLOW);
|
|
3065
|
+
let target;
|
|
3066
|
+
try {
|
|
3067
|
+
const before = fstatSync(source);
|
|
3068
|
+
if (!before.isFile())
|
|
3069
|
+
throw new Error(`${sourcePath} must be a regular file`);
|
|
3070
|
+
target = openSync2(targetPath, constants2.O_WRONLY | constants2.O_CREAT | constants2.O_EXCL | constants2.O_NOFOLLOW, 384);
|
|
3071
|
+
const buffer = Buffer.allocUnsafe(64 * 1024);
|
|
3072
|
+
while (true) {
|
|
3073
|
+
const read = readSync(source, buffer, 0, buffer.byteLength, null);
|
|
3074
|
+
if (read === 0)
|
|
3075
|
+
break;
|
|
3076
|
+
let written = 0;
|
|
3077
|
+
while (written < read)
|
|
3078
|
+
written += writeSync(target, buffer, written, read - written);
|
|
3079
|
+
}
|
|
3080
|
+
const after = fstatSync(source);
|
|
3081
|
+
if (before.dev !== after.dev || before.ino !== after.ino || before.size !== after.size) {
|
|
3082
|
+
throw new Error(`${sourcePath} changed while it was copied`);
|
|
3083
|
+
}
|
|
3084
|
+
fsyncSync2(target);
|
|
3085
|
+
} finally {
|
|
3086
|
+
if (target !== undefined)
|
|
3087
|
+
closeSync2(target);
|
|
3088
|
+
closeSync2(source);
|
|
3089
|
+
}
|
|
3090
|
+
}
|
|
3091
|
+
function quarantineRegularFile(source, quarantine) {
|
|
3092
|
+
const before = lstatSync2(source);
|
|
3093
|
+
if (!before.isFile() || before.isSymbolicLink()) {
|
|
3094
|
+
throw new Error("database sidecar must be a regular file");
|
|
3095
|
+
}
|
|
3096
|
+
renameSync2(source, quarantine);
|
|
3097
|
+
const after = lstatSync2(quarantine);
|
|
3098
|
+
if (!after.isFile() || after.isSymbolicLink() || before.dev !== after.dev || before.ino !== after.ino) {
|
|
3099
|
+
throw new Error("database sidecar changed while it was quarantined");
|
|
3100
|
+
}
|
|
3101
|
+
}
|
|
3102
|
+
function unlinkRegularFile(path, allowMissing = false) {
|
|
3103
|
+
try {
|
|
3104
|
+
const stat = lstatSync2(path);
|
|
3105
|
+
if (!stat.isFile() || stat.isSymbolicLink())
|
|
3106
|
+
throw new Error(`${path} must be a regular file`);
|
|
3107
|
+
rmSync(path);
|
|
3108
|
+
} catch (error) {
|
|
3109
|
+
if (allowMissing && error && typeof error === "object" && "code" in error && error.code === "ENOENT") {
|
|
3110
|
+
return;
|
|
3111
|
+
}
|
|
3112
|
+
throw error;
|
|
3113
|
+
}
|
|
310
3114
|
}
|
|
311
3115
|
function escapeSql(value) {
|
|
312
3116
|
return value.replaceAll("'", "''");
|
|
313
3117
|
}
|
|
314
|
-
function
|
|
315
|
-
const
|
|
3118
|
+
function fsyncPath2(path) {
|
|
3119
|
+
const stat = lstatSync2(path);
|
|
3120
|
+
if (platform2() === "win32" && stat.isDirectory())
|
|
3121
|
+
return;
|
|
3122
|
+
const descriptor = openSync2(path, platform2() === "win32" ? "r+" : "r");
|
|
316
3123
|
try {
|
|
317
|
-
|
|
3124
|
+
fsyncSync2(descriptor);
|
|
318
3125
|
} finally {
|
|
319
|
-
|
|
3126
|
+
closeSync2(descriptor);
|
|
320
3127
|
}
|
|
321
3128
|
}
|
|
322
3129
|
function checkpointSource(path) {
|
|
323
3130
|
let db;
|
|
324
3131
|
try {
|
|
325
|
-
db = new
|
|
3132
|
+
db = new Database2(path);
|
|
326
3133
|
const checkpoint = db.query("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
327
3134
|
if (checkpoint.busy !== 0)
|
|
328
3135
|
throw new Error("source database WAL checkpoint is busy");
|
|
@@ -341,7 +3148,7 @@ function checkpointSource(path) {
|
|
|
341
3148
|
}
|
|
342
3149
|
}
|
|
343
3150
|
function verifyDatabaseFile(path) {
|
|
344
|
-
const db = new
|
|
3151
|
+
const db = new Database2(path, { readonly: true });
|
|
345
3152
|
try {
|
|
346
3153
|
assertHealthyDatabase(db);
|
|
347
3154
|
} finally {
|
|
@@ -358,40 +3165,40 @@ function isBusyError(error) {
|
|
|
358
3165
|
}
|
|
359
3166
|
|
|
360
3167
|
// src/db/migration-lock.ts
|
|
361
|
-
import { randomUUID as
|
|
362
|
-
import { existsSync as
|
|
363
|
-
import { hostname } from "os";
|
|
3168
|
+
import { randomUUID as randomUUID4 } from "crypto";
|
|
3169
|
+
import { existsSync as existsSync3, mkdirSync as mkdirSync3, readFileSync as readFileSync3, renameSync as renameSync3, rmSync as rmSync2, writeFileSync as writeFileSync3 } from "fs";
|
|
3170
|
+
import { hostname as hostname2 } from "os";
|
|
364
3171
|
function migrationLockPath(databasePath) {
|
|
365
3172
|
return `${databasePath}.migration.lock`;
|
|
366
3173
|
}
|
|
367
3174
|
function acquireMigrationLock(databasePath, targetSchema, timeoutMs = 30000) {
|
|
368
3175
|
const path = migrationLockPath(databasePath);
|
|
369
3176
|
const owner = {
|
|
370
|
-
ownerID:
|
|
3177
|
+
ownerID: randomUUID4(),
|
|
371
3178
|
pid: process.pid,
|
|
372
|
-
processStartMarker:
|
|
373
|
-
hostname:
|
|
3179
|
+
processStartMarker: processStartMarker2(process.pid) ?? "unavailable",
|
|
3180
|
+
hostname: hostname2(),
|
|
374
3181
|
startedAt: Date.now(),
|
|
375
3182
|
targetSchema
|
|
376
3183
|
};
|
|
377
3184
|
const deadline = Date.now() + timeoutMs;
|
|
378
3185
|
const stagedOwner = `${path}.owner-${owner.ownerID}.tmp`;
|
|
379
|
-
|
|
3186
|
+
writeFileSync3(stagedOwner, `${JSON.stringify(owner, null, 2)}
|
|
380
3187
|
`, { mode: 384 });
|
|
381
3188
|
try {
|
|
382
3189
|
while (true) {
|
|
383
3190
|
let created = false;
|
|
384
3191
|
try {
|
|
385
|
-
|
|
3192
|
+
mkdirSync3(path, { mode: 448 });
|
|
386
3193
|
created = true;
|
|
387
|
-
|
|
3194
|
+
renameSync3(stagedOwner, `${path}/owner.json`);
|
|
388
3195
|
break;
|
|
389
3196
|
} catch (error) {
|
|
390
3197
|
if (created) {
|
|
391
3198
|
rmSync2(path, { recursive: true, force: true });
|
|
392
3199
|
throw error;
|
|
393
3200
|
}
|
|
394
|
-
if (!
|
|
3201
|
+
if (!isAlreadyExistsError(error))
|
|
395
3202
|
throw error;
|
|
396
3203
|
if (Date.now() >= deadline) {
|
|
397
3204
|
const current = readMigrationLockOwner(path);
|
|
@@ -421,7 +3228,7 @@ function acquireMigrationLock(databasePath, targetSchema, timeoutMs = 30000) {
|
|
|
421
3228
|
}
|
|
422
3229
|
function readMigrationLockOwner(path) {
|
|
423
3230
|
try {
|
|
424
|
-
return JSON.parse(
|
|
3231
|
+
return JSON.parse(readFileSync3(`${path}/owner.json`, "utf8"));
|
|
425
3232
|
} catch {
|
|
426
3233
|
return;
|
|
427
3234
|
}
|
|
@@ -433,300 +3240,43 @@ function breakMigrationLock(databasePath, ownerID, confirmation) {
|
|
|
433
3240
|
const path = migrationLockPath(databasePath);
|
|
434
3241
|
const owner = readMigrationLockOwner(path);
|
|
435
3242
|
if (!owner) {
|
|
436
|
-
if (ownerID !== "ORPHANED" || !
|
|
3243
|
+
if (ownerID !== "ORPHANED" || !existsSync3(path))
|
|
437
3244
|
throw new Error("migration lock owner mismatch");
|
|
438
3245
|
rmSync2(path, { recursive: true, force: true });
|
|
439
3246
|
return;
|
|
440
3247
|
}
|
|
441
3248
|
if (owner.ownerID !== ownerID)
|
|
442
3249
|
throw new Error("migration lock owner mismatch");
|
|
443
|
-
if (owner.hostname ===
|
|
444
|
-
const currentMarker =
|
|
3250
|
+
if (owner.hostname === hostname2() && processIsAlive(owner.pid)) {
|
|
3251
|
+
const currentMarker = processStartMarker2(owner.pid);
|
|
445
3252
|
if (!currentMarker || currentMarker === owner.processStartMarker) {
|
|
446
3253
|
throw new Error(`migration lock process ${owner.pid} is still alive`);
|
|
447
3254
|
}
|
|
448
3255
|
}
|
|
449
3256
|
rmSync2(path, { recursive: true, force: true });
|
|
450
3257
|
}
|
|
451
|
-
function
|
|
3258
|
+
function processStartMarker2(pid) {
|
|
452
3259
|
try {
|
|
453
|
-
const fields =
|
|
3260
|
+
const fields = readFileSync3(`/proc/${pid}/stat`, "utf8").trim().split(/\s+/);
|
|
454
3261
|
return fields[21];
|
|
455
3262
|
} catch {
|
|
456
3263
|
return;
|
|
457
3264
|
}
|
|
458
3265
|
}
|
|
459
3266
|
function processIsAlive(pid) {
|
|
460
|
-
try {
|
|
461
|
-
process.kill(pid, 0);
|
|
462
|
-
return true;
|
|
463
|
-
} catch {
|
|
464
|
-
return false;
|
|
465
|
-
}
|
|
466
|
-
}
|
|
467
|
-
|
|
468
|
-
// src/db/migrations/v009.ts
|
|
469
|
-
import { createHash as createHash3, randomUUID as randomUUID3 } from "crypto";
|
|
470
|
-
|
|
471
|
-
// src/capture/contract.ts
|
|
472
|
-
import * as z from "zod/v4";
|
|
473
|
-
var CAPTURE_SCHEMA = "agz-memory.capture/1";
|
|
474
|
-
var SUPPORTED_OPENCODE_VERSION = "0.0.0-beta-18743";
|
|
475
|
-
var CAPTURE_EVENT_MAX_BYTES = 16 * 1024;
|
|
476
|
-
var CAPTURE_CONTENT_MAX_CHARACTERS = 4800;
|
|
477
|
-
var CAPTURE_SUMMARY_MAX_CHARACTERS = 1200;
|
|
478
|
-
var CAPTURE_SUBJECT_MAX_CHARACTERS = 240;
|
|
479
|
-
var candidateSchema = z.object({
|
|
480
|
-
kind: z.enum(KINDS),
|
|
481
|
-
title: z.string().min(1).max(240),
|
|
482
|
-
summary: z.string().min(1).max(CAPTURE_SUMMARY_MAX_CHARACTERS),
|
|
483
|
-
content: z.string().min(1).max(CAPTURE_CONTENT_MAX_CHARACTERS),
|
|
484
|
-
subjectKey: z.string().min(1).max(CAPTURE_SUBJECT_MAX_CHARACTERS).optional(),
|
|
485
|
-
intent: z.enum(["create", "supersede", "ignore", "review"]),
|
|
486
|
-
targetNoteID: z.string().min(1).max(240).optional(),
|
|
487
|
-
confidence: z.number().finite().min(0).max(1),
|
|
488
|
-
evidence: z.enum(["explicit-user", "verified-outcome", "session-summary"])
|
|
489
|
-
}).strict();
|
|
490
|
-
var signalSchema = z.object({
|
|
491
|
-
tool: z.string().min(1).max(160),
|
|
492
|
-
status: z.enum(["completed", "error"]),
|
|
493
|
-
errorType: z.string().min(1).max(160).optional()
|
|
494
|
-
}).strict();
|
|
495
|
-
var captureEventSchema = z.object({
|
|
496
|
-
schema: z.literal(CAPTURE_SCHEMA),
|
|
497
|
-
idempotencyKey: z.string().regex(/^[0-9a-f]{64}$/),
|
|
498
|
-
projectID: z.uuid(),
|
|
499
|
-
bindingKey: z.string().regex(/^[0-9a-f]{64}$/),
|
|
500
|
-
kind: z.enum(["user-candidate", "assistant-candidate", "session-summary", "tool-signal"]),
|
|
501
|
-
source: z.object({
|
|
502
|
-
system: z.literal("opencode-v2"),
|
|
503
|
-
opencodeVersion: z.literal(SUPPORTED_OPENCODE_VERSION),
|
|
504
|
-
pluginVersion: z.string().min(1).max(80),
|
|
505
|
-
sessionID: z.string().min(1).max(240),
|
|
506
|
-
messageID: z.string().min(1).max(240).optional(),
|
|
507
|
-
ordinal: z.number().int().nonnegative().optional(),
|
|
508
|
-
toolCallID: z.string().min(1).max(240).optional(),
|
|
509
|
-
observedAt: z.number().int().nonnegative()
|
|
510
|
-
}).strict(),
|
|
511
|
-
candidate: candidateSchema.optional(),
|
|
512
|
-
signal: signalSchema.optional(),
|
|
513
|
-
redaction: z.object({
|
|
514
|
-
policyVersion: z.string().min(1).max(80),
|
|
515
|
-
replacements: z.number().int().nonnegative(),
|
|
516
|
-
truncated: z.boolean()
|
|
517
|
-
}).strict()
|
|
518
|
-
}).strict().superRefine((event, context) => {
|
|
519
|
-
if (event.kind === "tool-signal" && !event.signal) {
|
|
520
|
-
context.addIssue({ code: "custom", message: "tool-signal requires signal" });
|
|
521
|
-
}
|
|
522
|
-
if (event.kind !== "tool-signal" && !event.candidate) {
|
|
523
|
-
context.addIssue({ code: "custom", message: `${event.kind} requires candidate` });
|
|
524
|
-
}
|
|
525
|
-
if (event.kind === "tool-signal" && event.candidate) {
|
|
526
|
-
context.addIssue({ code: "custom", message: "tool-signal cannot carry candidate" });
|
|
527
|
-
}
|
|
528
|
-
if (event.kind !== "tool-signal" && event.signal) {
|
|
529
|
-
context.addIssue({ code: "custom", message: `${event.kind} cannot carry signal` });
|
|
530
|
-
}
|
|
531
|
-
});
|
|
532
|
-
function parseCaptureEvent(value) {
|
|
533
|
-
const event = captureEventSchema.parse(value);
|
|
534
|
-
if (Buffer.byteLength(JSON.stringify(event), "utf8") > CAPTURE_EVENT_MAX_BYTES) {
|
|
535
|
-
throw new Error(`capture event exceeds ${CAPTURE_EVENT_MAX_BYTES} UTF-8 bytes`);
|
|
536
|
-
}
|
|
537
|
-
return event;
|
|
538
|
-
}
|
|
539
|
-
|
|
540
|
-
// src/db/schema.ts
|
|
541
|
-
var SCHEMA_TABLES = `
|
|
542
|
-
CREATE TABLE IF NOT EXISTS projects (
|
|
543
|
-
id TEXT PRIMARY KEY,
|
|
544
|
-
name TEXT NOT NULL,
|
|
545
|
-
normalized_name TEXT NOT NULL UNIQUE,
|
|
546
|
-
created_at INTEGER NOT NULL,
|
|
547
|
-
updated_at INTEGER NOT NULL
|
|
548
|
-
);
|
|
549
|
-
CREATE TABLE IF NOT EXISTS notes (
|
|
550
|
-
id TEXT PRIMARY KEY,
|
|
551
|
-
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
552
|
-
kind TEXT NOT NULL CHECK (kind IN ('decision','fact','procedure','context','research','preference','task')),
|
|
553
|
-
title TEXT NOT NULL,
|
|
554
|
-
summary TEXT NOT NULL,
|
|
555
|
-
content TEXT NOT NULL,
|
|
556
|
-
size_class TEXT NOT NULL CHECK (size_class IN ('inline','indexed')),
|
|
557
|
-
pinned INTEGER NOT NULL DEFAULT 0 CHECK (pinned IN (0,1)),
|
|
558
|
-
status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('active','superseded','archived')),
|
|
559
|
-
supersedes_id TEXT,
|
|
560
|
-
current_revision INTEGER NOT NULL DEFAULT 1 CHECK (current_revision >= 1),
|
|
561
|
-
subject_key TEXT,
|
|
562
|
-
content_hash TEXT NOT NULL CHECK (length(content_hash) = 64),
|
|
563
|
-
created_at INTEGER NOT NULL,
|
|
564
|
-
updated_at INTEGER NOT NULL,
|
|
565
|
-
UNIQUE(project_id, id)
|
|
566
|
-
);
|
|
567
|
-
CREATE INDEX IF NOT EXISTS notes_project_idx ON notes(project_id, status);
|
|
568
|
-
CREATE UNIQUE INDEX IF NOT EXISTS notes_active_subject_idx
|
|
569
|
-
ON notes(project_id, kind, subject_key)
|
|
570
|
-
WHERE status = 'active' AND subject_key IS NOT NULL;
|
|
571
|
-
CREATE TABLE IF NOT EXISTS note_edges (
|
|
572
|
-
id TEXT PRIMARY KEY,
|
|
573
|
-
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
574
|
-
source_id TEXT NOT NULL,
|
|
575
|
-
target_id TEXT NOT NULL,
|
|
576
|
-
predicate TEXT NOT NULL CHECK (predicate IN ('SUPPORTS','DERIVED_FROM','PART_OF','ABOUT','PRECEDES','SUPERSEDES')),
|
|
577
|
-
created_at INTEGER NOT NULL,
|
|
578
|
-
UNIQUE(project_id, source_id, target_id, predicate),
|
|
579
|
-
FOREIGN KEY (project_id, source_id) REFERENCES notes(project_id, id) ON DELETE CASCADE,
|
|
580
|
-
FOREIGN KEY (project_id, target_id) REFERENCES notes(project_id, id) ON DELETE CASCADE
|
|
581
|
-
);
|
|
582
|
-
CREATE INDEX IF NOT EXISTS note_edges_source_idx ON note_edges(project_id, source_id);
|
|
583
|
-
CREATE INDEX IF NOT EXISTS note_edges_target_idx ON note_edges(project_id, target_id);
|
|
584
|
-
CREATE TABLE IF NOT EXISTS project_bindings (
|
|
585
|
-
binding_key TEXT PRIMARY KEY,
|
|
586
|
-
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
587
|
-
source TEXT NOT NULL CHECK (source = 'opencode-v2'),
|
|
588
|
-
source_project_id TEXT NOT NULL,
|
|
589
|
-
workspace_id TEXT NOT NULL,
|
|
590
|
-
canonical_path_hash TEXT NOT NULL CHECK (length(canonical_path_hash) = 64),
|
|
591
|
-
created_at INTEGER NOT NULL,
|
|
592
|
-
updated_at INTEGER NOT NULL,
|
|
593
|
-
UNIQUE(source, source_project_id, workspace_id)
|
|
594
|
-
);
|
|
595
|
-
CREATE TABLE IF NOT EXISTS capture_checkpoints (
|
|
596
|
-
session_id TEXT PRIMARY KEY,
|
|
597
|
-
binding_key TEXT NOT NULL REFERENCES project_bindings(binding_key) ON DELETE CASCADE,
|
|
598
|
-
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
599
|
-
state TEXT NOT NULL CHECK (state IN ('active','idle','unavailable','closed')),
|
|
600
|
-
last_message_id TEXT,
|
|
601
|
-
last_reconciled_at INTEGER,
|
|
602
|
-
next_reconcile_at INTEGER NOT NULL,
|
|
603
|
-
failure_count INTEGER NOT NULL DEFAULT 0 CHECK (failure_count >= 0),
|
|
604
|
-
lease_owner TEXT,
|
|
605
|
-
lease_expires_at INTEGER,
|
|
606
|
-
created_at INTEGER NOT NULL,
|
|
607
|
-
updated_at INTEGER NOT NULL
|
|
608
|
-
);
|
|
609
|
-
CREATE INDEX IF NOT EXISTS capture_checkpoints_due_idx
|
|
610
|
-
ON capture_checkpoints(state, next_reconcile_at);
|
|
611
|
-
${captureEventsTable()}
|
|
612
|
-
CREATE INDEX IF NOT EXISTS capture_events_state_idx ON capture_events(state, updated_at);
|
|
613
|
-
CREATE INDEX IF NOT EXISTS capture_events_session_idx ON capture_events(project_id, source_session_id);
|
|
614
|
-
CREATE INDEX IF NOT EXISTS capture_events_note_idx ON capture_events(project_id, note_id);
|
|
615
|
-
CREATE TABLE IF NOT EXISTS note_provenance (
|
|
616
|
-
id TEXT PRIMARY KEY,
|
|
617
|
-
project_id TEXT NOT NULL,
|
|
618
|
-
note_id TEXT NOT NULL,
|
|
619
|
-
source_type TEXT NOT NULL CHECK (source_type IN ('mcp-manual','opencode-capture','migration','legacy-import','admin')),
|
|
620
|
-
capture_event_id TEXT,
|
|
621
|
-
source_session_id TEXT,
|
|
622
|
-
source_message_id TEXT,
|
|
623
|
-
source_ordinal INTEGER,
|
|
624
|
-
source_tool_call_id TEXT,
|
|
625
|
-
redaction_version TEXT,
|
|
626
|
-
extractor_version TEXT,
|
|
627
|
-
confidence REAL CHECK (confidence IS NULL OR (confidence >= 0 AND confidence <= 1)),
|
|
628
|
-
created_at INTEGER NOT NULL,
|
|
629
|
-
UNIQUE(project_id, id),
|
|
630
|
-
FOREIGN KEY (project_id, note_id) REFERENCES notes(project_id, id) ON DELETE CASCADE
|
|
631
|
-
);
|
|
632
|
-
CREATE TABLE IF NOT EXISTS note_revisions (
|
|
633
|
-
project_id TEXT NOT NULL,
|
|
634
|
-
note_id TEXT NOT NULL,
|
|
635
|
-
revision INTEGER NOT NULL CHECK (revision >= 1),
|
|
636
|
-
kind TEXT NOT NULL,
|
|
637
|
-
title TEXT NOT NULL,
|
|
638
|
-
summary TEXT NOT NULL,
|
|
639
|
-
content TEXT NOT NULL,
|
|
640
|
-
size_class TEXT NOT NULL,
|
|
641
|
-
pinned INTEGER NOT NULL CHECK (pinned IN (0,1)),
|
|
642
|
-
status TEXT NOT NULL CHECK (status IN ('active','superseded','archived')),
|
|
643
|
-
supersedes_id TEXT,
|
|
644
|
-
subject_key TEXT,
|
|
645
|
-
content_hash TEXT NOT NULL,
|
|
646
|
-
provenance_id TEXT NOT NULL,
|
|
647
|
-
created_at INTEGER NOT NULL,
|
|
648
|
-
PRIMARY KEY(project_id, note_id, revision),
|
|
649
|
-
FOREIGN KEY (project_id, note_id) REFERENCES notes(project_id, id) ON DELETE CASCADE,
|
|
650
|
-
FOREIGN KEY (project_id, provenance_id) REFERENCES note_provenance(project_id, id)
|
|
651
|
-
);
|
|
652
|
-
CREATE TABLE IF NOT EXISTS index_outbox (
|
|
653
|
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
654
|
-
backend TEXT NOT NULL,
|
|
655
|
-
operation TEXT NOT NULL CHECK (operation IN ('upsert-note','delete-note','purge-project')),
|
|
656
|
-
project_id TEXT NOT NULL,
|
|
657
|
-
note_id TEXT,
|
|
658
|
-
revision INTEGER,
|
|
659
|
-
content_hash TEXT,
|
|
660
|
-
state TEXT NOT NULL DEFAULT 'pending' CHECK (state IN ('pending','leased','succeeded','dead')),
|
|
661
|
-
attempt_count INTEGER NOT NULL DEFAULT 0 CHECK (attempt_count >= 0),
|
|
662
|
-
available_at INTEGER NOT NULL,
|
|
663
|
-
lease_owner TEXT,
|
|
664
|
-
lease_expires_at INTEGER,
|
|
665
|
-
last_error_code TEXT,
|
|
666
|
-
created_at INTEGER NOT NULL,
|
|
667
|
-
completed_at INTEGER,
|
|
668
|
-
UNIQUE(backend, operation, project_id, note_id, revision)
|
|
669
|
-
);
|
|
670
|
-
CREATE INDEX IF NOT EXISTS index_outbox_due_idx
|
|
671
|
-
ON index_outbox(backend, project_id, state, available_at, id);
|
|
672
|
-
CREATE TABLE IF NOT EXISTS schema_state (version INTEGER PRIMARY KEY);
|
|
673
|
-
`;
|
|
674
|
-
var FTS_V9 = `
|
|
675
|
-
CREATE VIRTUAL TABLE IF NOT EXISTS notes_fts USING fts5(
|
|
676
|
-
title, summary, content,
|
|
677
|
-
content='notes', content_rowid='rowid',
|
|
678
|
-
tokenize='unicode61'
|
|
679
|
-
);
|
|
680
|
-
CREATE TRIGGER IF NOT EXISTS notes_fts_ai AFTER INSERT ON notes BEGIN
|
|
681
|
-
INSERT INTO notes_fts(rowid, title, summary, content)
|
|
682
|
-
VALUES (new.rowid, new.title, new.summary, new.content);
|
|
683
|
-
END;
|
|
684
|
-
CREATE TRIGGER IF NOT EXISTS notes_fts_ad AFTER DELETE ON notes BEGIN
|
|
685
|
-
INSERT INTO notes_fts(notes_fts, rowid, title, summary, content)
|
|
686
|
-
VALUES ('delete', old.rowid, old.title, old.summary, old.content);
|
|
687
|
-
END;
|
|
688
|
-
CREATE TRIGGER IF NOT EXISTS notes_fts_au AFTER UPDATE OF title, summary, content ON notes BEGIN
|
|
689
|
-
INSERT INTO notes_fts(notes_fts, rowid, title, summary, content)
|
|
690
|
-
VALUES ('delete', old.rowid, old.title, old.summary, old.content);
|
|
691
|
-
INSERT INTO notes_fts(rowid, title, summary, content)
|
|
692
|
-
VALUES (new.rowid, new.title, new.summary, new.content);
|
|
693
|
-
END;
|
|
694
|
-
`;
|
|
695
|
-
function createSchema(db) {
|
|
696
|
-
db.exec(SCHEMA_TABLES);
|
|
697
|
-
db.exec(FTS_V9);
|
|
698
|
-
db.query("DELETE FROM schema_state").run();
|
|
699
|
-
db.query("INSERT INTO schema_state(version) VALUES (10)").run();
|
|
700
|
-
}
|
|
701
|
-
function rebuildFts(db) {
|
|
702
|
-
db.query("INSERT INTO notes_fts(notes_fts) VALUES ('rebuild')").run();
|
|
3267
|
+
try {
|
|
3268
|
+
process.kill(pid, 0);
|
|
3269
|
+
return true;
|
|
3270
|
+
} catch {
|
|
3271
|
+
return false;
|
|
3272
|
+
}
|
|
703
3273
|
}
|
|
704
|
-
function
|
|
705
|
-
return
|
|
706
|
-
idempotency_key TEXT PRIMARY KEY,
|
|
707
|
-
contract TEXT NOT NULL CHECK (contract = '${CAPTURE_SCHEMA}'),
|
|
708
|
-
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
709
|
-
binding_key TEXT NOT NULL REFERENCES project_bindings(binding_key) ON DELETE CASCADE,
|
|
710
|
-
event_kind TEXT NOT NULL CHECK (event_kind IN ('user-candidate','assistant-candidate','session-summary','tool-signal')),
|
|
711
|
-
source_session_id TEXT NOT NULL,
|
|
712
|
-
source_message_id TEXT,
|
|
713
|
-
source_ordinal INTEGER CHECK (source_ordinal IS NULL OR source_ordinal >= 0),
|
|
714
|
-
source_tool_call_id TEXT,
|
|
715
|
-
payload_json TEXT CHECK (payload_json IS NULL OR json_valid(payload_json)),
|
|
716
|
-
payload_hash TEXT,
|
|
717
|
-
redaction_version TEXT NOT NULL,
|
|
718
|
-
state TEXT NOT NULL CHECK (state IN ('pending','shadowed','review','materialized','duplicate','ignored','rejected','quarantined','failed','dead')),
|
|
719
|
-
attempt_count INTEGER NOT NULL DEFAULT 0 CHECK (attempt_count >= 0),
|
|
720
|
-
note_id TEXT,
|
|
721
|
-
last_error_code TEXT,
|
|
722
|
-
generation INTEGER NOT NULL DEFAULT 0 CHECK (generation >= 0),
|
|
723
|
-
created_at INTEGER NOT NULL,
|
|
724
|
-
updated_at INTEGER NOT NULL,
|
|
725
|
-
processed_at INTEGER
|
|
726
|
-
);`;
|
|
3274
|
+
function isAlreadyExistsError(error) {
|
|
3275
|
+
return Boolean(error && typeof error === "object" && "code" in error && String(error.code) === "EEXIST");
|
|
727
3276
|
}
|
|
728
3277
|
|
|
729
3278
|
// src/db/migrations/v009.ts
|
|
3279
|
+
import { createHash as createHash5, randomUUID as randomUUID5 } from "crypto";
|
|
730
3280
|
function migrateV8ToV9(db) {
|
|
731
3281
|
const notes = db.query("SELECT * FROM notes ORDER BY rowid").all();
|
|
732
3282
|
db.exec(`
|
|
@@ -779,7 +3329,7 @@ function migrateV8ToV9(db) {
|
|
|
779
3329
|
`);
|
|
780
3330
|
const hashes = new Map;
|
|
781
3331
|
for (const note of notes) {
|
|
782
|
-
const hash =
|
|
3332
|
+
const hash = noteContentHash2(note.kind, note.title, note.summary, note.content);
|
|
783
3333
|
hashes.set(note.id, hash);
|
|
784
3334
|
insert.run(note.id, note.project_id, note.kind, note.title, note.summary, note.content, note.size_class, note.pinned, note.status, note.supersedes_id, hash, note.created_at, note.updated_at);
|
|
785
3335
|
}
|
|
@@ -795,7 +3345,7 @@ function migrateV8ToV9(db) {
|
|
|
795
3345
|
`);
|
|
796
3346
|
db.exec(SCHEMA_TABLES);
|
|
797
3347
|
for (const note of notes) {
|
|
798
|
-
const provenanceID =
|
|
3348
|
+
const provenanceID = randomUUID5();
|
|
799
3349
|
db.query(`
|
|
800
3350
|
INSERT INTO note_provenance
|
|
801
3351
|
(id, project_id, note_id, source_type, created_at)
|
|
@@ -813,12 +3363,12 @@ function migrateV8ToV9(db) {
|
|
|
813
3363
|
db.query("DELETE FROM schema_state").run();
|
|
814
3364
|
db.query("INSERT INTO schema_state(version) VALUES (9)").run();
|
|
815
3365
|
}
|
|
816
|
-
function
|
|
817
|
-
return
|
|
3366
|
+
function noteContentHash2(kind, title, summary, content) {
|
|
3367
|
+
return createHash5("sha256").update(`${kind}\x00${title}\x00${summary}\x00${content}`, "utf8").digest("hex");
|
|
818
3368
|
}
|
|
819
3369
|
|
|
820
3370
|
// src/db/migrations/v010.ts
|
|
821
|
-
import { createHash as
|
|
3371
|
+
import { createHash as createHash6 } from "crypto";
|
|
822
3372
|
function migrateV9ToV10(db) {
|
|
823
3373
|
const payloads = db.query("SELECT idempotency_key, payload_json FROM capture_events WHERE payload_json IS NOT NULL").all();
|
|
824
3374
|
const migratedPayloads = payloads.map((row) => {
|
|
@@ -831,7 +3381,7 @@ function migrateV9ToV10(db) {
|
|
|
831
3381
|
return {
|
|
832
3382
|
idempotencyKey: row.idempotency_key,
|
|
833
3383
|
payload,
|
|
834
|
-
payloadHash:
|
|
3384
|
+
payloadHash: createHash6("sha256").update(payload, "utf8").digest("hex")
|
|
835
3385
|
};
|
|
836
3386
|
});
|
|
837
3387
|
db.exec("DROP TABLE IF EXISTS capture_events_v10");
|
|
@@ -866,7 +3416,7 @@ function migrateV9ToV10(db) {
|
|
|
866
3416
|
}
|
|
867
3417
|
|
|
868
3418
|
// src/version.ts
|
|
869
|
-
var PRODUCT_VERSION = "0.
|
|
3419
|
+
var PRODUCT_VERSION = "0.5.0";
|
|
870
3420
|
|
|
871
3421
|
// src/db.ts
|
|
872
3422
|
var DDL = `
|
|
@@ -908,34 +3458,152 @@ CREATE INDEX IF NOT EXISTS note_edges_source_idx ON note_edges(project_id, sourc
|
|
|
908
3458
|
CREATE INDEX IF NOT EXISTS note_edges_target_idx ON note_edges(project_id, target_id);
|
|
909
3459
|
CREATE TABLE IF NOT EXISTS schema_state (version INTEGER PRIMARY KEY);
|
|
910
3460
|
`;
|
|
3461
|
+
var PRE_OPEN_PROBE_TIMEOUT_MS = 5000;
|
|
911
3462
|
function openMemoryDatabase(path) {
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
3463
|
+
ensureDatabaseParent(path);
|
|
3464
|
+
assertSupportedDatabaseBeforeOpen(path);
|
|
3465
|
+
recoverStaleMaintenanceGate(path, () => assertSupportedDatabaseBeforeOpen(path));
|
|
3466
|
+
let lock = acquireMigrationLock(path, SCHEMA_VERSION);
|
|
3467
|
+
let lease = acquireDatabaseLease(path);
|
|
3468
|
+
let db;
|
|
3469
|
+
try {
|
|
3470
|
+
assertSupportedDatabaseBeforeOpen(path);
|
|
3471
|
+
db = openDatabase(path);
|
|
3472
|
+
} catch (error) {
|
|
3473
|
+
lease.release();
|
|
3474
|
+
lock.release();
|
|
3475
|
+
throw error;
|
|
3476
|
+
}
|
|
3477
|
+
let dbOpen = true;
|
|
3478
|
+
let maintenance;
|
|
915
3479
|
let backup;
|
|
916
3480
|
try {
|
|
917
|
-
db.exec("PRAGMA busy_timeout=5000");
|
|
918
|
-
db.exec("PRAGMA journal_mode=WAL");
|
|
919
3481
|
const existingVersion = getSchemaVersion(db);
|
|
920
3482
|
if (existingVersion && existingVersion.version > SCHEMA_VERSION) {
|
|
921
3483
|
throw new Error(`database schema v${existingVersion.version} is newer than supported v${SCHEMA_VERSION}`);
|
|
922
3484
|
}
|
|
923
|
-
const
|
|
924
|
-
if (!
|
|
3485
|
+
const hasObjects = hasApplicationObjects(db);
|
|
3486
|
+
if (!hasObjects) {
|
|
3487
|
+
db.close();
|
|
3488
|
+
dbOpen = false;
|
|
3489
|
+
lease.release();
|
|
3490
|
+
lease = undefined;
|
|
3491
|
+
lease = acquireDatabaseLease(path);
|
|
3492
|
+
assertSupportedDatabaseBeforeOpen(path);
|
|
3493
|
+
db = openDatabase(path);
|
|
3494
|
+
dbOpen = true;
|
|
3495
|
+
if (hasApplicationObjects(db)) {
|
|
3496
|
+
const initializedVersion = getSchemaVersion(db);
|
|
3497
|
+
if (initializedVersion?.version !== SCHEMA_VERSION) {
|
|
3498
|
+
throw new Error("database changed during initialization");
|
|
3499
|
+
}
|
|
3500
|
+
db.exec("PRAGMA foreign_keys=ON");
|
|
3501
|
+
assertHealthyDatabase(db);
|
|
3502
|
+
lock.release();
|
|
3503
|
+
lock = undefined;
|
|
3504
|
+
const opened3 = openedWithLease(db, lease);
|
|
3505
|
+
lease = undefined;
|
|
3506
|
+
return opened3;
|
|
3507
|
+
}
|
|
3508
|
+
db.exec("PRAGMA foreign_keys=ON");
|
|
3509
|
+
db.exec(`PRAGMA application_id = ${APPLICATION_ID}`);
|
|
3510
|
+
db.transaction(() => createSchemaV11(db))();
|
|
3511
|
+
assertHealthyDatabase(db);
|
|
3512
|
+
lock.release();
|
|
3513
|
+
lock = undefined;
|
|
3514
|
+
const opened2 = openedWithLease(db, lease);
|
|
3515
|
+
lease = undefined;
|
|
3516
|
+
return opened2;
|
|
3517
|
+
}
|
|
3518
|
+
const hasV11Marker = hasV11IdentityMarker(db);
|
|
3519
|
+
if (!existingVersion && hasV11Marker) {
|
|
3520
|
+
assertSchemaV11(db);
|
|
3521
|
+
}
|
|
3522
|
+
if (existingVersion && existingVersion.version < SCHEMA_VERSION && hasV11Marker) {
|
|
3523
|
+
assertSchemaV11(db);
|
|
3524
|
+
}
|
|
3525
|
+
if (!existingVersion && !hasLegacyV2(db))
|
|
3526
|
+
throw new Error("unrecognized_database");
|
|
3527
|
+
if (existingVersion?.version === SCHEMA_VERSION) {
|
|
925
3528
|
db.exec("PRAGMA foreign_keys=ON");
|
|
926
|
-
db.transaction(() => createSchema(db))();
|
|
927
3529
|
assertHealthyDatabase(db);
|
|
928
|
-
|
|
3530
|
+
lock.release();
|
|
3531
|
+
lock = undefined;
|
|
3532
|
+
const opened2 = openedWithLease(db, lease);
|
|
3533
|
+
lease = undefined;
|
|
3534
|
+
return opened2;
|
|
929
3535
|
}
|
|
930
3536
|
if ((existingVersion?.version ?? 0) < SCHEMA_VERSION) {
|
|
931
|
-
|
|
932
|
-
|
|
3537
|
+
db.close();
|
|
3538
|
+
dbOpen = false;
|
|
3539
|
+
lease.release();
|
|
3540
|
+
lease = undefined;
|
|
3541
|
+
lease = acquireDatabaseLease(path);
|
|
3542
|
+
assertSupportedDatabaseBeforeOpen(path);
|
|
3543
|
+
db = openDatabase(path);
|
|
3544
|
+
dbOpen = true;
|
|
3545
|
+
let migrationVersion = getSchemaVersion(db);
|
|
3546
|
+
if (migrationVersion && migrationVersion.version > SCHEMA_VERSION) {
|
|
3547
|
+
throw new Error(`database schema v${migrationVersion.version} is newer than supported v${SCHEMA_VERSION}`);
|
|
3548
|
+
}
|
|
3549
|
+
if (!migrationVersion && hasV11IdentityMarker(db))
|
|
3550
|
+
assertSchemaV11(db);
|
|
3551
|
+
if (migrationVersion && migrationVersion.version < SCHEMA_VERSION && hasV11IdentityMarker(db)) {
|
|
3552
|
+
assertSchemaV11(db);
|
|
3553
|
+
}
|
|
3554
|
+
if (!migrationVersion && !hasLegacyV2(db))
|
|
3555
|
+
throw new Error("unrecognized_database");
|
|
3556
|
+
if ((migrationVersion?.version ?? 0) === SCHEMA_VERSION) {
|
|
3557
|
+
db.exec("PRAGMA foreign_keys=ON");
|
|
3558
|
+
assertHealthyDatabase(db);
|
|
3559
|
+
lock.release();
|
|
3560
|
+
lock = undefined;
|
|
3561
|
+
const opened3 = openedWithLease(db, lease);
|
|
3562
|
+
lease = undefined;
|
|
3563
|
+
return opened3;
|
|
3564
|
+
}
|
|
3565
|
+
db.close();
|
|
3566
|
+
dbOpen = false;
|
|
3567
|
+
lease.release();
|
|
3568
|
+
lease = undefined;
|
|
3569
|
+
maintenance = acquireMaintenanceGate(path);
|
|
3570
|
+
assertSupportedDatabaseBeforeOpen(path);
|
|
3571
|
+
db = openDatabase(path);
|
|
3572
|
+
dbOpen = true;
|
|
3573
|
+
migrationVersion = getSchemaVersion(db);
|
|
3574
|
+
if (migrationVersion && migrationVersion.version > SCHEMA_VERSION) {
|
|
3575
|
+
throw new Error(`database schema v${migrationVersion.version} is newer than supported v${SCHEMA_VERSION}`);
|
|
3576
|
+
}
|
|
3577
|
+
if (!migrationVersion && hasV11IdentityMarker(db))
|
|
3578
|
+
assertSchemaV11(db);
|
|
3579
|
+
if (migrationVersion && migrationVersion.version < SCHEMA_VERSION && hasV11IdentityMarker(db)) {
|
|
3580
|
+
assertSchemaV11(db);
|
|
3581
|
+
}
|
|
3582
|
+
if (!migrationVersion && !hasLegacyV2(db))
|
|
3583
|
+
throw new Error("unrecognized_database");
|
|
3584
|
+
if ((migrationVersion?.version ?? 0) === SCHEMA_VERSION) {
|
|
3585
|
+
db.close();
|
|
3586
|
+
dbOpen = false;
|
|
3587
|
+
const handoff2 = handoffMaintenanceToLease(path, maintenance);
|
|
3588
|
+
maintenance = undefined;
|
|
3589
|
+
db = handoff2.db;
|
|
3590
|
+
dbOpen = true;
|
|
3591
|
+
lease = handoff2.lease;
|
|
3592
|
+
db.exec("PRAGMA foreign_keys=ON");
|
|
3593
|
+
assertHealthyDatabase(db);
|
|
3594
|
+
lock.release();
|
|
3595
|
+
lock = undefined;
|
|
3596
|
+
const opened3 = openedWithLease(db, lease);
|
|
3597
|
+
lease = undefined;
|
|
3598
|
+
return opened3;
|
|
3599
|
+
}
|
|
3600
|
+
backup = createVerifiedBackup(db, path, migrationVersion?.version ?? 2, SCHEMA_VERSION, PRODUCT_VERSION);
|
|
933
3601
|
db.exec("PRAGMA foreign_keys=OFF");
|
|
934
|
-
if (!
|
|
3602
|
+
if (!migrationVersion && hasLegacyV2(db)) {
|
|
935
3603
|
db.exec(DDL);
|
|
936
3604
|
db.exec("CREATE VIRTUAL TABLE IF NOT EXISTS notes_fts USING fts5(id UNINDEXED, title, summary, content, tokenize='unicode61')");
|
|
937
3605
|
migrateFromV2(db, path);
|
|
938
|
-
} else if (!
|
|
3606
|
+
} else if (!migrationVersion) {
|
|
939
3607
|
db.exec(DDL);
|
|
940
3608
|
db.exec("CREATE VIRTUAL TABLE IF NOT EXISTS notes_fts USING fts5(id UNINDEXED, title, summary, content, tokenize='unicode61')");
|
|
941
3609
|
db.transaction(() => {
|
|
@@ -943,7 +3611,7 @@ function openMemoryDatabase(path) {
|
|
|
943
3611
|
db.query("DELETE FROM schema_state").run();
|
|
944
3612
|
db.query("INSERT INTO schema_state (version) VALUES (8)").run();
|
|
945
3613
|
})();
|
|
946
|
-
} else if (
|
|
3614
|
+
} else if (migrationVersion.version < 8) {
|
|
947
3615
|
db.exec(DDL);
|
|
948
3616
|
db.exec("CREATE VIRTUAL TABLE IF NOT EXISTS notes_fts USING fts5(id UNINDEXED, title, summary, content, tokenize='unicode61')");
|
|
949
3617
|
migrateToV8(db);
|
|
@@ -953,46 +3621,219 @@ function openMemoryDatabase(path) {
|
|
|
953
3621
|
db.transaction(() => migrateV8ToV9(db))();
|
|
954
3622
|
version = 9;
|
|
955
3623
|
}
|
|
956
|
-
if (version < 10)
|
|
3624
|
+
if (version < 10) {
|
|
957
3625
|
db.transaction(() => migrateV9ToV10(db))();
|
|
3626
|
+
version = 10;
|
|
3627
|
+
}
|
|
3628
|
+
if (version < SCHEMA_VERSION) {
|
|
3629
|
+
db.transaction(() => {
|
|
3630
|
+
db.exec(`PRAGMA application_id = ${APPLICATION_ID}`);
|
|
3631
|
+
migrateV10ToV11(db);
|
|
3632
|
+
})();
|
|
3633
|
+
}
|
|
958
3634
|
db.exec("PRAGMA foreign_keys=ON");
|
|
959
3635
|
if (db.query("PRAGMA foreign_keys").get().foreign_keys !== 1) {
|
|
960
3636
|
throw new Error("failed to enable database foreign keys");
|
|
961
3637
|
}
|
|
962
3638
|
assertHealthyDatabase(db);
|
|
963
3639
|
console.warn(`[agz-memory] migrated to v${SCHEMA_VERSION} (backup: ${backup.manifestPath})`);
|
|
3640
|
+
backup = undefined;
|
|
3641
|
+
db.close();
|
|
3642
|
+
dbOpen = false;
|
|
3643
|
+
const handoff = handoffMaintenanceToLease(path, maintenance);
|
|
3644
|
+
maintenance = undefined;
|
|
3645
|
+
db = handoff.db;
|
|
3646
|
+
dbOpen = true;
|
|
3647
|
+
lease = handoff.lease;
|
|
3648
|
+
db.exec("PRAGMA foreign_keys=ON");
|
|
3649
|
+
assertHealthyDatabase(db);
|
|
964
3650
|
lock.release();
|
|
965
3651
|
lock = undefined;
|
|
966
|
-
|
|
3652
|
+
const opened2 = openedWithLease(db, lease);
|
|
3653
|
+
lease = undefined;
|
|
3654
|
+
return opened2;
|
|
967
3655
|
}
|
|
968
3656
|
db.exec("PRAGMA foreign_keys=ON");
|
|
969
|
-
db.exec(SCHEMA_TABLES);
|
|
970
|
-
db.exec(FTS_V9);
|
|
971
3657
|
assertHealthyDatabase(db);
|
|
972
3658
|
db.exec("PRAGMA foreign_keys=ON");
|
|
973
|
-
|
|
3659
|
+
lock.release();
|
|
3660
|
+
lock = undefined;
|
|
3661
|
+
const opened = openedWithLease(db, lease);
|
|
3662
|
+
lease = undefined;
|
|
3663
|
+
return opened;
|
|
974
3664
|
} catch (error) {
|
|
975
|
-
|
|
3665
|
+
if (dbOpen)
|
|
3666
|
+
db.close();
|
|
976
3667
|
if (backup) {
|
|
977
3668
|
try {
|
|
978
|
-
restoreVerifiedBackup(backup.manifestPath, path, "RESTORE_DATABASE_FROM_VERIFIED_BACKUP");
|
|
3669
|
+
restoreVerifiedBackup(backup.manifestPath, path, "RESTORE_DATABASE_FROM_VERIFIED_BACKUP", maintenance);
|
|
979
3670
|
} catch (restoreError) {
|
|
980
3671
|
throw new AggregateError([error, restoreError], "migration and automatic restore failed");
|
|
981
3672
|
}
|
|
982
3673
|
}
|
|
983
3674
|
throw error;
|
|
984
3675
|
} finally {
|
|
3676
|
+
lease?.release();
|
|
3677
|
+
maintenance?.release();
|
|
985
3678
|
lock?.release();
|
|
986
3679
|
}
|
|
987
3680
|
}
|
|
3681
|
+
function openReadOnlyMemoryDatabase(path) {
|
|
3682
|
+
assertSupportedDatabaseBeforeOpen(path);
|
|
3683
|
+
recoverStaleMaintenanceGate(path, () => assertSupportedDatabaseBeforeOpen(path));
|
|
3684
|
+
const lease = acquireDatabaseLease(path);
|
|
3685
|
+
try {
|
|
3686
|
+
assertSupportedDatabaseBeforeOpen(path);
|
|
3687
|
+
assertDatabasePath(path);
|
|
3688
|
+
const db = new Database3(path, { readonly: true });
|
|
3689
|
+
try {
|
|
3690
|
+
assertDatabasePath(path);
|
|
3691
|
+
assertSupportedDatabase(db);
|
|
3692
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
3693
|
+
return openedWithLease(db, lease);
|
|
3694
|
+
} catch (error) {
|
|
3695
|
+
db.close();
|
|
3696
|
+
throw error;
|
|
3697
|
+
}
|
|
3698
|
+
} catch (error) {
|
|
3699
|
+
lease.release();
|
|
3700
|
+
throw error;
|
|
3701
|
+
}
|
|
3702
|
+
}
|
|
3703
|
+
function handoffMaintenanceToLease(path, maintenance) {
|
|
3704
|
+
maintenance.release();
|
|
3705
|
+
const lease = acquireDatabaseLease(path);
|
|
3706
|
+
try {
|
|
3707
|
+
return { db: openDatabase(path), lease };
|
|
3708
|
+
} catch (error) {
|
|
3709
|
+
lease.release();
|
|
3710
|
+
throw error;
|
|
3711
|
+
}
|
|
3712
|
+
}
|
|
3713
|
+
function openedWithLease(db, lease) {
|
|
3714
|
+
let closed = false;
|
|
3715
|
+
return {
|
|
3716
|
+
db,
|
|
3717
|
+
close: () => {
|
|
3718
|
+
if (closed)
|
|
3719
|
+
return;
|
|
3720
|
+
closed = true;
|
|
3721
|
+
try {
|
|
3722
|
+
db.close();
|
|
3723
|
+
} finally {
|
|
3724
|
+
lease.release();
|
|
3725
|
+
}
|
|
3726
|
+
}
|
|
3727
|
+
};
|
|
3728
|
+
}
|
|
3729
|
+
function openDatabase(path) {
|
|
3730
|
+
assertDatabasePath(path);
|
|
3731
|
+
const db = new Database3(path, { create: true });
|
|
3732
|
+
try {
|
|
3733
|
+
assertDatabasePath(path);
|
|
3734
|
+
db.exec("PRAGMA query_only=ON");
|
|
3735
|
+
assertSupportedDatabase(db);
|
|
3736
|
+
db.exec("PRAGMA query_only=OFF");
|
|
3737
|
+
chmodSync3(path, 384);
|
|
3738
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
3739
|
+
db.exec("PRAGMA journal_mode=WAL");
|
|
3740
|
+
return db;
|
|
3741
|
+
} catch (error) {
|
|
3742
|
+
db.close();
|
|
3743
|
+
throw error;
|
|
3744
|
+
}
|
|
3745
|
+
}
|
|
3746
|
+
function assertSupportedDatabaseBeforeOpen(path) {
|
|
3747
|
+
const deadline = Date.now() + PRE_OPEN_PROBE_TIMEOUT_MS;
|
|
3748
|
+
while (true) {
|
|
3749
|
+
try {
|
|
3750
|
+
assertSupportedDatabaseBeforeOpenOnce(path);
|
|
3751
|
+
return;
|
|
3752
|
+
} catch (error) {
|
|
3753
|
+
if (!isSQLiteBusyError(error) || Date.now() >= deadline)
|
|
3754
|
+
throw error;
|
|
3755
|
+
Bun.sleepSync(Math.min(50, Math.max(1, deadline - Date.now())));
|
|
3756
|
+
}
|
|
3757
|
+
}
|
|
3758
|
+
}
|
|
3759
|
+
function assertSupportedDatabaseBeforeOpenOnce(path) {
|
|
3760
|
+
assertDatabasePath(path);
|
|
3761
|
+
if (!existsSync4(path))
|
|
3762
|
+
return;
|
|
3763
|
+
const db = new Database3(path, { readonly: true });
|
|
3764
|
+
try {
|
|
3765
|
+
assertDatabasePath(path);
|
|
3766
|
+
assertSupportedDatabase(db);
|
|
3767
|
+
} finally {
|
|
3768
|
+
db.close();
|
|
3769
|
+
}
|
|
3770
|
+
}
|
|
3771
|
+
function assertSupportedDatabase(db) {
|
|
3772
|
+
const existingVersion = getSchemaVersion(db);
|
|
3773
|
+
if (existingVersion && existingVersion.version > SCHEMA_VERSION) {
|
|
3774
|
+
throw new Error(`database schema v${existingVersion.version} is newer than supported v${SCHEMA_VERSION}`);
|
|
3775
|
+
}
|
|
3776
|
+
if (!hasApplicationObjects(db))
|
|
3777
|
+
return;
|
|
3778
|
+
const hasV11Marker = hasV11IdentityMarker(db);
|
|
3779
|
+
if (!existingVersion) {
|
|
3780
|
+
if (!hasLegacyV2(db))
|
|
3781
|
+
throw new Error("unrecognized_database");
|
|
3782
|
+
assertLegacySchemaIdentity(db, 2);
|
|
3783
|
+
return;
|
|
3784
|
+
}
|
|
3785
|
+
if (existingVersion.version === SCHEMA_VERSION || hasV11Marker) {
|
|
3786
|
+
assertSchemaV11(db);
|
|
3787
|
+
return;
|
|
3788
|
+
}
|
|
3789
|
+
if (existingVersion.version < 2 || existingVersion.version > 10) {
|
|
3790
|
+
throw new Error("unrecognized_database");
|
|
3791
|
+
}
|
|
3792
|
+
assertLegacySchemaIdentity(db, existingVersion.version);
|
|
3793
|
+
}
|
|
3794
|
+
function assertDatabasePath(path) {
|
|
3795
|
+
try {
|
|
3796
|
+
if (lstatSync3(path).isSymbolicLink()) {
|
|
3797
|
+
throw new Error("database path must not be a symbolic link");
|
|
3798
|
+
}
|
|
3799
|
+
} catch (error) {
|
|
3800
|
+
if (error instanceof Error && "code" in error && error.code === "ENOENT")
|
|
3801
|
+
return;
|
|
3802
|
+
throw error;
|
|
3803
|
+
}
|
|
3804
|
+
}
|
|
988
3805
|
function getSchemaVersion(db) {
|
|
989
|
-
if (!
|
|
3806
|
+
if (!hasTable4(db, "schema_state"))
|
|
990
3807
|
return;
|
|
991
|
-
|
|
3808
|
+
let rows;
|
|
3809
|
+
try {
|
|
3810
|
+
rows = db.query("SELECT version FROM schema_state").all();
|
|
3811
|
+
} catch (error) {
|
|
3812
|
+
if (isSQLiteBusyError(error))
|
|
3813
|
+
throw error;
|
|
3814
|
+
throw new Error("unrecognized_database");
|
|
3815
|
+
}
|
|
3816
|
+
if (rows.length !== 1 || typeof rows[0]?.version !== "number" || !Number.isSafeInteger(rows[0].version)) {
|
|
3817
|
+
throw new Error("unrecognized_database");
|
|
3818
|
+
}
|
|
3819
|
+
return { version: rows[0].version };
|
|
992
3820
|
}
|
|
993
3821
|
function migrateToV8(db) {
|
|
994
3822
|
db.transaction(() => {
|
|
995
3823
|
adoptLegacyProjectIDs(db);
|
|
3824
|
+
importLegacyAssociations(db);
|
|
3825
|
+
if (hasColumn(db, "notes", "current_revision")) {
|
|
3826
|
+
dropLegacyV2Tables(db);
|
|
3827
|
+
db.query("DELETE FROM schema_state").run();
|
|
3828
|
+
db.query("INSERT INTO schema_state (version) VALUES (8)").run();
|
|
3829
|
+
return;
|
|
3830
|
+
}
|
|
3831
|
+
db.exec(`
|
|
3832
|
+
DROP TRIGGER IF EXISTS notes_fts_ai;
|
|
3833
|
+
DROP TRIGGER IF EXISTS notes_fts_ad;
|
|
3834
|
+
DROP TRIGGER IF EXISTS notes_fts_au;
|
|
3835
|
+
DROP TABLE IF EXISTS notes_fts;
|
|
3836
|
+
`);
|
|
996
3837
|
const pinned = hasColumn(db, "notes", "pinned") ? "pinned" : "0";
|
|
997
3838
|
db.exec(`
|
|
998
3839
|
CREATE TABLE notes_v7 (
|
|
@@ -1037,7 +3878,7 @@ function migrateToV8(db) {
|
|
|
1037
3878
|
ALTER TABLE notes_v7 RENAME TO notes;
|
|
1038
3879
|
ALTER TABLE note_edges_v7 RENAME TO note_edges;
|
|
1039
3880
|
`);
|
|
1040
|
-
|
|
3881
|
+
dropLegacyV2Tables(db);
|
|
1041
3882
|
db.query("DELETE FROM schema_state").run();
|
|
1042
3883
|
db.query("INSERT INTO schema_state (version) VALUES (8)").run();
|
|
1043
3884
|
})();
|
|
@@ -1047,7 +3888,7 @@ function adoptLegacyProjectIDs(db) {
|
|
|
1047
3888
|
for (const { id: legacyID } of existingProjects) {
|
|
1048
3889
|
if (isUUID(legacyID))
|
|
1049
3890
|
continue;
|
|
1050
|
-
const id =
|
|
3891
|
+
const id = randomUUID6();
|
|
1051
3892
|
db.query("UPDATE projects SET id = ? WHERE id = ?").run(id, legacyID);
|
|
1052
3893
|
db.query("UPDATE notes SET project_id = ? WHERE project_id = ?").run(id, legacyID);
|
|
1053
3894
|
db.query("UPDATE note_edges SET project_id = ? WHERE project_id = ?").run(id, legacyID);
|
|
@@ -1056,7 +3897,7 @@ function adoptLegacyProjectIDs(db) {
|
|
|
1056
3897
|
for (const { project_id: legacyID } of rows) {
|
|
1057
3898
|
if (db.query("SELECT id FROM projects WHERE id = ?").get(legacyID))
|
|
1058
3899
|
continue;
|
|
1059
|
-
const id =
|
|
3900
|
+
const id = randomUUID6();
|
|
1060
3901
|
const name = uniqueLegacyProjectName(db, legacyID);
|
|
1061
3902
|
const now = Date.now();
|
|
1062
3903
|
db.query("INSERT INTO projects (id, name, normalized_name, created_at, updated_at) VALUES (?, ?, ?, ?, ?)").run(id, name, normalizeProjectName(name), now, now);
|
|
@@ -1081,9 +3922,18 @@ function hasColumn(db, table, column) {
|
|
|
1081
3922
|
return rows.some((row) => row.name === column);
|
|
1082
3923
|
}
|
|
1083
3924
|
function hasLegacyV2(db) {
|
|
1084
|
-
return
|
|
3925
|
+
return hasTable4(db, "memory_items");
|
|
1085
3926
|
}
|
|
1086
|
-
function
|
|
3927
|
+
function hasApplicationObjects(db) {
|
|
3928
|
+
const row = db.query("SELECT COUNT(*) AS count FROM sqlite_master WHERE name NOT LIKE 'sqlite_%'").get();
|
|
3929
|
+
return row.count > 0;
|
|
3930
|
+
}
|
|
3931
|
+
function hasV11IdentityMarker(db) {
|
|
3932
|
+
if (hasTable4(db, "agz_meta"))
|
|
3933
|
+
return true;
|
|
3934
|
+
return db.query("PRAGMA application_id").get().application_id === APPLICATION_ID;
|
|
3935
|
+
}
|
|
3936
|
+
function hasTable4(db, table) {
|
|
1087
3937
|
const row = db.query("SELECT COUNT(*) AS n FROM sqlite_master WHERE type='table' AND name = ?").get(table);
|
|
1088
3938
|
return (row?.n ?? 0) > 0;
|
|
1089
3939
|
}
|
|
@@ -1105,26 +3955,42 @@ var KIND_MAP = {
|
|
|
1105
3955
|
};
|
|
1106
3956
|
function migrateFromV2(db, path) {
|
|
1107
3957
|
const requiredTables = ["memory_items", "memory_versions", "memory_identities"];
|
|
1108
|
-
const missingTables = requiredTables.filter((table) => !
|
|
3958
|
+
const missingTables = requiredTables.filter((table) => !hasTable4(db, table));
|
|
1109
3959
|
if (missingTables.length > 0) {
|
|
1110
3960
|
throw new Error(`unsupported legacy schema; missing tables: ${missingTables.join(", ")}`);
|
|
1111
3961
|
}
|
|
1112
3962
|
const backup = `${path}.v2-backup`;
|
|
1113
|
-
if (!
|
|
3963
|
+
if (!existsSync4(backup)) {
|
|
1114
3964
|
db.exec("PRAGMA wal_checkpoint(TRUNCATE)");
|
|
1115
3965
|
copyFileSync2(path, backup);
|
|
1116
3966
|
}
|
|
1117
3967
|
db.transaction(() => {
|
|
1118
3968
|
migrateFromV2Data(db, backup, {
|
|
1119
|
-
documents: ["document_sources", "document_chunks", "memories"].every((table) =>
|
|
1120
|
-
links:
|
|
1121
|
-
edges:
|
|
3969
|
+
documents: ["document_sources", "document_chunks", "memories"].every((table) => hasTable4(db, table)),
|
|
3970
|
+
links: hasTable4(db, "memory_links"),
|
|
3971
|
+
edges: hasTable4(db, "memory_edges")
|
|
1122
3972
|
});
|
|
1123
3973
|
adoptLegacyProjectIDs(db);
|
|
3974
|
+
dropLegacyV2Tables(db);
|
|
1124
3975
|
db.query("DELETE FROM schema_state").run();
|
|
1125
3976
|
db.query("INSERT INTO schema_state (version) VALUES (8)").run();
|
|
1126
3977
|
})();
|
|
1127
3978
|
}
|
|
3979
|
+
function dropLegacyV2Tables(db) {
|
|
3980
|
+
for (const table of [
|
|
3981
|
+
"document_chunks",
|
|
3982
|
+
"document_sources",
|
|
3983
|
+
"memories",
|
|
3984
|
+
"memory_associations",
|
|
3985
|
+
"memory_edges",
|
|
3986
|
+
"memory_links",
|
|
3987
|
+
"memory_identities",
|
|
3988
|
+
"memory_versions",
|
|
3989
|
+
"memory_items"
|
|
3990
|
+
]) {
|
|
3991
|
+
db.exec(`DROP TABLE IF EXISTS ${table}`);
|
|
3992
|
+
}
|
|
3993
|
+
}
|
|
1128
3994
|
function migrateFromV2Data(db, backup, options) {
|
|
1129
3995
|
const now = Date.now();
|
|
1130
3996
|
db.query("DELETE FROM notes_fts").run();
|
|
@@ -1168,7 +4034,7 @@ function migrateFromV2Data(db, backup, options) {
|
|
|
1168
4034
|
const content = source.body ?? "";
|
|
1169
4035
|
if (!content.trim())
|
|
1170
4036
|
continue;
|
|
1171
|
-
const id =
|
|
4037
|
+
const id = randomUUID6();
|
|
1172
4038
|
db.query(`INSERT INTO notes (id, project_id, kind, title, summary, content, size_class, status, supersedes_id, created_at, updated_at)
|
|
1173
4039
|
VALUES (?, ?, 'research', ?, ?, ?, 'indexed', 'active', NULL, ?, ?)`).run(id, source.project_root ? hashRoot(source.project_root) : "legacy", source.title, content.slice(0, 200), content, source.created_at, source.updated_at);
|
|
1174
4040
|
db.query("INSERT INTO notes_fts (id, title, summary, content) VALUES (?, ?, ?, ?)").run(id, source.title, content.slice(0, 200), content);
|
|
@@ -1207,7 +4073,7 @@ function migrateFromV2Data(db, backup, options) {
|
|
|
1207
4073
|
console.warn(`[agz-memory] v2\u2192v3 migration complete: ${migratedNotes} notes, ${migratedEdges} edges (backup: ${backup})`);
|
|
1208
4074
|
}
|
|
1209
4075
|
function importLegacyAssociations(db) {
|
|
1210
|
-
if (!
|
|
4076
|
+
if (!hasTable4(db, "memory_associations"))
|
|
1211
4077
|
return 0;
|
|
1212
4078
|
const associations = db.query(`SELECT id, left_item_id, right_item_id, kind, created_at
|
|
1213
4079
|
FROM memory_associations
|
|
@@ -1247,6 +4113,13 @@ function doctorDatabase(db) {
|
|
|
1247
4113
|
failures.push("schema_newer_than_runtime");
|
|
1248
4114
|
else if (health.schemaVersion < SCHEMA_VERSION)
|
|
1249
4115
|
warnings.push("schema_upgrade_required");
|
|
4116
|
+
if (health.schemaVersion === SCHEMA_VERSION) {
|
|
4117
|
+
try {
|
|
4118
|
+
assertSchemaV11(db);
|
|
4119
|
+
} catch {
|
|
4120
|
+
failures.push("schema_fingerprint_mismatch");
|
|
4121
|
+
}
|
|
4122
|
+
}
|
|
1250
4123
|
if (hasTable(db, "notes_fts") && hasTable(db, "notes")) {
|
|
1251
4124
|
invariants.notes = count(db, "SELECT COUNT(*) AS count FROM notes");
|
|
1252
4125
|
invariants.fts = count(db, "SELECT COUNT(*) AS count FROM notes_fts");
|
|
@@ -1261,6 +4134,34 @@ function doctorDatabase(db) {
|
|
|
1261
4134
|
AND r.note_id = n.id
|
|
1262
4135
|
AND r.revision = n.current_revision
|
|
1263
4136
|
WHERE r.note_id IS NULL`);
|
|
4137
|
+
invariants.noteContentHashMismatches = canonicalHashMismatches(db, "notes");
|
|
4138
|
+
invariants.revisionContentHashMismatches = canonicalHashMismatches(db, "note_revisions");
|
|
4139
|
+
invariants.currentRevisionMismatches = count(db, `SELECT COUNT(*) AS count
|
|
4140
|
+
FROM notes n
|
|
4141
|
+
JOIN note_revisions r
|
|
4142
|
+
ON r.project_id = n.project_id
|
|
4143
|
+
AND r.note_id = n.id
|
|
4144
|
+
AND r.revision = n.current_revision
|
|
4145
|
+
WHERE n.kind IS NOT r.kind
|
|
4146
|
+
OR n.title IS NOT r.title
|
|
4147
|
+
OR n.summary IS NOT r.summary
|
|
4148
|
+
OR n.content IS NOT r.content
|
|
4149
|
+
OR n.size_class IS NOT r.size_class
|
|
4150
|
+
OR n.pinned IS NOT r.pinned
|
|
4151
|
+
OR n.status IS NOT r.status
|
|
4152
|
+
OR n.supersedes_id IS NOT r.supersedes_id
|
|
4153
|
+
OR n.subject_key IS NOT r.subject_key
|
|
4154
|
+
OR n.content_hash IS NOT r.content_hash`);
|
|
4155
|
+
invariants.revisionGaps = count(db, `SELECT COUNT(*) AS count FROM (
|
|
4156
|
+
SELECT n.project_id, n.id
|
|
4157
|
+
FROM notes n
|
|
4158
|
+
LEFT JOIN note_revisions r
|
|
4159
|
+
ON r.project_id = n.project_id AND r.note_id = n.id
|
|
4160
|
+
GROUP BY n.project_id, n.id, n.current_revision
|
|
4161
|
+
HAVING COUNT(r.revision) <> n.current_revision
|
|
4162
|
+
OR MIN(r.revision) IS NOT 1
|
|
4163
|
+
OR MAX(r.revision) IS NOT n.current_revision
|
|
4164
|
+
)`);
|
|
1264
4165
|
invariants.missingProvenance = count(db, `SELECT COUNT(*) AS count
|
|
1265
4166
|
FROM note_revisions r
|
|
1266
4167
|
LEFT JOIN note_provenance p
|
|
@@ -1273,120 +4174,104 @@ function doctorDatabase(db) {
|
|
|
1273
4174
|
HAVING COUNT(DISTINCT project_id) > 1
|
|
1274
4175
|
)`);
|
|
1275
4176
|
invariants.deadOutbox = count(db, "SELECT COUNT(*) AS count FROM index_outbox WHERE state = 'dead'");
|
|
4177
|
+
invariants.invalidOutboxOperations = count(db, `SELECT COUNT(*) AS count FROM index_outbox
|
|
4178
|
+
WHERE (operation = 'upsert-note'
|
|
4179
|
+
AND (note_id IS NULL OR revision IS NULL OR revision < 1
|
|
4180
|
+
OR content_hash IS NULL OR length(content_hash) <> 64))
|
|
4181
|
+
OR (operation = 'delete-note'
|
|
4182
|
+
AND (note_id IS NULL OR revision IS NULL OR revision < 1
|
|
4183
|
+
OR content_hash IS NOT NULL))
|
|
4184
|
+
OR (operation = 'purge-project'
|
|
4185
|
+
AND (note_id IS NOT NULL OR revision IS NOT NULL OR content_hash IS NOT NULL))
|
|
4186
|
+
OR operation NOT IN ('upsert-note', 'delete-note', 'purge-project')`);
|
|
4187
|
+
invariants.invalidOutboxLeases = count(db, `SELECT COUNT(*) AS count FROM index_outbox
|
|
4188
|
+
WHERE (state = 'leased' AND (lease_owner IS NULL OR lease_expires_at IS NULL))
|
|
4189
|
+
OR (state <> 'leased'
|
|
4190
|
+
AND (lease_owner IS NOT NULL OR lease_expires_at IS NOT NULL OR heartbeat_at IS NOT NULL))`);
|
|
4191
|
+
invariants.orphanedOutboxProjects = count(db, `SELECT COUNT(*) AS count FROM index_outbox o
|
|
4192
|
+
LEFT JOIN projects p ON p.id = o.project_id
|
|
4193
|
+
WHERE p.id IS NULL`);
|
|
4194
|
+
invariants.outboxOperationKeyMismatches = outboxOperationKeyMismatches(db);
|
|
4195
|
+
invariants.derivedHashMismatches = derivedHashMismatches(db);
|
|
1276
4196
|
invariants.dueCheckpoints = count(db, "SELECT COUNT(*) AS count FROM capture_checkpoints WHERE next_reconcile_at <= ? AND state = 'active'", Date.now());
|
|
1277
4197
|
for (const [name, value] of Object.entries(invariants)) {
|
|
1278
|
-
if ([
|
|
4198
|
+
if ([
|
|
4199
|
+
"missingCurrentRevisions",
|
|
4200
|
+
"noteContentHashMismatches",
|
|
4201
|
+
"revisionContentHashMismatches",
|
|
4202
|
+
"currentRevisionMismatches",
|
|
4203
|
+
"revisionGaps",
|
|
4204
|
+
"missingProvenance",
|
|
4205
|
+
"bindingConflicts",
|
|
4206
|
+
"invalidOutboxOperations",
|
|
4207
|
+
"invalidOutboxLeases",
|
|
4208
|
+
"outboxOperationKeyMismatches",
|
|
4209
|
+
"derivedHashMismatches"
|
|
4210
|
+
].includes(name) && value > 0) {
|
|
1279
4211
|
failures.push(name);
|
|
1280
4212
|
}
|
|
1281
4213
|
}
|
|
1282
4214
|
}
|
|
1283
4215
|
return { ok: failures.length === 0, health, warnings, failures, invariants };
|
|
1284
4216
|
}
|
|
4217
|
+
function canonicalHashMismatches(db, table) {
|
|
4218
|
+
const rows = db.query(`SELECT kind, title, summary, content, content_hash FROM ${table}`).iterate();
|
|
4219
|
+
let mismatches = 0;
|
|
4220
|
+
for (const row of rows) {
|
|
4221
|
+
if (row.content_hash !== noteContentHash(row.kind, row.title, row.summary, row.content)) {
|
|
4222
|
+
mismatches++;
|
|
4223
|
+
}
|
|
4224
|
+
}
|
|
4225
|
+
return mismatches;
|
|
4226
|
+
}
|
|
1285
4227
|
function count(db, sql, ...bindings) {
|
|
1286
4228
|
return db.query(sql).get(...bindings).count;
|
|
1287
4229
|
}
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
{
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
{ name: "github-token", pattern: /\b(?:gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,})\b/g },
|
|
1307
|
-
{ name: "gitlab-token", pattern: /\bglpat-[A-Za-z0-9_-]{16,}\b/g },
|
|
1308
|
-
{ name: "aws-access-key", pattern: /\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/g },
|
|
1309
|
-
{ name: "jwt", pattern: /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g },
|
|
1310
|
-
{
|
|
1311
|
-
name: "secret-assignment",
|
|
1312
|
-
pattern: /\b(?:PASSWORD|PASSWD|SECRET|TOKEN|API_KEY|PRIVATE_KEY)\s*[:=]\s*["']?[^\s,"']{8,}["']?/gi
|
|
1313
|
-
}
|
|
1314
|
-
];
|
|
1315
|
-
function redactText(value, options = {}) {
|
|
1316
|
-
const maxCharacters = options.maxCharacters ?? Number.MAX_SAFE_INTEGER;
|
|
1317
|
-
let text = value;
|
|
1318
|
-
let replacements = 0;
|
|
1319
|
-
let highRisk = 0;
|
|
1320
|
-
const classes = {};
|
|
1321
|
-
for (const literal2 of options.denylist ?? []) {
|
|
1322
|
-
if (!literal2)
|
|
1323
|
-
continue;
|
|
1324
|
-
const count2 = text.split(literal2).length - 1;
|
|
1325
|
-
if (count2 === 0)
|
|
1326
|
-
continue;
|
|
1327
|
-
replacements += count2;
|
|
1328
|
-
classes.denylist = (classes.denylist ?? 0) + count2;
|
|
1329
|
-
text = text.replaceAll(literal2, "[REDACTED:denylist]");
|
|
1330
|
-
}
|
|
1331
|
-
for (const rule of RULES) {
|
|
1332
|
-
text = text.replace(rule.pattern, () => {
|
|
1333
|
-
replacements++;
|
|
1334
|
-
classes[rule.name] = (classes[rule.name] ?? 0) + 1;
|
|
1335
|
-
if (rule.highRisk)
|
|
1336
|
-
highRisk++;
|
|
1337
|
-
return `[REDACTED:${rule.name}]`;
|
|
1338
|
-
});
|
|
4230
|
+
function outboxOperationKeyMismatches(db) {
|
|
4231
|
+
const rows = db.query(`SELECT backend, operation, project_id, note_id, revision, content_hash,
|
|
4232
|
+
generation, operation_key
|
|
4233
|
+
FROM index_outbox
|
|
4234
|
+
ORDER BY id`).all();
|
|
4235
|
+
let mismatches = 0;
|
|
4236
|
+
for (const row of rows) {
|
|
4237
|
+
const expected = hashTuple("outbox-operation", 2, [
|
|
4238
|
+
row.backend,
|
|
4239
|
+
row.operation,
|
|
4240
|
+
row.project_id,
|
|
4241
|
+
row.note_id,
|
|
4242
|
+
row.revision,
|
|
4243
|
+
row.content_hash,
|
|
4244
|
+
row.generation
|
|
4245
|
+
]);
|
|
4246
|
+
if (row.operation_key !== expected)
|
|
4247
|
+
mismatches++;
|
|
1339
4248
|
}
|
|
1340
|
-
|
|
1341
|
-
if (!looksHighEntropy(candidate))
|
|
1342
|
-
return candidate;
|
|
1343
|
-
replacements++;
|
|
1344
|
-
classes.entropy = (classes.entropy ?? 0) + 1;
|
|
1345
|
-
return "[REDACTED:entropy]";
|
|
1346
|
-
});
|
|
1347
|
-
const truncated = text.length > maxCharacters;
|
|
1348
|
-
if (truncated)
|
|
1349
|
-
text = text.slice(0, maxCharacters);
|
|
1350
|
-
return {
|
|
1351
|
-
text,
|
|
1352
|
-
replacements,
|
|
1353
|
-
classes,
|
|
1354
|
-
truncated,
|
|
1355
|
-
quarantined: highRisk > 0 || replacements >= 3
|
|
1356
|
-
};
|
|
4249
|
+
return mismatches;
|
|
1357
4250
|
}
|
|
1358
|
-
function
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
4251
|
+
function derivedHashMismatches(db) {
|
|
4252
|
+
const rows = db.query(`SELECT o.content_hash, n.project_id, n.id, n.current_revision,
|
|
4253
|
+
n.kind, n.title, n.summary, n.content
|
|
4254
|
+
FROM index_outbox o
|
|
4255
|
+
JOIN notes n ON n.project_id = o.project_id AND n.id = o.note_id
|
|
4256
|
+
WHERE o.operation = 'upsert-note'
|
|
4257
|
+
AND o.state IN ('pending', 'leased')
|
|
4258
|
+
AND n.status = 'active'
|
|
4259
|
+
AND n.current_revision = o.revision`).all();
|
|
4260
|
+
let mismatches = 0;
|
|
4261
|
+
for (const row of rows) {
|
|
4262
|
+
const document = deriveDocument({
|
|
4263
|
+
projectID: row.project_id,
|
|
4264
|
+
noteID: row.id,
|
|
4265
|
+
revision: row.current_revision,
|
|
4266
|
+
kind: row.kind,
|
|
4267
|
+
title: row.title,
|
|
4268
|
+
summary: row.summary,
|
|
4269
|
+
content: row.content
|
|
4270
|
+
});
|
|
4271
|
+
if (!document || document.contentHash !== row.content_hash)
|
|
4272
|
+
mismatches++;
|
|
1368
4273
|
}
|
|
1369
|
-
return
|
|
1370
|
-
}
|
|
1371
|
-
|
|
1372
|
-
// src/retrieval/derived.ts
|
|
1373
|
-
function deriveDocument(source) {
|
|
1374
|
-
const title = redactText(source.title);
|
|
1375
|
-
const summary = redactText(source.summary);
|
|
1376
|
-
const content = redactText(source.content);
|
|
1377
|
-
if (title.quarantined || summary.quarantined || content.quarantined)
|
|
1378
|
-
return;
|
|
1379
|
-
const contentHash = createHash5("sha256").update(`${source.kind}\x00${title.text}\x00${summary.text}\x00${content.text}`, "utf8").digest("hex");
|
|
1380
|
-
return {
|
|
1381
|
-
projectID: source.projectID,
|
|
1382
|
-
noteID: source.noteID,
|
|
1383
|
-
revision: source.revision,
|
|
1384
|
-
kind: source.kind,
|
|
1385
|
-
title: title.text,
|
|
1386
|
-
summary: summary.text,
|
|
1387
|
-
content: content.text,
|
|
1388
|
-
contentHash
|
|
1389
|
-
};
|
|
4274
|
+
return mismatches;
|
|
1390
4275
|
}
|
|
1391
4276
|
|
|
1392
4277
|
// src/admin/index.ts
|
|
@@ -1397,24 +4282,24 @@ async function runAdmin(argv = process.argv.slice(2)) {
|
|
|
1397
4282
|
throw new Error("admin command is required");
|
|
1398
4283
|
if (command === "doctor") {
|
|
1399
4284
|
requireExistingDatabase(databasePath);
|
|
1400
|
-
const
|
|
4285
|
+
const opened = openReadOnlyMemoryDatabase(databasePath);
|
|
1401
4286
|
try {
|
|
1402
|
-
return doctorDatabase(db);
|
|
4287
|
+
return doctorDatabase(opened.db);
|
|
1403
4288
|
} finally {
|
|
1404
|
-
|
|
4289
|
+
opened.close();
|
|
1405
4290
|
}
|
|
1406
4291
|
}
|
|
1407
4292
|
if (command === "backup" && subcommand !== "prune") {
|
|
1408
4293
|
requireExistingDatabase(databasePath);
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
}
|
|
4294
|
+
return withExclusiveMaintenance(databasePath, readSchemaVersion(databasePath), () => {
|
|
4295
|
+
const db = new Database4(databasePath);
|
|
4296
|
+
try {
|
|
4297
|
+
const version = schemaVersion(db);
|
|
4298
|
+
return createVerifiedBackup(db, databasePath, version, version, PRODUCT_VERSION);
|
|
4299
|
+
} finally {
|
|
4300
|
+
db.close();
|
|
4301
|
+
}
|
|
4302
|
+
});
|
|
1418
4303
|
}
|
|
1419
4304
|
if (command === "upgrade") {
|
|
1420
4305
|
if (option(argv, "--to") !== String(SCHEMA_VERSION)) {
|
|
@@ -1440,13 +4325,22 @@ async function runAdmin(argv = process.argv.slice(2)) {
|
|
|
1440
4325
|
}
|
|
1441
4326
|
if (expectedHash !== verified.manifest.sha256)
|
|
1442
4327
|
throw new Error("restore manifest hash mismatch");
|
|
1443
|
-
const
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
}
|
|
1448
|
-
|
|
4328
|
+
const maintenanceOwner = option(argv, "--maintenance-owner");
|
|
4329
|
+
const maintenanceConfirmation = option(argv, "--maintenance-confirm");
|
|
4330
|
+
if (maintenanceOwner === undefined !== (maintenanceConfirmation === undefined)) {
|
|
4331
|
+
throw new Error("retained maintenance recovery requires both owner and confirmation");
|
|
4332
|
+
}
|
|
4333
|
+
const recovery = maintenanceOwner && maintenanceConfirmation ? {
|
|
4334
|
+
ownerID: maintenanceOwner,
|
|
4335
|
+
confirmation: maintenanceConfirmation
|
|
4336
|
+
} : undefined;
|
|
4337
|
+
if (recovery && recovery.confirmation !== "RECOVER_RETAINED_MAINTENANCE_GATE") {
|
|
4338
|
+
throw new Error("invalid retained maintenance recovery confirmation");
|
|
1449
4339
|
}
|
|
4340
|
+
return withExclusiveMaintenance(databasePath, verified.manifest.sourceSchema, (maintenance) => {
|
|
4341
|
+
const preservedPath = restoreVerifiedBackup(manifestPath, databasePath, confirmation, maintenance, expectedHash);
|
|
4342
|
+
return { restored: true, preservedPath, manifest: verified.manifest };
|
|
4343
|
+
}, recovery);
|
|
1450
4344
|
}
|
|
1451
4345
|
if (command === "unlock") {
|
|
1452
4346
|
const ownerID = option(argv, "--owner");
|
|
@@ -1465,14 +4359,26 @@ async function runAdmin(argv = process.argv.slice(2)) {
|
|
|
1465
4359
|
try {
|
|
1466
4360
|
const now = Date.now();
|
|
1467
4361
|
let queued = 0;
|
|
1468
|
-
|
|
4362
|
+
let purges = 0;
|
|
4363
|
+
const quarantined = {};
|
|
4364
|
+
let generation = 0;
|
|
1469
4365
|
const insert = opened.db.query(`
|
|
1470
|
-
INSERT
|
|
1471
|
-
(backend, operation, project_id, note_id, revision, content_hash,
|
|
1472
|
-
state, attempt_count, available_at,
|
|
1473
|
-
|
|
4366
|
+
INSERT INTO index_outbox
|
|
4367
|
+
(backend, operation_key, operation, project_id, note_id, revision, content_hash,
|
|
4368
|
+
generation, lease_generation, fence, state, attempt_count, available_at,
|
|
4369
|
+
heartbeat_at, created_at)
|
|
4370
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0, 0, 'pending', 0, ?, NULL, ?)
|
|
1474
4371
|
`);
|
|
1475
|
-
opened.db.
|
|
4372
|
+
opened.db.exec("BEGIN IMMEDIATE");
|
|
4373
|
+
try {
|
|
4374
|
+
generation = opened.db.query("SELECT COALESCE(MAX(generation), 0) + 1 AS generation FROM index_outbox WHERE backend = ?").get(backend).generation;
|
|
4375
|
+
const projects = opened.db.query("SELECT id FROM projects ORDER BY id").all();
|
|
4376
|
+
const notes = opened.db.query("SELECT * FROM notes WHERE status = 'active' ORDER BY project_id, id").all();
|
|
4377
|
+
for (const project of projects) {
|
|
4378
|
+
const operation = "purge-project";
|
|
4379
|
+
const operationKey = outboxOperationKey(backend, operation, project.id, null, null, null, generation);
|
|
4380
|
+
purges += insert.run(backend, operationKey, operation, project.id, null, null, null, generation, now, now).changes;
|
|
4381
|
+
}
|
|
1476
4382
|
for (const note of notes) {
|
|
1477
4383
|
const document = deriveDocument({
|
|
1478
4384
|
projectID: note.project_id,
|
|
@@ -1483,10 +4389,22 @@ async function runAdmin(argv = process.argv.slice(2)) {
|
|
|
1483
4389
|
summary: note.summary,
|
|
1484
4390
|
content: note.content
|
|
1485
4391
|
});
|
|
1486
|
-
|
|
4392
|
+
if (!document) {
|
|
4393
|
+
quarantined.derived_document_unavailable = (quarantined.derived_document_unavailable ?? 0) + 1;
|
|
4394
|
+
continue;
|
|
4395
|
+
}
|
|
4396
|
+
const operation = "upsert-note";
|
|
4397
|
+
const operationKey = outboxOperationKey(backend, operation, note.project_id, note.id, note.current_revision, document.contentHash, generation);
|
|
4398
|
+
queued += insert.run(backend, operationKey, operation, note.project_id, note.id, note.current_revision, document.contentHash, generation, now, now).changes;
|
|
1487
4399
|
}
|
|
1488
|
-
|
|
1489
|
-
|
|
4400
|
+
opened.db.exec("COMMIT");
|
|
4401
|
+
} catch (error) {
|
|
4402
|
+
try {
|
|
4403
|
+
opened.db.exec("ROLLBACK");
|
|
4404
|
+
} catch {}
|
|
4405
|
+
throw error;
|
|
4406
|
+
}
|
|
4407
|
+
return { backend, generation, purges, queued, quarantined };
|
|
1490
4408
|
} finally {
|
|
1491
4409
|
opened.close();
|
|
1492
4410
|
}
|
|
@@ -1504,7 +4422,8 @@ async function runAdmin(argv = process.argv.slice(2)) {
|
|
|
1504
4422
|
return withDatabase(databasePath, (db) => {
|
|
1505
4423
|
const result = db.query(`UPDATE index_outbox
|
|
1506
4424
|
SET state = 'pending', available_at = ?, lease_owner = NULL,
|
|
1507
|
-
lease_expires_at = NULL,
|
|
4425
|
+
lease_expires_at = NULL, heartbeat_at = NULL,
|
|
4426
|
+
completed_at = NULL, last_error_code = NULL
|
|
1508
4427
|
WHERE id = ? AND state = 'dead'`).run(Date.now(), id);
|
|
1509
4428
|
return { id, retried: result.changes === 1 };
|
|
1510
4429
|
});
|
|
@@ -1516,28 +4435,30 @@ async function runAdmin(argv = process.argv.slice(2)) {
|
|
|
1516
4435
|
}));
|
|
1517
4436
|
}
|
|
1518
4437
|
if (command === "backup" && subcommand === "prune") {
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
4438
|
+
return withExclusiveMaintenance(databasePath, SCHEMA_VERSION, () => {
|
|
4439
|
+
const entries = backupEntries(databasePath);
|
|
4440
|
+
const root = resolve3(`${databasePath}.backup`);
|
|
4441
|
+
const digest = createHash7("sha256").update(`${resolve3(databasePath)}\x00${root}
|
|
4442
|
+
${entries.map((entry) => `${basename3(entry.manifest)}\x00${basename3(entry.database)}\x00${entry.sha256}\x00${entry.size}\x00${entry.manifestHash}`).join(`
|
|
1523
4443
|
`)}`).digest("hex");
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
4444
|
+
if (option(argv, "--confirm") !== "DELETE_VERIFIED_BACKUPS") {
|
|
4445
|
+
return { dryRun: true, digest, backups: entries };
|
|
4446
|
+
}
|
|
4447
|
+
if (option(argv, "--digest") !== digest)
|
|
4448
|
+
throw new Error("backup prune digest mismatch");
|
|
4449
|
+
const currentEntries = entries.map((entry) => {
|
|
4450
|
+
const current = verifiedBackupEntry(root, entry.manifest);
|
|
4451
|
+
if (current.database !== entry.database || current.sha256 !== entry.sha256 || current.size !== entry.size || current.manifestHash !== entry.manifestHash) {
|
|
4452
|
+
throw new Error(`backup changed after confirmation: ${basename3(entry.manifest)}`);
|
|
4453
|
+
}
|
|
4454
|
+
return current;
|
|
4455
|
+
});
|
|
4456
|
+
for (const current of currentEntries) {
|
|
4457
|
+
rmSync3(current.database, { force: true });
|
|
4458
|
+
rmSync3(current.manifest, { force: true });
|
|
1533
4459
|
}
|
|
1534
|
-
return
|
|
4460
|
+
return { deleted: entries.length, digest };
|
|
1535
4461
|
});
|
|
1536
|
-
for (const current of currentEntries) {
|
|
1537
|
-
rmSync3(current.database, { force: true });
|
|
1538
|
-
rmSync3(current.manifest, { force: true });
|
|
1539
|
-
}
|
|
1540
|
-
return { deleted: entries.length, digest };
|
|
1541
4462
|
}
|
|
1542
4463
|
throw new Error(`unknown admin command: ${argv.join(" ")}`);
|
|
1543
4464
|
}
|
|
@@ -1549,16 +4470,29 @@ function withDatabase(databasePath, action) {
|
|
|
1549
4470
|
opened.close();
|
|
1550
4471
|
}
|
|
1551
4472
|
}
|
|
4473
|
+
function withExclusiveMaintenance(databasePath, schemaVersion, action, recovery) {
|
|
4474
|
+
const lock = acquireMigrationLock(databasePath, schemaVersion);
|
|
4475
|
+
try {
|
|
4476
|
+
const maintenance = acquireMaintenanceGate(databasePath, recovery);
|
|
4477
|
+
try {
|
|
4478
|
+
return action(maintenance);
|
|
4479
|
+
} finally {
|
|
4480
|
+
maintenance.release();
|
|
4481
|
+
}
|
|
4482
|
+
} finally {
|
|
4483
|
+
lock.release();
|
|
4484
|
+
}
|
|
4485
|
+
}
|
|
1552
4486
|
function option(argv, name) {
|
|
1553
4487
|
const index = argv.indexOf(name);
|
|
1554
4488
|
return index >= 0 ? argv[index + 1] : undefined;
|
|
1555
4489
|
}
|
|
1556
4490
|
function requireExistingDatabase(databasePath) {
|
|
1557
|
-
if (!
|
|
4491
|
+
if (!existsSync5(databasePath))
|
|
1558
4492
|
throw new Error(`database does not exist: ${databasePath}`);
|
|
1559
4493
|
}
|
|
1560
4494
|
function readSchemaVersion(databasePath) {
|
|
1561
|
-
const db = new
|
|
4495
|
+
const db = new Database4(databasePath, { readonly: true });
|
|
1562
4496
|
try {
|
|
1563
4497
|
return schemaVersion(db);
|
|
1564
4498
|
} finally {
|
|
@@ -1572,27 +4506,27 @@ function schemaVersion(db) {
|
|
|
1572
4506
|
return row.version;
|
|
1573
4507
|
}
|
|
1574
4508
|
function assertInsideBackupRoot(databasePath, manifestPath) {
|
|
1575
|
-
const root =
|
|
1576
|
-
const candidate =
|
|
1577
|
-
if (
|
|
4509
|
+
const root = resolve3(`${databasePath}.backup`);
|
|
4510
|
+
const candidate = resolve3(manifestPath);
|
|
4511
|
+
if (dirname3(candidate) !== root)
|
|
1578
4512
|
throw new Error("manifest must be inside the database backup directory");
|
|
1579
4513
|
}
|
|
1580
4514
|
function backupEntries(databasePath) {
|
|
1581
|
-
const root =
|
|
1582
|
-
if (!
|
|
4515
|
+
const root = resolve3(`${databasePath}.backup`);
|
|
4516
|
+
if (!existsSync5(root))
|
|
1583
4517
|
return [];
|
|
1584
|
-
return
|
|
4518
|
+
return readdirSync2(root).filter((name) => name.endsWith(".manifest.json")).sort().map((name) => verifiedBackupEntry(root, resolve3(root, name)));
|
|
1585
4519
|
}
|
|
1586
4520
|
function verifiedBackupEntry(root, manifest) {
|
|
1587
|
-
const candidate =
|
|
1588
|
-
if (
|
|
4521
|
+
const candidate = resolve3(manifest);
|
|
4522
|
+
if (dirname3(candidate) !== root)
|
|
1589
4523
|
throw new Error("backup manifest escaped the backup directory");
|
|
1590
|
-
const stat =
|
|
4524
|
+
const stat = lstatSync4(candidate);
|
|
1591
4525
|
if (!stat.isFile() || stat.isSymbolicLink())
|
|
1592
4526
|
throw new Error("backup manifest must be a regular file");
|
|
1593
|
-
const bytes =
|
|
4527
|
+
const bytes = readFileSync4(candidate);
|
|
1594
4528
|
const verified = verifyBackupManifest(candidate);
|
|
1595
|
-
if (
|
|
4529
|
+
if (dirname3(verified.databasePath) !== root) {
|
|
1596
4530
|
throw new Error("backup database escaped the backup directory");
|
|
1597
4531
|
}
|
|
1598
4532
|
return {
|
|
@@ -1600,9 +4534,20 @@ function verifiedBackupEntry(root, manifest) {
|
|
|
1600
4534
|
database: verified.databasePath,
|
|
1601
4535
|
sha256: verified.manifest.sha256,
|
|
1602
4536
|
size: verified.manifest.size,
|
|
1603
|
-
manifestHash:
|
|
4537
|
+
manifestHash: createHash7("sha256").update(bytes).digest("hex")
|
|
1604
4538
|
};
|
|
1605
4539
|
}
|
|
4540
|
+
function outboxOperationKey(backend, operation, projectID, noteID, revision, contentHash, generation) {
|
|
4541
|
+
return hashTuple("outbox-operation", 2, [
|
|
4542
|
+
backend,
|
|
4543
|
+
operation,
|
|
4544
|
+
projectID,
|
|
4545
|
+
noteID,
|
|
4546
|
+
revision,
|
|
4547
|
+
contentHash,
|
|
4548
|
+
generation
|
|
4549
|
+
]);
|
|
4550
|
+
}
|
|
1606
4551
|
if (import.meta.main) {
|
|
1607
4552
|
try {
|
|
1608
4553
|
const result = await runAdmin();
|