@sema-agent/core 5.35.0 → 5.37.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +115 -0
- package/dist/agents/subagent.d.ts +10 -0
- package/dist/agents/subagent.js +29 -2
- package/dist/core/auto-compaction.d.ts +23 -0
- package/dist/core/auto-compaction.js +8 -0
- package/dist/core/checkpoint-store.d.ts +16 -0
- package/dist/core/context-guard.d.ts +41 -0
- package/dist/core/context-guard.js +76 -0
- package/dist/core/governance-codes.js +4 -0
- package/dist/core/memory-engine/engine.d.ts +142 -0
- package/dist/core/memory-engine/engine.js +265 -3
- package/dist/core/memory-engine/file-backend.d.ts +490 -16
- package/dist/core/memory-engine/file-backend.js +1099 -36
- package/dist/core/memory-engine/index.d.ts +2 -2
- package/dist/core/memory-engine/index.js +1 -1
- package/dist/core/memory-engine/layout.d.ts +42 -2
- package/dist/core/memory-engine/layout.js +76 -12
- package/dist/core/memory-engine/memory-backend-contract.d.ts +13 -0
- package/dist/core/memory-engine/memory-backend-contract.js +89 -0
- package/dist/core/park-selfcheck.d.ts +5 -0
- package/dist/core/protocol-table.d.ts +4 -4
- package/dist/core/runner/assemble-result.d.ts +8 -0
- package/dist/core/runner/assemble-result.js +4 -1
- package/dist/core/runner/git-status-frame.d.ts +219 -0
- package/dist/core/runner/git-status-frame.js +212 -0
- package/dist/core/runner/prepare-memory.d.ts +11 -1
- package/dist/core/runner/prepare-memory.js +48 -2
- package/dist/core/runner/prepare-task.d.ts +21 -0
- package/dist/core/runner/prepare-task.js +28 -35
- package/dist/core/runner/runtask.js +270 -5
- package/dist/core/task-registry-agent.d.ts +15 -0
- package/dist/core/task-registry-agent.js +9 -0
- package/dist/core/task-registry.d.ts +3 -0
- package/dist/core/task-registry.js +4 -1
- package/dist/core/types.d.ts +122 -7
- package/dist/engine/harness/types.d.ts +65 -1
- package/dist/engine/harness/types.js +20 -0
- package/dist/engine/session/import-validate.js +10 -1
- package/dist/engine/session/session.d.ts +37 -1
- package/dist/engine/session/session.js +56 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.js +1 -1
- package/dist/internal/harness-types.d.ts +1 -0
- package/dist/internal/harness.d.ts +2 -0
- package/dist/internal/harness.js +2 -0
- package/dist/prompt-assembly/epoch.js +1 -1
- package/dist/prompt-assembly/event-registry.js +1 -0
- package/dist/prompts/default.d.ts +20 -7
- package/dist/prompts/default.js +2 -7
- package/package.json +1 -1
- package/test/export-surface.snapshot.json +13 -1
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import { lstatSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, openSync, fsyncSync, closeSync } from "node:fs";
|
|
1
|
+
import { existsSync, lstatSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, openSync, fsyncSync, closeSync } from "node:fs";
|
|
2
2
|
import { createHash, randomUUID } from "node:crypto";
|
|
3
3
|
import { dirname, join, relative } from "node:path";
|
|
4
4
|
import { jaccardDistance, termSet } from "../memory-vector.js";
|
|
5
5
|
import { MAX_MEMORY_BYTES } from "../memory.js";
|
|
6
6
|
import { inlineUntrusted } from "../untrusted-text.js";
|
|
7
7
|
import { computeEntryRev, entryFromFile, isValidEntryId, parseEntryFile, serializeEntryFile } from "./frontmatter.js";
|
|
8
|
-
import { ControlPlaneCorruptError, CURSORS_FILE, QUARANTINE_DIR, atomicWriteFileSync, quarantineAndTombstone, claimRootScope, adoptCanonicalKeyedControlDir, deriveControlPlaneDir, ensureDirExists, enqueueMemoryAnnouncement, isContainedIn, registerScope, registeredScopes, resolveMemoryEngineRoot, scopeDirFor, scopeDirName, writeAllSync, } from "./layout.js";
|
|
8
|
+
import { ControlPlaneCorruptError, CURSORS_FILE, QUARANTINE_DIR, atomicWriteFileSync, quarantineAndTombstone, captureAndClearLineageForEntries, claimRootScope, adoptCanonicalKeyedControlDir, deriveControlPlaneDir, ensureDirExists, enqueueMemoryAnnouncement, isContainedIn, lineageContributionsOfSession, readLineageRecord, registerScope, registeredScopes, resolveMemoryEngineRoot, scopeDirFor, scopeDirName, writeAllSync, } from "./layout.js";
|
|
9
9
|
import { scanMemoryFileName, scanMemoryWrite } from "./scan.js";
|
|
10
10
|
export const MEMORY_INDEX_FILENAME = "MEMORY.md";
|
|
11
11
|
export const DEFAULT_MAX_ENTRY_DEPTH = 3;
|
|
@@ -62,10 +62,98 @@ export function scanEntryFiles(dir, opts = {}) {
|
|
|
62
62
|
walk(dir, 0);
|
|
63
63
|
return out;
|
|
64
64
|
}
|
|
65
|
+
export function canonicalJsonStringify(value) {
|
|
66
|
+
if (Array.isArray(value))
|
|
67
|
+
return `[${value.map((v) => (v === undefined || typeof v === "function" || typeof v === "symbol" ? "null" : canonicalJsonStringify(v))).join(",")}]`;
|
|
68
|
+
if (value !== null && typeof value === "object") {
|
|
69
|
+
const rec = value;
|
|
70
|
+
const keys = Object.keys(rec)
|
|
71
|
+
.filter((k) => rec[k] !== undefined)
|
|
72
|
+
.sort();
|
|
73
|
+
return `{${keys.map((k) => `${JSON.stringify(k)}:${canonicalJsonStringify(rec[k])}`).join(",")}}`;
|
|
74
|
+
}
|
|
75
|
+
return JSON.stringify(value);
|
|
76
|
+
}
|
|
77
|
+
export function erasureSelectHash(select) {
|
|
78
|
+
return createHash("sha256").update(`${canonicalJsonStringify(select)}\n`, "utf8").digest("hex");
|
|
79
|
+
}
|
|
80
|
+
export function erasureRequestInvalid(input) {
|
|
81
|
+
if (!input || typeof input !== "object")
|
|
82
|
+
return "input is not an object";
|
|
83
|
+
if (typeof input.requestId !== "string" || input.requestId.length === 0)
|
|
84
|
+
return "requestId is required (idempotency identity — a retry must reuse it; the engine never mints one)";
|
|
85
|
+
const sel = input.select;
|
|
86
|
+
if (!sel || typeof sel !== "object" || Array.isArray(sel))
|
|
87
|
+
return "select is not an object";
|
|
88
|
+
const rec = sel;
|
|
89
|
+
const keys = Object.keys(rec);
|
|
90
|
+
const kind = keys[0];
|
|
91
|
+
if (keys.length !== 1 || (kind !== "ids" && kind !== "scope" && kind !== "sessionId"))
|
|
92
|
+
return "select must carry exactly one of ids/scope/sessionId";
|
|
93
|
+
if (kind === "ids") {
|
|
94
|
+
const list = Array.isArray(rec.ids) ? rec.ids : undefined;
|
|
95
|
+
if (list === undefined)
|
|
96
|
+
return "select.ids is not an array";
|
|
97
|
+
const seen = new Set();
|
|
98
|
+
for (const id of list) {
|
|
99
|
+
if (typeof id !== "string" || id.length === 0)
|
|
100
|
+
return "select.ids carries a non-string/empty id";
|
|
101
|
+
if (seen.has(id))
|
|
102
|
+
return "select.ids carries a duplicate id (the pinned set is a set — deduplicate before calling)";
|
|
103
|
+
seen.add(id);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
else {
|
|
107
|
+
const v = rec[kind];
|
|
108
|
+
if (typeof v !== "string" || v.length === 0)
|
|
109
|
+
return `select.${kind} is not a non-empty string`;
|
|
110
|
+
}
|
|
111
|
+
if (input.allowUnevidenced !== undefined && typeof input.allowUnevidenced !== "boolean")
|
|
112
|
+
return "allowUnevidenced must be a boolean when present";
|
|
113
|
+
return undefined;
|
|
114
|
+
}
|
|
115
|
+
function erasureCodedError(message, code) {
|
|
116
|
+
const e = new Error(message);
|
|
117
|
+
e.code = code;
|
|
118
|
+
return e;
|
|
119
|
+
}
|
|
120
|
+
function cloneErasureSelect(select) {
|
|
121
|
+
if ("ids" in select)
|
|
122
|
+
return { ids: [...select.ids] };
|
|
123
|
+
if ("scope" in select)
|
|
124
|
+
return { scope: select.scope };
|
|
125
|
+
return { sessionId: select.sessionId };
|
|
126
|
+
}
|
|
127
|
+
export function captureErasureInput(input) {
|
|
128
|
+
if (!input || typeof input !== "object")
|
|
129
|
+
return input;
|
|
130
|
+
const requestId = input.requestId;
|
|
131
|
+
const allowUnevidenced = input.allowUnevidenced;
|
|
132
|
+
const rawSel = input.select;
|
|
133
|
+
let select;
|
|
134
|
+
if (rawSel && typeof rawSel === "object" && !Array.isArray(rawSel)) {
|
|
135
|
+
const s = rawSel;
|
|
136
|
+
const captured = {};
|
|
137
|
+
for (const k of Object.keys(s)) {
|
|
138
|
+
const v = s[k];
|
|
139
|
+
captured[k] = Array.isArray(v) ? [...v] : v;
|
|
140
|
+
}
|
|
141
|
+
select = captured;
|
|
142
|
+
}
|
|
143
|
+
else {
|
|
144
|
+
select = rawSel;
|
|
145
|
+
}
|
|
146
|
+
return { requestId, select, ...(allowUnevidenced !== undefined ? { allowUnevidenced } : {}) };
|
|
147
|
+
}
|
|
148
|
+
class AuditSnapshotContendedError extends Error {
|
|
149
|
+
}
|
|
150
|
+
class AuditObservationUnstableError extends Error {
|
|
151
|
+
}
|
|
65
152
|
const LEDGER_FILE = "revs.json";
|
|
66
153
|
const LEDGER_SCHEMA_VERSION = 2;
|
|
67
154
|
const LEDGER_V1_BACKUP_FILE = "revs.v1.json.bak";
|
|
68
155
|
const TRANSFERS_FILE = "transfers.jsonl";
|
|
156
|
+
const CHAIN_DEGRADED_FILE = "transfers.chain-degraded.json";
|
|
69
157
|
const JOURNAL_FILE = "journal.json";
|
|
70
158
|
const TXN_LOCK_STALE_MS = 30_000;
|
|
71
159
|
const TXN_LOCK_WAIT_MS = 15_000;
|
|
@@ -113,7 +201,7 @@ function bindRowTracked(rows, id, next, transfers, channel) {
|
|
|
113
201
|
rows[id] = row;
|
|
114
202
|
}
|
|
115
203
|
function cloneRows(rows) {
|
|
116
|
-
const out =
|
|
204
|
+
const out = Object.create(null);
|
|
117
205
|
for (const [id, row] of rowsEntries(rows))
|
|
118
206
|
out[id] = { ...row, ...(row.prev !== undefined ? { prev: { ...row.prev } } : {}) };
|
|
119
207
|
return out;
|
|
@@ -130,7 +218,7 @@ function validateLedgerRowsV2(rowsRaw, path) {
|
|
|
130
218
|
if (!rowsRaw || typeof rowsRaw !== "object" || Array.isArray(rowsRaw)) {
|
|
131
219
|
throw new ControlPlaneCorruptError(`committed-rev ledger rows have the wrong shape: ${path}`);
|
|
132
220
|
}
|
|
133
|
-
const out =
|
|
221
|
+
const out = Object.create(null);
|
|
134
222
|
for (const [id, rowRaw] of Object.entries(rowsRaw)) {
|
|
135
223
|
if (!rowRaw || typeof rowRaw !== "object" || Array.isArray(rowRaw)) {
|
|
136
224
|
throw new ControlPlaneCorruptError(`committed-rev ledger row ${JSON.stringify(id)} has the wrong shape: ${path}`);
|
|
@@ -197,7 +285,7 @@ function parseLedgerText(raw, path) {
|
|
|
197
285
|
}
|
|
198
286
|
return { form: "v2", rows, envelopeExtras };
|
|
199
287
|
}
|
|
200
|
-
const rows =
|
|
288
|
+
const rows = Object.create(null);
|
|
201
289
|
for (const [k, v] of Object.entries(rec)) {
|
|
202
290
|
if (typeof v !== "string") {
|
|
203
291
|
throw new ControlPlaneCorruptError(`committed-rev ledger entry ${JSON.stringify(k)} is not a string: ${path}`);
|
|
@@ -206,6 +294,7 @@ function parseLedgerText(raw, path) {
|
|
|
206
294
|
}
|
|
207
295
|
return { form: "v1", rows };
|
|
208
296
|
}
|
|
297
|
+
const KNOWN_TRANSFER_CHANNELS = new Set(["adopted-move", "applyPatches-move", "migration-bind", "migration", "delete", "erasure-anchor", "erasure-request"]);
|
|
209
298
|
function transferEventInvalid(raw) {
|
|
210
299
|
if (!raw || typeof raw !== "object" || Array.isArray(raw))
|
|
211
300
|
return "event is not an object";
|
|
@@ -218,8 +307,36 @@ function transferEventInvalid(raw) {
|
|
|
218
307
|
if (!b || typeof b !== "object" || Array.isArray(b))
|
|
219
308
|
return true;
|
|
220
309
|
const r = b;
|
|
221
|
-
|
|
310
|
+
if (typeof r.scope !== "string" || r.scope.length === 0)
|
|
311
|
+
return true;
|
|
312
|
+
return typeof r.slug !== "string" || slugEscapes(r.slug);
|
|
313
|
+
};
|
|
314
|
+
const idListInvalid = (v, what) => {
|
|
315
|
+
const list = Array.isArray(v) ? v : undefined;
|
|
316
|
+
if (list === undefined)
|
|
317
|
+
return `${what} is not an array`;
|
|
318
|
+
const seen = new Set();
|
|
319
|
+
for (const id of list) {
|
|
320
|
+
if (typeof id !== "string" || id.length === 0)
|
|
321
|
+
return `${what} carries a non-string/empty id`;
|
|
322
|
+
if (seen.has(id))
|
|
323
|
+
return `${what} carries a duplicate id`;
|
|
324
|
+
seen.add(id);
|
|
325
|
+
}
|
|
326
|
+
return undefined;
|
|
222
327
|
};
|
|
328
|
+
if ((e.origin !== undefined) !== (e.srcEv !== undefined))
|
|
329
|
+
return "'origin' and 'srcEv' must appear together (import rows carry both; local rows carry neither)";
|
|
330
|
+
if (e.origin !== undefined && (typeof e.origin !== "string" || e.origin.length === 0))
|
|
331
|
+
return "non-string/empty 'origin'";
|
|
332
|
+
if (e.srcEv !== undefined && (typeof e.srcEv !== "string" || e.srcEv.length === 0))
|
|
333
|
+
return "non-string/empty 'srcEv'";
|
|
334
|
+
if (e.redacted !== undefined) {
|
|
335
|
+
if (e.redacted !== true)
|
|
336
|
+
return "'redacted', when present, must be literal true";
|
|
337
|
+
if (e.origin === undefined)
|
|
338
|
+
return "'redacted' requires the import pair (origin/srcEv) — a local row is never redacted";
|
|
339
|
+
}
|
|
223
340
|
if (e.channel === "migration") {
|
|
224
341
|
if (typeof e.boundRows !== "number" || !Number.isInteger(e.boundRows))
|
|
225
342
|
return "migration summary missing integer 'boundRows'";
|
|
@@ -243,6 +360,91 @@ function transferEventInvalid(raw) {
|
|
|
243
360
|
}
|
|
244
361
|
return undefined;
|
|
245
362
|
}
|
|
363
|
+
if (e.channel === "delete") {
|
|
364
|
+
if (typeof e.id !== "string" || e.id.length === 0)
|
|
365
|
+
return "delete event missing 'id'";
|
|
366
|
+
if (typeof e.rev !== "string" || e.rev.length === 0)
|
|
367
|
+
return "delete event missing 'rev'";
|
|
368
|
+
if (e.boundRows !== undefined || e.unboundRows !== undefined || e.to !== undefined || e.select !== undefined)
|
|
369
|
+
return "delete event must not carry boundRows/unboundRows/to/select";
|
|
370
|
+
if (e.from !== undefined && bindingInvalid(e.from))
|
|
371
|
+
return "delete event has a malformed 'from'";
|
|
372
|
+
if (e.req !== undefined && (typeof e.req !== "string" || e.req.length === 0))
|
|
373
|
+
return "delete event has a non-string/empty 'req'";
|
|
374
|
+
if (e.sessions !== undefined) {
|
|
375
|
+
const sessions = Array.isArray(e.sessions) ? e.sessions : undefined;
|
|
376
|
+
if (sessions === undefined || sessions.some((s) => typeof s !== "string"))
|
|
377
|
+
return "delete event has a malformed 'sessions'";
|
|
378
|
+
if (e.req === undefined)
|
|
379
|
+
return "delete event carries 'sessions' without 'req' (only the request leg captures lineage)";
|
|
380
|
+
}
|
|
381
|
+
if ((e.legacyId !== undefined) === isValidEntryId(e.id)) {
|
|
382
|
+
return e.legacyId !== undefined ? "legacyId claimed for a well-formed entry id (downgrade laundering refused)" : "delete event for a non-entry-id ledger key must declare legacyId";
|
|
383
|
+
}
|
|
384
|
+
if (e.legacyId !== undefined && e.legacyId !== true)
|
|
385
|
+
return "legacyId, when present, must be literal true";
|
|
386
|
+
if (e.legacyId !== undefined && e.from !== undefined)
|
|
387
|
+
return "legacyId and 'from' are mutually exclusive (a legacy row has no derivable projection)";
|
|
388
|
+
return undefined;
|
|
389
|
+
}
|
|
390
|
+
if (e.channel === "erasure-anchor") {
|
|
391
|
+
if (typeof e.req !== "string" || e.req.length === 0)
|
|
392
|
+
return "erasure anchor missing 'req'";
|
|
393
|
+
if (e.id !== undefined || e.from !== undefined || e.to !== undefined || e.scope !== undefined || e.boundRows !== undefined)
|
|
394
|
+
return "erasure anchor must not carry id/from/to/scope/boundRows";
|
|
395
|
+
const sel = e.select;
|
|
396
|
+
if (!sel || typeof sel !== "object" || Array.isArray(sel))
|
|
397
|
+
return "erasure anchor 'select' is not an object";
|
|
398
|
+
const selRec = sel;
|
|
399
|
+
const selKeys = Object.keys(selRec);
|
|
400
|
+
if (selKeys.length !== 1 || (selKeys[0] !== "ids" && selKeys[0] !== "scope" && selKeys[0] !== "sessionId"))
|
|
401
|
+
return "erasure anchor 'select' must carry exactly one of ids/scope/sessionId";
|
|
402
|
+
if (selKeys[0] === "ids") {
|
|
403
|
+
const bad = idListInvalid(selRec.ids, "'select.ids'");
|
|
404
|
+
if (bad !== undefined)
|
|
405
|
+
return bad;
|
|
406
|
+
}
|
|
407
|
+
else if (typeof selRec[selKeys[0]] !== "string" || selRec[selKeys[0]].length === 0) {
|
|
408
|
+
return `erasure anchor 'select.${selKeys[0]}' is not a non-empty string`;
|
|
409
|
+
}
|
|
410
|
+
if (e.selectHash !== erasureSelectHash(sel))
|
|
411
|
+
return "erasure anchor 'selectHash' does not equal the selector's canonical hash";
|
|
412
|
+
const idsBad = idListInvalid(e.ids, "erasure anchor 'ids'");
|
|
413
|
+
if (idsBad !== undefined)
|
|
414
|
+
return idsBad;
|
|
415
|
+
const nfBad = idListInvalid(e.notFound, "erasure anchor 'notFound'");
|
|
416
|
+
if (nfBad !== undefined)
|
|
417
|
+
return nfBad;
|
|
418
|
+
const pinned = new Set(e.ids);
|
|
419
|
+
for (const id of e.notFound) {
|
|
420
|
+
if (!pinned.has(id))
|
|
421
|
+
return "erasure anchor 'notFound' is not a subset of 'ids'";
|
|
422
|
+
}
|
|
423
|
+
if (selKeys[0] === "ids") {
|
|
424
|
+
const selIds = selRec.ids;
|
|
425
|
+
const anchorIds = e.ids;
|
|
426
|
+
if (selIds.length !== anchorIds.length || selIds.some((id, i) => anchorIds[i] !== id)) {
|
|
427
|
+
return "erasure anchor 'ids' does not equal the ids-selector's verbatim id list (a pinned set the selector never named)";
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
return undefined;
|
|
431
|
+
}
|
|
432
|
+
if (e.channel === "erasure-request") {
|
|
433
|
+
if (typeof e.req !== "string" || e.req.length === 0)
|
|
434
|
+
return "erasure request row missing 'req'";
|
|
435
|
+
if (e.id !== undefined || e.from !== undefined || e.to !== undefined || e.select !== undefined || e.notFound !== undefined)
|
|
436
|
+
return "erasure request row must not carry id/from/to/select/notFound";
|
|
437
|
+
if (typeof e.scope !== "string" || e.scope.length === 0)
|
|
438
|
+
return "erasure request row missing 'scope'";
|
|
439
|
+
if (typeof e.selectHash !== "string" || !/^[0-9a-f]{64}$/.test(e.selectHash))
|
|
440
|
+
return "erasure request row 'selectHash' is not a sha256 hex";
|
|
441
|
+
const idsBad = idListInvalid(e.ids, "erasure request row 'ids'");
|
|
442
|
+
if (idsBad !== undefined)
|
|
443
|
+
return idsBad;
|
|
444
|
+
if (e.ids.length === 0)
|
|
445
|
+
return "erasure request row carries an empty 'ids' (a projection row exists only for a scope with bound ids)";
|
|
446
|
+
return undefined;
|
|
447
|
+
}
|
|
246
448
|
return `unknown channel ${JSON.stringify(e.channel)}`;
|
|
247
449
|
}
|
|
248
450
|
function readControlFileOrAbsent(path, what) {
|
|
@@ -314,6 +516,7 @@ export class FileMemoryEngineBackend {
|
|
|
314
516
|
unboundRowsKnown = false;
|
|
315
517
|
adoptionNoticeKeys = new Set();
|
|
316
518
|
transfersAppendFault;
|
|
519
|
+
deletedIdsDigest;
|
|
317
520
|
inboundFindings = [];
|
|
318
521
|
batchScan;
|
|
319
522
|
constructor(dir, opts = {}) {
|
|
@@ -365,7 +568,7 @@ export class FileMemoryEngineBackend {
|
|
|
365
568
|
const path = join(this.controlPlaneRoot, LEDGER_FILE);
|
|
366
569
|
const raw = readControlFileOrAbsent(path, "committed-rev ledger");
|
|
367
570
|
if (raw === undefined) {
|
|
368
|
-
this.ledger =
|
|
571
|
+
this.ledger = Object.create(null);
|
|
369
572
|
this.persistedSchemaVersion = "v2";
|
|
370
573
|
this.unboundRowsKnown = false;
|
|
371
574
|
return this.ledger;
|
|
@@ -391,7 +594,7 @@ export class FileMemoryEngineBackend {
|
|
|
391
594
|
this.unboundRowsKnown = hasUnboundRows(this.ledger);
|
|
392
595
|
}
|
|
393
596
|
persistLedgerV1(rows) {
|
|
394
|
-
const map =
|
|
597
|
+
const map = Object.create(null);
|
|
395
598
|
for (const [id, row] of rowsEntries(rows))
|
|
396
599
|
map[id] = row.rev;
|
|
397
600
|
atomicWriteFileSync(join(this.controlPlaneRoot, LEDGER_FILE), `${JSON.stringify(map, null, 2)}\n`);
|
|
@@ -610,7 +813,7 @@ export class FileMemoryEngineBackend {
|
|
|
610
813
|
return;
|
|
611
814
|
}
|
|
612
815
|
const now = this.now();
|
|
613
|
-
const next =
|
|
816
|
+
const next = Object.create(null);
|
|
614
817
|
const bound = [];
|
|
615
818
|
const unbound = [];
|
|
616
819
|
const events = [];
|
|
@@ -654,12 +857,38 @@ export class FileMemoryEngineBackend {
|
|
|
654
857
|
: `memory ledger unbound row(s) converged to bound: ${bound.slice(0, 8).join(", ")}${bound.length > 8 ? ", …" : ""}`,
|
|
655
858
|
]);
|
|
656
859
|
}
|
|
657
|
-
|
|
860
|
+
chainDegradedMarkerPresent() {
|
|
861
|
+
return readControlFileOrAbsent(join(this.controlPlaneRoot, CHAIN_DEGRADED_FILE), "evidence-chain degradation marker") !== undefined;
|
|
862
|
+
}
|
|
863
|
+
mintChainDegradedMarker(fragmentPath) {
|
|
864
|
+
const path = join(this.controlPlaneRoot, CHAIN_DEGRADED_FILE);
|
|
865
|
+
let fd;
|
|
866
|
+
try {
|
|
867
|
+
fd = openSync(path, "wx", 0o600);
|
|
868
|
+
}
|
|
869
|
+
catch (err) {
|
|
870
|
+
if (err.code === "EEXIST")
|
|
871
|
+
return;
|
|
872
|
+
throw new ControlPlaneCorruptError(`evidence-chain degradation marker could not be created at ${path} (${err.code ?? "io error"}) — heal aborted, the chain is unchanged (fail-closed)`);
|
|
873
|
+
}
|
|
874
|
+
try {
|
|
875
|
+
writeAllSync(fd, `${JSON.stringify({ at: this.now(), ...(fragmentPath !== undefined ? { fragment: fragmentPath } : {}) }, null, 2)}\n`);
|
|
876
|
+
fsyncSync(fd);
|
|
877
|
+
}
|
|
878
|
+
finally {
|
|
879
|
+
closeSync(fd);
|
|
880
|
+
}
|
|
881
|
+
if (readControlFileOrAbsent(path, "evidence-chain degradation marker") === undefined) {
|
|
882
|
+
throw new ControlPlaneCorruptError(`evidence-chain degradation marker vanished after its write at ${path} — heal aborted, the chain is unchanged (fail-closed)`);
|
|
883
|
+
}
|
|
884
|
+
}
|
|
885
|
+
readTransferChain() {
|
|
658
886
|
const path = join(this.controlPlaneRoot, TRANSFERS_FILE);
|
|
659
887
|
const raw = readControlFileOrAbsent(path, "transfer evidence log");
|
|
660
|
-
const
|
|
888
|
+
const out = [];
|
|
661
889
|
if (raw === undefined)
|
|
662
|
-
return
|
|
890
|
+
return out;
|
|
891
|
+
const byEv = new Map();
|
|
663
892
|
const lines = raw.split("\n");
|
|
664
893
|
for (let i = 0; i < lines.length; i++) {
|
|
665
894
|
const line = lines[i];
|
|
@@ -671,29 +900,149 @@ export class FileMemoryEngineBackend {
|
|
|
671
900
|
}
|
|
672
901
|
catch {
|
|
673
902
|
const isTail = lines.slice(i + 1).every((l) => l === "");
|
|
674
|
-
if (isTail)
|
|
903
|
+
if (!isTail)
|
|
904
|
+
throw new ControlPlaneCorruptError(`transfer evidence log line ${i + 1} is unparseable (not a torn tail — fail-closed): ${path}`);
|
|
905
|
+
let journalHasPendingEvidence = false;
|
|
906
|
+
const jraw = readControlFileOrAbsent(join(this.controlPlaneRoot, JOURNAL_FILE), "transaction journal");
|
|
907
|
+
if (jraw !== undefined) {
|
|
675
908
|
try {
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
909
|
+
const jparsed = JSON.parse(jraw);
|
|
910
|
+
const jt = typeof jparsed === "object" && jparsed !== null ? jparsed.transfers : undefined;
|
|
911
|
+
journalHasPendingEvidence = Array.isArray(jt) && jt.length > 0;
|
|
679
912
|
}
|
|
680
913
|
catch {
|
|
681
|
-
|
|
914
|
+
journalHasPendingEvidence = false;
|
|
682
915
|
}
|
|
683
|
-
this.enqueueExternalItems([`transfer evidence log had a torn tail (crash mid-append) — the fragment is quarantined and the sound prefix stands: ${TRANSFERS_FILE}`]);
|
|
684
|
-
break;
|
|
685
916
|
}
|
|
686
|
-
|
|
917
|
+
const outOfBand = !journalHasPendingEvidence;
|
|
918
|
+
let fragmentPath;
|
|
919
|
+
try {
|
|
920
|
+
ensureDirExists(join(this.controlPlaneRoot, QUARANTINE_DIR));
|
|
921
|
+
fragmentPath = join(this.controlPlaneRoot, QUARANTINE_DIR, `transfers-torn-tail-${this.now()}.fragment`);
|
|
922
|
+
atomicWriteFileSync(fragmentPath, line);
|
|
923
|
+
}
|
|
924
|
+
catch {
|
|
925
|
+
throw new ControlPlaneCorruptError(`transfer evidence log has a torn tail that could not be quarantined: ${path}`);
|
|
926
|
+
}
|
|
927
|
+
if (outOfBand)
|
|
928
|
+
this.mintChainDegradedMarker(fragmentPath);
|
|
929
|
+
try {
|
|
930
|
+
atomicWriteFileSync(path, lines.slice(0, i).join("\n") + (i > 0 ? "\n" : ""));
|
|
931
|
+
}
|
|
932
|
+
catch {
|
|
933
|
+
throw new ControlPlaneCorruptError(`transfer evidence log has a torn tail that could not be healed (truncation failed): ${path}`);
|
|
934
|
+
}
|
|
935
|
+
this.enqueueExternalItems([
|
|
936
|
+
outOfBand
|
|
937
|
+
? `transfer evidence log had a torn tail with NO transaction journal beside it (out-of-band damage) — the fragment is quarantined, the sound prefix stands, and the chain is durably marked degraded: new id-bearing files are refused adoption until an explicit rebuild clears ${CHAIN_DEGRADED_FILE}`
|
|
938
|
+
: `transfer evidence log had a torn tail (crash mid-append) — the fragment is quarantined and the sound prefix stands: ${TRANSFERS_FILE}`,
|
|
939
|
+
]);
|
|
940
|
+
break;
|
|
687
941
|
}
|
|
688
942
|
const invalid = transferEventInvalid(parsed);
|
|
689
943
|
if (invalid !== undefined)
|
|
690
944
|
throw new ControlPlaneCorruptError(`transfer evidence log line ${i + 1} is invalid (${invalid}): ${path}`);
|
|
691
|
-
|
|
945
|
+
const rec = parsed;
|
|
946
|
+
const ev = rec.ev;
|
|
947
|
+
const canonical = canonicalJsonStringify(rec);
|
|
948
|
+
const prior = byEv.get(ev);
|
|
949
|
+
if (prior !== undefined) {
|
|
950
|
+
if (prior !== canonical) {
|
|
951
|
+
throw new ControlPlaneCorruptError(`transfer evidence log carries event id ${JSON.stringify(ev)} twice with DIFFERENT payloads (line ${i + 1}) — one ev is one identity; refusing to pick a side (fail-closed): ${path}`);
|
|
952
|
+
}
|
|
953
|
+
continue;
|
|
954
|
+
}
|
|
955
|
+
byEv.set(ev, canonical);
|
|
956
|
+
out.push({ ev, canonical, parsed: rec });
|
|
957
|
+
}
|
|
958
|
+
return out;
|
|
959
|
+
}
|
|
960
|
+
readTransferEvents() {
|
|
961
|
+
const out = new Map();
|
|
962
|
+
for (const row of this.readTransferChain())
|
|
963
|
+
out.set(row.ev, row.canonical);
|
|
964
|
+
return out;
|
|
965
|
+
}
|
|
966
|
+
deletedIdsDigestLocked() {
|
|
967
|
+
const path = join(this.controlPlaneRoot, TRANSFERS_FILE);
|
|
968
|
+
let fp;
|
|
969
|
+
try {
|
|
970
|
+
const st = statSync(path);
|
|
971
|
+
fp = `${st.size}:${st.mtimeMs}`;
|
|
972
|
+
}
|
|
973
|
+
catch (err) {
|
|
974
|
+
if (err.code === "ENOENT")
|
|
975
|
+
return { state: "complete", ids: new Set() };
|
|
976
|
+
throw new ControlPlaneCorruptError(`transfer evidence log could not be probed (${err.code ?? "io error"}) at ${path} — a probe failure is not an empty chain (fail-closed)`);
|
|
977
|
+
}
|
|
978
|
+
const cached = this.deletedIdsDigest;
|
|
979
|
+
if (cached !== undefined && cached.fp === fp)
|
|
980
|
+
return { state: cached.state, ids: cached.ids };
|
|
981
|
+
const raw = readControlFileOrAbsent(path, "transfer evidence log");
|
|
982
|
+
if (raw === undefined)
|
|
983
|
+
return { state: "complete", ids: new Set() };
|
|
984
|
+
let state = "complete";
|
|
985
|
+
const ids = new Set();
|
|
986
|
+
const lines = raw.split("\n");
|
|
987
|
+
for (let i = 0; i < lines.length; i++) {
|
|
988
|
+
const line = lines[i];
|
|
989
|
+
if (line === undefined || line === "")
|
|
990
|
+
continue;
|
|
991
|
+
let parsed;
|
|
992
|
+
try {
|
|
993
|
+
parsed = JSON.parse(line);
|
|
994
|
+
}
|
|
995
|
+
catch {
|
|
996
|
+
if (lines.slice(i + 1).every((l) => l === "")) {
|
|
997
|
+
state = "damaged";
|
|
998
|
+
break;
|
|
999
|
+
}
|
|
1000
|
+
throw new ControlPlaneCorruptError(`transfer evidence log line ${i + 1} is unparseable (not a torn tail — fail-closed): ${path}`);
|
|
1001
|
+
}
|
|
1002
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
1003
|
+
throw new ControlPlaneCorruptError(`transfer evidence log line ${i + 1} is not an object (fail-closed): ${path}`);
|
|
1004
|
+
}
|
|
1005
|
+
const e = parsed;
|
|
1006
|
+
if (KNOWN_TRANSFER_CHANNELS.has(e.channel)) {
|
|
1007
|
+
const invalid = transferEventInvalid(parsed);
|
|
1008
|
+
if (invalid !== undefined)
|
|
1009
|
+
throw new ControlPlaneCorruptError(`transfer evidence log line ${i + 1} is invalid (${invalid}): ${path}`);
|
|
1010
|
+
if (e.channel === "delete")
|
|
1011
|
+
ids.add(e.id);
|
|
1012
|
+
}
|
|
1013
|
+
}
|
|
1014
|
+
this.deletedIdsDigest = { fp, state, ids };
|
|
1015
|
+
return { state, ids };
|
|
1016
|
+
}
|
|
1017
|
+
resurrectionBackstopVerdict(id) {
|
|
1018
|
+
const digest = this.deletedIdsDigestLocked();
|
|
1019
|
+
if (digest.ids.has(id))
|
|
1020
|
+
return "erased";
|
|
1021
|
+
if (digest.state === "damaged" || this.chainDegradedMarkerPresent())
|
|
1022
|
+
return "degraded";
|
|
1023
|
+
return "clear";
|
|
1024
|
+
}
|
|
1025
|
+
retrievalDeletedIds() {
|
|
1026
|
+
return this.deletedIdsDigestLocked().ids;
|
|
1027
|
+
}
|
|
1028
|
+
freshCommittedRowsPure() {
|
|
1029
|
+
try {
|
|
1030
|
+
const raw = readControlFileOrAbsent(join(this.controlPlaneRoot, LEDGER_FILE), "committed-rev ledger");
|
|
1031
|
+
if (raw === undefined)
|
|
1032
|
+
return Object.create(null);
|
|
1033
|
+
const parsed = parseLedgerText(raw, join(this.controlPlaneRoot, LEDGER_FILE));
|
|
1034
|
+
return parsed.form === "v2" ? parsed.rows : Object.create(null);
|
|
1035
|
+
}
|
|
1036
|
+
catch (err) {
|
|
1037
|
+
const code = err.code ?? (err instanceof ControlPlaneCorruptError ? "corrupt" : "io error");
|
|
1038
|
+
this.announceAdoptionNotice(`retrieval-ledger-fault|${code}`, `memory retrieval guard: the committed-rev ledger could not be read on the lock-free retrieval face (${err instanceof Error ? err.message : String(err)}) — ` +
|
|
1039
|
+
`ids carrying delete evidence are WITHHELD from retrieval (a legitimate re-add cannot be verified over an unreadable account) until the fault clears; ` +
|
|
1040
|
+
`entries with no delete evidence serve normally`);
|
|
1041
|
+
return Object.create(null);
|
|
692
1042
|
}
|
|
693
|
-
return evs;
|
|
694
1043
|
}
|
|
695
1044
|
precheckTransfersAppendable() {
|
|
696
|
-
this.
|
|
1045
|
+
this.readTransferEvents();
|
|
697
1046
|
const fd = openSync(join(this.controlPlaneRoot, TRANSFERS_FILE), "a", 0o600);
|
|
698
1047
|
closeSync(fd);
|
|
699
1048
|
}
|
|
@@ -701,13 +1050,35 @@ export class FileMemoryEngineBackend {
|
|
|
701
1050
|
if (events.length === 0)
|
|
702
1051
|
return;
|
|
703
1052
|
this.transfersAppendFault?.();
|
|
704
|
-
const existing = this.
|
|
705
|
-
const
|
|
706
|
-
|
|
1053
|
+
const existing = this.readTransferEvents();
|
|
1054
|
+
const batch = new Map();
|
|
1055
|
+
const toWrite = [];
|
|
1056
|
+
for (const e of events) {
|
|
1057
|
+
const canonical = canonicalJsonStringify(e);
|
|
1058
|
+
const inBatch = batch.get(e.ev);
|
|
1059
|
+
if (inBatch !== undefined) {
|
|
1060
|
+
if (inBatch !== canonical) {
|
|
1061
|
+
throw new ControlPlaneCorruptError(`transfer evidence batch mints event id ${JSON.stringify(e.ev)} twice with DIFFERENT payloads — one ev is one identity (minting bug, fail-closed)`);
|
|
1062
|
+
}
|
|
1063
|
+
continue;
|
|
1064
|
+
}
|
|
1065
|
+
batch.set(e.ev, canonical);
|
|
1066
|
+
const onChain = existing.get(e.ev);
|
|
1067
|
+
if (onChain !== undefined) {
|
|
1068
|
+
if (onChain !== canonical) {
|
|
1069
|
+
throw new ControlPlaneCorruptError(`transfer evidence append collides on event id ${JSON.stringify(e.ev)}: the chain already carries a DIFFERENT payload under it — one ev is one identity (fail-closed)`);
|
|
1070
|
+
}
|
|
1071
|
+
continue;
|
|
1072
|
+
}
|
|
1073
|
+
toWrite.push(canonical);
|
|
1074
|
+
}
|
|
1075
|
+
if (toWrite.length === 0)
|
|
707
1076
|
return;
|
|
1077
|
+
const raw = readControlFileOrAbsent(join(this.controlPlaneRoot, TRANSFERS_FILE), "transfer evidence log");
|
|
1078
|
+
const needsSeparator = raw !== undefined && raw.length > 0 && !raw.endsWith("\n");
|
|
708
1079
|
const fd = openSync(join(this.controlPlaneRoot, TRANSFERS_FILE), "a", 0o600);
|
|
709
1080
|
try {
|
|
710
|
-
writeAllSync(fd,
|
|
1081
|
+
writeAllSync(fd, (needsSeparator ? "\n" : "") + toWrite.map((c) => `${c}\n`).join(""));
|
|
711
1082
|
fsyncSync(fd);
|
|
712
1083
|
}
|
|
713
1084
|
finally {
|
|
@@ -715,9 +1086,21 @@ export class FileMemoryEngineBackend {
|
|
|
715
1086
|
}
|
|
716
1087
|
}
|
|
717
1088
|
recoverJournal(opts = {}) {
|
|
718
|
-
|
|
719
|
-
|
|
1089
|
+
if (opts.unlocked) {
|
|
1090
|
+
if (this.txnInFlight())
|
|
1091
|
+
return;
|
|
1092
|
+
const lock = this.tryAcquireTxnLockSync();
|
|
1093
|
+
if (lock === undefined)
|
|
1094
|
+
return;
|
|
1095
|
+
try {
|
|
1096
|
+
this.recoverJournal();
|
|
1097
|
+
}
|
|
1098
|
+
finally {
|
|
1099
|
+
lock.release();
|
|
1100
|
+
}
|
|
720
1101
|
return;
|
|
1102
|
+
}
|
|
1103
|
+
const jp = join(this.controlPlaneRoot, JOURNAL_FILE);
|
|
721
1104
|
const raw = readControlFileOrAbsent(jp, "transaction journal");
|
|
722
1105
|
if (raw !== undefined) {
|
|
723
1106
|
let journal;
|
|
@@ -752,7 +1135,7 @@ export class FileMemoryEngineBackend {
|
|
|
752
1135
|
const map = journal.ledger;
|
|
753
1136
|
if (!map || typeof map !== "object" || Array.isArray(map))
|
|
754
1137
|
throw new ControlPlaneCorruptError(`transaction journal ledger snapshot has the wrong shape: ${jp}`);
|
|
755
|
-
snapshotRows =
|
|
1138
|
+
snapshotRows = Object.create(null);
|
|
756
1139
|
for (const [id, rev] of Object.entries(map)) {
|
|
757
1140
|
if (typeof rev !== "string")
|
|
758
1141
|
throw new ControlPlaneCorruptError(`transaction journal v1 ledger snapshot entry ${JSON.stringify(id)} is not a string: ${jp}`);
|
|
@@ -768,16 +1151,150 @@ export class FileMemoryEngineBackend {
|
|
|
768
1151
|
if (journal.transfers !== undefined) {
|
|
769
1152
|
if (!Array.isArray(journal.transfers))
|
|
770
1153
|
throw new ControlPlaneCorruptError(`transaction journal transfers is not an array: ${jp}`);
|
|
1154
|
+
const journalEvs = new Map();
|
|
771
1155
|
for (const e of journal.transfers) {
|
|
772
1156
|
const invalid = transferEventInvalid(e);
|
|
773
1157
|
if (invalid !== undefined)
|
|
774
1158
|
throw new ControlPlaneCorruptError(`transaction journal transfer event is invalid (${invalid}): ${jp}`);
|
|
775
|
-
if (e.
|
|
776
|
-
throw new ControlPlaneCorruptError(`transaction journal
|
|
1159
|
+
if (e.origin !== undefined) {
|
|
1160
|
+
throw new ControlPlaneCorruptError(`transaction journal carries an origin-bearing (imported) transfer event — imports never ride a journal: ${jp}`);
|
|
1161
|
+
}
|
|
1162
|
+
const canonical = canonicalJsonStringify(e);
|
|
1163
|
+
const prior = journalEvs.get(e.ev);
|
|
1164
|
+
if (prior !== undefined && prior !== canonical) {
|
|
1165
|
+
throw new ControlPlaneCorruptError(`transaction journal mints event id ${JSON.stringify(e.ev)} twice with DIFFERENT payloads (fail-closed before redo): ${jp}`);
|
|
777
1166
|
}
|
|
1167
|
+
journalEvs.set(e.ev, canonical);
|
|
778
1168
|
}
|
|
779
1169
|
if (hasV1)
|
|
780
1170
|
throw new ControlPlaneCorruptError(`transaction journal carries transfers with a v1 snapshot (impossible form): ${jp}`);
|
|
1171
|
+
if (journalEvs.size > 0) {
|
|
1172
|
+
for (const row of this.readTransferChain()) {
|
|
1173
|
+
const j = journalEvs.get(row.ev);
|
|
1174
|
+
if (j !== undefined && j !== row.canonical) {
|
|
1175
|
+
throw new ControlPlaneCorruptError(`transaction journal event id ${JSON.stringify(row.ev)} collides with a DIFFERENT payload already on the evidence chain (fail-closed before redo): ${jp}`);
|
|
1176
|
+
}
|
|
1177
|
+
}
|
|
1178
|
+
}
|
|
1179
|
+
const transfers = journal.transfers;
|
|
1180
|
+
const rowPresent = (id) => Object.prototype.hasOwnProperty.call(snapshotRows, id);
|
|
1181
|
+
const deleteRows = transfers.filter((t) => t.channel === "delete");
|
|
1182
|
+
const anchorRows = transfers.filter((t) => t.channel === "erasure-anchor");
|
|
1183
|
+
const requestRows = transfers.filter((t) => t.channel === "erasure-request");
|
|
1184
|
+
let chainAnchors;
|
|
1185
|
+
for (const e of transfers) {
|
|
1186
|
+
if (e.channel === "adopted-move" || e.channel === "applyPatches-move" || e.channel === "migration-bind") {
|
|
1187
|
+
if (!bindingEquals(rowsBinding(snapshotRows, e.id), e.to)) {
|
|
1188
|
+
throw new ControlPlaneCorruptError(`transaction journal transfer event for ${JSON.stringify(e.id)} disagrees with the snapshot binding: ${jp}`);
|
|
1189
|
+
}
|
|
1190
|
+
continue;
|
|
1191
|
+
}
|
|
1192
|
+
if (e.channel !== "delete")
|
|
1193
|
+
continue;
|
|
1194
|
+
if (rowPresent(e.id)) {
|
|
1195
|
+
throw new ControlPlaneCorruptError(`transaction journal delete evidence for ${JSON.stringify(e.id)} disagrees with its snapshot (the row survives) — a delete claim over a surviving row is refused: ${jp}`);
|
|
1196
|
+
}
|
|
1197
|
+
if (e.legacyId !== true) {
|
|
1198
|
+
if (!journal.ops.some((op) => op.kind === "shadow-delete" && op.target === e.id)) {
|
|
1199
|
+
throw new ControlPlaneCorruptError(`transaction journal delete evidence for ${JSON.stringify(e.id)} has no same-journal shadow-delete op (uncorrelated delete claim, fail-closed): ${jp}`);
|
|
1200
|
+
}
|
|
1201
|
+
if (e.from !== undefined) {
|
|
1202
|
+
const derived = this.bindingAbsPath(e.from);
|
|
1203
|
+
if (!journal.ops.some((op) => op.kind === "delete" && op.target === derived)) {
|
|
1204
|
+
throw new ControlPlaneCorruptError(`transaction journal delete evidence for ${JSON.stringify(e.id)} has no same-journal physical delete op at its bound projection path (an account-only delete would leave the disk copy to resurrect, fail-closed): ${jp}`);
|
|
1205
|
+
}
|
|
1206
|
+
}
|
|
1207
|
+
}
|
|
1208
|
+
if (e.req !== undefined && !anchorRows.some((a) => a.req === e.req)) {
|
|
1209
|
+
chainAnchors ??= this.readTransferChain()
|
|
1210
|
+
.map((r) => r.parsed)
|
|
1211
|
+
.filter((p) => p.channel === "erasure-anchor" && p.origin === undefined);
|
|
1212
|
+
const matching = chainAnchors.filter((p) => p.req === e.req);
|
|
1213
|
+
const pinned = matching.length === 1 ? matching[0]?.ids : undefined;
|
|
1214
|
+
if (pinned === undefined || !pinned.includes(e.id)) {
|
|
1215
|
+
throw new ControlPlaneCorruptError(`transaction journal carries a request-driven delete for ${JSON.stringify(e.id)} (req ${JSON.stringify(e.req)}) with no same-journal anchor and no matching single chain anchor pinning the id — an unanchored erasure claim is refused: ${jp}`);
|
|
1216
|
+
}
|
|
1217
|
+
}
|
|
1218
|
+
}
|
|
1219
|
+
const chainAnchorEvsByReq = new Map();
|
|
1220
|
+
if (anchorRows.length > 0) {
|
|
1221
|
+
for (const p of (chainAnchors ??= this.readTransferChain()
|
|
1222
|
+
.map((r) => r.parsed)
|
|
1223
|
+
.filter((p) => p.channel === "erasure-anchor" && p.origin === undefined))) {
|
|
1224
|
+
const set = chainAnchorEvsByReq.get(p.req) ?? new Set();
|
|
1225
|
+
set.add(p.ev);
|
|
1226
|
+
chainAnchorEvsByReq.set(p.req, set);
|
|
1227
|
+
}
|
|
1228
|
+
}
|
|
1229
|
+
const anchorReqs = new Set();
|
|
1230
|
+
for (const a of anchorRows) {
|
|
1231
|
+
if (anchorReqs.has(a.req))
|
|
1232
|
+
throw new ControlPlaneCorruptError(`transaction journal carries two erasure anchors for req ${JSON.stringify(a.req)} (one request, one anchor): ${jp}`);
|
|
1233
|
+
const chainEvs = chainAnchorEvsByReq.get(a.req);
|
|
1234
|
+
if (chainEvs !== undefined && [...chainEvs].some((ev) => ev !== a.ev)) {
|
|
1235
|
+
throw new ControlPlaneCorruptError(`transaction journal mints an erasure anchor for req ${JSON.stringify(a.req)} under a DIFFERENT ev than the one already on the evidence chain (one request, one anchor — spliced journal refused): ${jp}`);
|
|
1236
|
+
}
|
|
1237
|
+
anchorReqs.add(a.req);
|
|
1238
|
+
const pinned = new Set(a.ids);
|
|
1239
|
+
const reqDeletes = deleteRows.filter((d) => d.req === a.req);
|
|
1240
|
+
const seenDeleteIds = new Set();
|
|
1241
|
+
for (const d of reqDeletes) {
|
|
1242
|
+
if (!pinned.has(d.id))
|
|
1243
|
+
throw new ControlPlaneCorruptError(`transaction journal delete evidence for ${JSON.stringify(d.id)} is outside its anchor's pinned set (req ${JSON.stringify(a.req)}): ${jp}`);
|
|
1244
|
+
if (seenDeleteIds.has(d.id))
|
|
1245
|
+
throw new ControlPlaneCorruptError(`transaction journal carries two delete rows for ${JSON.stringify(d.id)} under one request (req ${JSON.stringify(a.req)}): ${jp}`);
|
|
1246
|
+
seenDeleteIds.add(d.id);
|
|
1247
|
+
}
|
|
1248
|
+
const reqProjections = requestRows.filter((r) => r.req === a.req);
|
|
1249
|
+
const projScopesOf = new Map();
|
|
1250
|
+
const seenProjScopes = new Set();
|
|
1251
|
+
for (const r of reqProjections) {
|
|
1252
|
+
if (seenProjScopes.has(r.scope))
|
|
1253
|
+
throw new ControlPlaneCorruptError(`transaction journal carries two erasure projection rows for scope ${JSON.stringify(r.scope)} under one request (req ${JSON.stringify(a.req)}): ${jp}`);
|
|
1254
|
+
seenProjScopes.add(r.scope);
|
|
1255
|
+
if (r.selectHash !== a.selectHash)
|
|
1256
|
+
throw new ControlPlaneCorruptError(`transaction journal erasure projection row (req ${JSON.stringify(a.req)}, scope ${JSON.stringify(r.scope)}) disagrees with its anchor's selectHash: ${jp}`);
|
|
1257
|
+
for (const id of r.ids) {
|
|
1258
|
+
if (!pinned.has(id))
|
|
1259
|
+
throw new ControlPlaneCorruptError(`transaction journal erasure projection row carries ${JSON.stringify(id)} outside its anchor's pinned set (req ${JSON.stringify(a.req)}): ${jp}`);
|
|
1260
|
+
const list = projScopesOf.get(id) ?? [];
|
|
1261
|
+
list.push(r.scope);
|
|
1262
|
+
projScopesOf.set(id, list);
|
|
1263
|
+
}
|
|
1264
|
+
}
|
|
1265
|
+
const expectedNotFound = new Set();
|
|
1266
|
+
for (const id of a.ids) {
|
|
1267
|
+
const dRows = reqDeletes.filter((d) => d.id === id);
|
|
1268
|
+
let expectScope;
|
|
1269
|
+
if (dRows.length > 0) {
|
|
1270
|
+
const withFrom = dRows.find((d) => d.from !== undefined);
|
|
1271
|
+
expectScope = withFrom?.from?.scope;
|
|
1272
|
+
}
|
|
1273
|
+
else if (rowPresent(id)) {
|
|
1274
|
+
expectScope = rowsBinding(snapshotRows, id)?.scope;
|
|
1275
|
+
}
|
|
1276
|
+
else {
|
|
1277
|
+
expectedNotFound.add(id);
|
|
1278
|
+
}
|
|
1279
|
+
const cells = projScopesOf.get(id) ?? [];
|
|
1280
|
+
if (expectScope === undefined) {
|
|
1281
|
+
if (cells.length !== 0)
|
|
1282
|
+
throw new ControlPlaneCorruptError(`transaction journal erasure partition violation: ${JSON.stringify(id)} appears in a projection row but belongs to no scope cell (req ${JSON.stringify(a.req)}): ${jp}`);
|
|
1283
|
+
}
|
|
1284
|
+
else if (cells.length !== 1 || cells[0] !== expectScope) {
|
|
1285
|
+
throw new ControlPlaneCorruptError(`transaction journal erasure partition violation: ${JSON.stringify(id)} must appear exactly once in the ${JSON.stringify(expectScope)} projection row (req ${JSON.stringify(a.req)}): ${jp}`);
|
|
1286
|
+
}
|
|
1287
|
+
}
|
|
1288
|
+
const claimedNotFound = new Set(a.notFound);
|
|
1289
|
+
if (claimedNotFound.size !== expectedNotFound.size || [...expectedNotFound].some((id) => !claimedNotFound.has(id))) {
|
|
1290
|
+
throw new ControlPlaneCorruptError(`transaction journal erasure anchor's notFound set disagrees with the journal's own terminal facts (req ${JSON.stringify(a.req)}): ${jp}`);
|
|
1291
|
+
}
|
|
1292
|
+
}
|
|
1293
|
+
for (const r of requestRows) {
|
|
1294
|
+
if (!anchorRows.some((a) => a.req === r.req)) {
|
|
1295
|
+
throw new ControlPlaneCorruptError(`transaction journal erasure projection row (req ${JSON.stringify(r.req)}) has no same-journal anchor — projection rows ride the first-execution journal only: ${jp}`);
|
|
1296
|
+
}
|
|
1297
|
+
}
|
|
781
1298
|
}
|
|
782
1299
|
for (const op of journal.ops) {
|
|
783
1300
|
try {
|
|
@@ -914,6 +1431,8 @@ export class FileMemoryEngineBackend {
|
|
|
914
1431
|
const files = scanEntryFiles(dir, { exclude: isRoot ? this.excludedSubdirNames(scope) : undefined });
|
|
915
1432
|
const entries = [];
|
|
916
1433
|
const rows = sync ? this.loadLedger() : undefined;
|
|
1434
|
+
const retrievalDeleted = !sync && files.length > 0 ? this.retrievalDeletedIds() : undefined;
|
|
1435
|
+
const retrievalGuard = retrievalDeleted !== undefined && retrievalDeleted.size > 0 ? { deleted: retrievalDeleted, account: this.freshCommittedRowsPure() } : undefined;
|
|
917
1436
|
let ledgerChanged = false;
|
|
918
1437
|
const adoptedExternal = [];
|
|
919
1438
|
const moveEvents = [];
|
|
@@ -932,6 +1451,8 @@ export class FileMemoryEngineBackend {
|
|
|
932
1451
|
if (parsed.id === undefined)
|
|
933
1452
|
continue;
|
|
934
1453
|
let entry = entryFromFile(text, parsed.id, f.slug, scope);
|
|
1454
|
+
if (retrievalGuard !== undefined && retrievalGuard.deleted.has(entry.id) && rowsRev(retrievalGuard.account, entry.id) === undefined)
|
|
1455
|
+
continue;
|
|
935
1456
|
if (rows) {
|
|
936
1457
|
const committed = rowsRev(rows, entry.id);
|
|
937
1458
|
const binding = rowsBinding(rows, entry.id);
|
|
@@ -962,6 +1483,21 @@ export class FileMemoryEngineBackend {
|
|
|
962
1483
|
return undefined;
|
|
963
1484
|
if (lockToken !== undefined)
|
|
964
1485
|
this.assertTxnLockOwnership(lockToken);
|
|
1486
|
+
if (committed === undefined) {
|
|
1487
|
+
const verdict = this.resurrectionBackstopVerdict(entry.id);
|
|
1488
|
+
if (verdict === "erased") {
|
|
1489
|
+
this.containInboundReject({
|
|
1490
|
+
path: relPath,
|
|
1491
|
+
code: "invalid",
|
|
1492
|
+
reason: `resurrected entry refused: id ${entry.id} carries a delete evidence row on the store's custody chain and has no committed account row — a deleted entry's bytes reappearing on disk are never re-adopted (quarantined; restoring the content is an explicit write minting a NEW id)`,
|
|
1493
|
+
}, f.path, text, entry.id, undefined);
|
|
1494
|
+
continue;
|
|
1495
|
+
}
|
|
1496
|
+
if (verdict === "degraded") {
|
|
1497
|
+
this.announceAdoptionNotice(`chain-degraded|${relPath}|${entry.rev}`, `new memory entry file ${JSON.stringify(inlineUntrusted(relPath))} is NOT adopted: the transfer evidence chain is degraded (out-of-band tail damage), so "no delete row for this id" is no longer evidence it was never deleted — the file stays on disk unserved until the chain state is explicitly rebuilt (${CHAIN_DEGRADED_FILE})`);
|
|
1498
|
+
continue;
|
|
1499
|
+
}
|
|
1500
|
+
}
|
|
965
1501
|
let finding = this.inboundGate(relPath, text);
|
|
966
1502
|
if (finding === undefined && committed !== undefined)
|
|
967
1503
|
finding = this.whitewashInboundFinding(relPath, entry, "restored");
|
|
@@ -1603,6 +2139,7 @@ export class FileMemoryEngineBackend {
|
|
|
1603
2139
|
}
|
|
1604
2140
|
}
|
|
1605
2141
|
async applyPatchesLocked(patches, lockToken) {
|
|
2142
|
+
this.assertTxnLockOwnership(lockToken);
|
|
1606
2143
|
this.ledger = undefined;
|
|
1607
2144
|
this.recoverJournal();
|
|
1608
2145
|
this.migrateLockedIfNeeded(lockToken);
|
|
@@ -1658,7 +2195,13 @@ export class FileMemoryEngineBackend {
|
|
|
1658
2195
|
}
|
|
1659
2196
|
op.staged = staged;
|
|
1660
2197
|
}
|
|
1661
|
-
const coherentTransfers = transfers.filter((e) =>
|
|
2198
|
+
const coherentTransfers = transfers.filter((e) => {
|
|
2199
|
+
if (e.channel === "migration" || e.channel === "erasure-anchor" || e.channel === "erasure-request")
|
|
2200
|
+
return true;
|
|
2201
|
+
if (e.channel === "delete")
|
|
2202
|
+
return !Object.prototype.hasOwnProperty.call(rows, e.id);
|
|
2203
|
+
return bindingEquals(rowsBinding(rows, e.id), e.to);
|
|
2204
|
+
});
|
|
1662
2205
|
this.assertTxnLockOwnership(lockToken);
|
|
1663
2206
|
if (coherentTransfers.length > 0)
|
|
1664
2207
|
this.precheckTransfersAppendable();
|
|
@@ -1681,9 +2224,28 @@ export class FileMemoryEngineBackend {
|
|
|
1681
2224
|
this.ledger = rows;
|
|
1682
2225
|
this.saveLedger();
|
|
1683
2226
|
this.appendTransfers(coherentTransfers);
|
|
1684
|
-
|
|
2227
|
+
this.closeJournalIfOurs(txn);
|
|
1685
2228
|
return report;
|
|
1686
2229
|
}
|
|
2230
|
+
closeJournalIfOurs(txn) {
|
|
2231
|
+
const jp = join(this.controlPlaneRoot, JOURNAL_FILE);
|
|
2232
|
+
let raw;
|
|
2233
|
+
try {
|
|
2234
|
+
raw = readFileSync(jp, "utf8");
|
|
2235
|
+
}
|
|
2236
|
+
catch {
|
|
2237
|
+
return;
|
|
2238
|
+
}
|
|
2239
|
+
try {
|
|
2240
|
+
const jp = JSON.parse(raw);
|
|
2241
|
+
if (typeof jp !== "object" || jp === null || jp.txn !== txn)
|
|
2242
|
+
return;
|
|
2243
|
+
}
|
|
2244
|
+
catch {
|
|
2245
|
+
return;
|
|
2246
|
+
}
|
|
2247
|
+
rmSync(jp, { force: true });
|
|
2248
|
+
}
|
|
1687
2249
|
planOne(patch, report, ops, rows, plannedTargets, plannedDeletes, transfers) {
|
|
1688
2250
|
if (!isValidEntryId(patch.id)) {
|
|
1689
2251
|
report.conflicts.push({ op: patch.op, id: patch.id, reason: `invalid_id_shape (id ${JSON.stringify(patch.id)} fails the frontmatter id contract — the read side would drop it, leaving an applied-but-invisible entry)` });
|
|
@@ -1792,6 +2354,19 @@ export class FileMemoryEngineBackend {
|
|
|
1792
2354
|
ops.push({ kind: "delete", target: found.path });
|
|
1793
2355
|
ops.push({ kind: "shadow-delete", target: patch.id });
|
|
1794
2356
|
const deletedBinding = rowsBinding(rows, patch.id);
|
|
2357
|
+
if (deletedBinding !== undefined) {
|
|
2358
|
+
const derived = this.bindingAbsPath(deletedBinding);
|
|
2359
|
+
if (derived !== found.path)
|
|
2360
|
+
ops.push({ kind: "delete", target: derived });
|
|
2361
|
+
}
|
|
2362
|
+
transfers.push({
|
|
2363
|
+
ev: randomUUID(),
|
|
2364
|
+
channel: "delete",
|
|
2365
|
+
id: patch.id,
|
|
2366
|
+
...(deletedBinding !== undefined ? { from: deletedBinding } : {}),
|
|
2367
|
+
rev: currentRev,
|
|
2368
|
+
at: this.now(),
|
|
2369
|
+
});
|
|
1795
2370
|
deleteRow(rows, patch.id);
|
|
1796
2371
|
plannedDeletes.set(patch.id, deletedBinding);
|
|
1797
2372
|
report.applied.push({ op: "delete", id: patch.id, slug: found.entry.slug });
|
|
@@ -1825,6 +2400,490 @@ export class FileMemoryEngineBackend {
|
|
|
1825
2400
|
}
|
|
1826
2401
|
return undefined;
|
|
1827
2402
|
}
|
|
2403
|
+
async eraseWithEvidence(input) {
|
|
2404
|
+
const captured = captureErasureInput(input);
|
|
2405
|
+
const bad = erasureRequestInvalid(captured);
|
|
2406
|
+
if (bad !== undefined)
|
|
2407
|
+
throw erasureCodedError(`eraseWithEvidence: ${bad}`, "config.memory_erasure_request");
|
|
2408
|
+
const requestId = captured.requestId;
|
|
2409
|
+
const select = captured.select;
|
|
2410
|
+
const lock = await this.acquireTxnLock();
|
|
2411
|
+
try {
|
|
2412
|
+
this.assertTxnLockOwnership(lock.token);
|
|
2413
|
+
this.ledger = undefined;
|
|
2414
|
+
this.recoverJournal();
|
|
2415
|
+
this.migrateLockedIfNeeded(lock.token);
|
|
2416
|
+
const rows = cloneRows(this.loadLedger());
|
|
2417
|
+
const chain = this.readTransferChain();
|
|
2418
|
+
const degraded = this.chainDegradedMarkerPresent();
|
|
2419
|
+
const custodyState = degraded ? "damaged" : "complete";
|
|
2420
|
+
const selectHash = erasureSelectHash(select);
|
|
2421
|
+
const now = this.now();
|
|
2422
|
+
const localAnchors = chain.filter((r) => r.parsed.channel === "erasure-anchor" && r.parsed.origin === undefined && r.parsed.req === requestId);
|
|
2423
|
+
if (localAnchors.length > 1) {
|
|
2424
|
+
throw new ControlPlaneCorruptError(`erasure request ${JSON.stringify(requestId)} has ${localAnchors.length} anchors on the evidence chain — a second anchor cannot be minted by this store (the lock-held chain check runs first), so the chain was spliced out of band; refusing to pick one by file order (a human adjudicates)`);
|
|
2425
|
+
}
|
|
2426
|
+
const priorAnchor = localAnchors[0]?.parsed;
|
|
2427
|
+
if (priorAnchor !== undefined && priorAnchor.selectHash !== selectHash) {
|
|
2428
|
+
throw erasureCodedError(`eraseWithEvidence: requestId ${JSON.stringify(requestId)} was first executed with a DIFFERENT selector — a requestId names ONE request (pinned set ${JSON.stringify(priorAnchor.selectHash)}, this call ${JSON.stringify(selectHash)}); use a new requestId for a new selector`, "memory.erasure_selector_mismatch");
|
|
2429
|
+
}
|
|
2430
|
+
const replay = priorAnchor !== undefined;
|
|
2431
|
+
let census;
|
|
2432
|
+
const takeCensus = () => {
|
|
2433
|
+
if (census === undefined) {
|
|
2434
|
+
census = this.censusProjectionsLocked();
|
|
2435
|
+
if (!census.complete) {
|
|
2436
|
+
throw erasureCodedError("eraseWithEvidence: the store-wide projection census could not read every scope — an erasure attestation cannot claim a clean delete over a store that cannot be fully read; fix the filesystem fault and retry (nothing was deleted, nothing was recorded)", "memory.erasure_census_incomplete");
|
|
2437
|
+
}
|
|
2438
|
+
}
|
|
2439
|
+
return census;
|
|
2440
|
+
};
|
|
2441
|
+
const rowPresent = (id) => Object.prototype.hasOwnProperty.call(rows, id);
|
|
2442
|
+
let resolvedIds;
|
|
2443
|
+
if (replay) {
|
|
2444
|
+
resolvedIds = [...priorAnchor.ids];
|
|
2445
|
+
}
|
|
2446
|
+
else if ("ids" in select) {
|
|
2447
|
+
resolvedIds = [...select.ids];
|
|
2448
|
+
}
|
|
2449
|
+
else if ("scope" in select) {
|
|
2450
|
+
const scope = select.scope;
|
|
2451
|
+
const ledgerIds = rowsEntries(rows)
|
|
2452
|
+
.filter(([id]) => rowsBinding(rows, id)?.scope === scope)
|
|
2453
|
+
.map(([id]) => id);
|
|
2454
|
+
const ghostIds = [...takeCensus().byId.entries()]
|
|
2455
|
+
.filter(([id, projections]) => !rowPresent(id) && projections.some((p) => p.scope === scope))
|
|
2456
|
+
.map(([id]) => id);
|
|
2457
|
+
resolvedIds = [...new Set([...ledgerIds, ...ghostIds])].sort();
|
|
2458
|
+
}
|
|
2459
|
+
else {
|
|
2460
|
+
resolvedIds = lineageContributionsOfSession(this.controlPlaneRoot, select.sessionId)
|
|
2461
|
+
.map((c) => c.entryId)
|
|
2462
|
+
.sort();
|
|
2463
|
+
}
|
|
2464
|
+
const priorDeletes = new Map();
|
|
2465
|
+
for (const r of chain) {
|
|
2466
|
+
if (r.parsed.channel === "delete" && r.parsed.origin === undefined && r.parsed.req === requestId && typeof r.parsed.id === "string") {
|
|
2467
|
+
const rawFrom = r.parsed.from;
|
|
2468
|
+
const from = rawFrom !== undefined && typeof rawFrom.scope === "string" && typeof rawFrom.slug === "string" ? { scope: rawFrom.scope, slug: rawFrom.slug } : undefined;
|
|
2469
|
+
const prev = priorDeletes.get(r.parsed.id);
|
|
2470
|
+
priorDeletes.set(r.parsed.id, { ev: r.ev, at: r.parsed.at, ...(from !== undefined ? { from } : prev?.from !== undefined ? { from: prev.from } : {}) });
|
|
2471
|
+
}
|
|
2472
|
+
}
|
|
2473
|
+
const transfers = [];
|
|
2474
|
+
const ops = [];
|
|
2475
|
+
const erased = [];
|
|
2476
|
+
const erasedPreviously = [];
|
|
2477
|
+
const notFoundRows = [];
|
|
2478
|
+
const conflicts = [];
|
|
2479
|
+
const rowIds = resolvedIds.filter((id) => rowPresent(id));
|
|
2480
|
+
const accountAbsentIds = resolvedIds.filter((id) => !rowPresent(id));
|
|
2481
|
+
if (rowIds.some((id) => isValidEntryId(id)) || accountAbsentIds.length > 0)
|
|
2482
|
+
takeCensus();
|
|
2483
|
+
const censusOf = (id) => census?.byId.get(id) ?? [];
|
|
2484
|
+
const unledgeredLiveIds = accountAbsentIds.filter((id) => censusOf(id).length > 0);
|
|
2485
|
+
const trulyAbsentIds = accountAbsentIds.filter((id) => censusOf(id).length === 0);
|
|
2486
|
+
if (!replay) {
|
|
2487
|
+
transfers.push({ ev: randomUUID(), channel: "erasure-anchor", req: requestId, select: cloneErasureSelect(select), selectHash, ids: [...resolvedIds], notFound: trulyAbsentIds, at: now });
|
|
2488
|
+
const byScope = new Map();
|
|
2489
|
+
for (const id of rowIds) {
|
|
2490
|
+
if (!isValidEntryId(id))
|
|
2491
|
+
continue;
|
|
2492
|
+
const binding = rowsBinding(rows, id);
|
|
2493
|
+
if (binding === undefined)
|
|
2494
|
+
continue;
|
|
2495
|
+
const list = byScope.get(binding.scope) ?? [];
|
|
2496
|
+
list.push(id);
|
|
2497
|
+
byScope.set(binding.scope, list);
|
|
2498
|
+
}
|
|
2499
|
+
for (const scope of [...byScope.keys()].sort()) {
|
|
2500
|
+
transfers.push({ ev: randomUUID(), channel: "erasure-request", req: requestId, scope, selectHash, ids: byScope.get(scope) ?? [], at: now });
|
|
2501
|
+
}
|
|
2502
|
+
}
|
|
2503
|
+
const lineageCommitted = resolvedIds.length > 0 ? readLineageRecord(this.controlPlaneRoot).committed : {};
|
|
2504
|
+
const preSessionsOf = (id) => Object.prototype.hasOwnProperty.call(lineageCommitted, id) ? Object.keys(lineageCommitted[id] ?? {}).sort() : [];
|
|
2505
|
+
for (const id of rowIds) {
|
|
2506
|
+
const row = rows[id];
|
|
2507
|
+
if (row === undefined)
|
|
2508
|
+
continue;
|
|
2509
|
+
const preSessions = preSessionsOf(id);
|
|
2510
|
+
if (!isValidEntryId(id)) {
|
|
2511
|
+
const ev = randomUUID();
|
|
2512
|
+
transfers.push({ ev, channel: "delete", id, rev: row.rev, req: requestId, sessions: preSessions, legacyId: true, at: now });
|
|
2513
|
+
deleteRow(rows, id);
|
|
2514
|
+
erased.push({ id, rev: row.rev, binding: { state: "unbound" }, evidenceEv: ev, projectionsRemoved: 0, sessions: preSessions });
|
|
2515
|
+
continue;
|
|
2516
|
+
}
|
|
2517
|
+
try {
|
|
2518
|
+
const binding = rowsBinding(rows, id);
|
|
2519
|
+
const idOps = [{ kind: "shadow-delete", target: id }];
|
|
2520
|
+
const targets = new Set();
|
|
2521
|
+
if (binding !== undefined)
|
|
2522
|
+
targets.add(this.bindingAbsPath(binding));
|
|
2523
|
+
const censusProjections = censusOf(id);
|
|
2524
|
+
for (const p of censusProjections)
|
|
2525
|
+
targets.add(this.bindingAbsPath(p));
|
|
2526
|
+
for (const t of targets)
|
|
2527
|
+
idOps.push({ kind: "delete", target: t });
|
|
2528
|
+
const ev = randomUUID();
|
|
2529
|
+
transfers.push({ ev, channel: "delete", id, ...(binding !== undefined ? { from: binding } : {}), rev: row.rev, req: requestId, sessions: preSessions, at: now });
|
|
2530
|
+
ops.push(...idOps);
|
|
2531
|
+
deleteRow(rows, id);
|
|
2532
|
+
erased.push({
|
|
2533
|
+
id,
|
|
2534
|
+
rev: row.rev,
|
|
2535
|
+
binding: binding !== undefined ? { state: "bound", scope: binding.scope, slug: binding.slug } : { state: "unbound" },
|
|
2536
|
+
evidenceEv: ev,
|
|
2537
|
+
projectionsRemoved: censusProjections.length,
|
|
2538
|
+
sessions: preSessions,
|
|
2539
|
+
});
|
|
2540
|
+
}
|
|
2541
|
+
catch (err) {
|
|
2542
|
+
conflicts.push({ id, reason: `io error: ${err instanceof Error ? err.message : String(err)}` });
|
|
2543
|
+
}
|
|
2544
|
+
}
|
|
2545
|
+
for (const id of unledgeredLiveIds) {
|
|
2546
|
+
const preSessions = preSessionsOf(id);
|
|
2547
|
+
try {
|
|
2548
|
+
const projections = censusOf(id);
|
|
2549
|
+
let rev = "unledgered";
|
|
2550
|
+
const first = projections[0];
|
|
2551
|
+
if (first !== undefined) {
|
|
2552
|
+
const txt = readSafe(this.bindingAbsPath(first));
|
|
2553
|
+
if (txt !== undefined)
|
|
2554
|
+
rev = computeEntryRev(entryFromFile(txt, id, first.slug, first.scope));
|
|
2555
|
+
}
|
|
2556
|
+
const idOps = [{ kind: "shadow-delete", target: id }];
|
|
2557
|
+
for (const p of projections)
|
|
2558
|
+
idOps.push({ kind: "delete", target: this.bindingAbsPath(p) });
|
|
2559
|
+
const ev = randomUUID();
|
|
2560
|
+
transfers.push({ ev, channel: "delete", id, rev, req: requestId, sessions: preSessions, at: now });
|
|
2561
|
+
ops.push(...idOps);
|
|
2562
|
+
erased.push({ id, rev, binding: { state: "unbound" }, evidenceEv: ev, projectionsRemoved: projections.length, sessions: preSessions });
|
|
2563
|
+
}
|
|
2564
|
+
catch (err) {
|
|
2565
|
+
conflicts.push({ id, reason: `io error: ${err instanceof Error ? err.message : String(err)}` });
|
|
2566
|
+
}
|
|
2567
|
+
}
|
|
2568
|
+
for (const id of trulyAbsentIds) {
|
|
2569
|
+
if (erased.some((e) => e.id === id))
|
|
2570
|
+
continue;
|
|
2571
|
+
const prior = priorDeletes.get(id);
|
|
2572
|
+
if (prior !== undefined && custodyState === "complete") {
|
|
2573
|
+
const fromFree = prior.from !== undefined &&
|
|
2574
|
+
!Object.values(rows).some((row) => row.scope === prior.from.scope && row.slug === prior.from.slug) &&
|
|
2575
|
+
!existsSync(this.bindingAbsPath({ scope: prior.from.scope, slug: prior.from.slug }));
|
|
2576
|
+
erasedPreviously.push({ id, ev: prior.ev, at: prior.at, ...(fromFree ? { from: prior.from } : {}) });
|
|
2577
|
+
}
|
|
2578
|
+
else {
|
|
2579
|
+
notFoundRows.push({ id, ...(custodyState !== "complete" ? { historyUnknown: true } : {}) });
|
|
2580
|
+
}
|
|
2581
|
+
}
|
|
2582
|
+
if (ops.length > 0 || transfers.length > 0) {
|
|
2583
|
+
const txn = `${Date.now()}-erase-${randomUUID().slice(0, 8)}`;
|
|
2584
|
+
this.assertTxnLockOwnership(lock.token);
|
|
2585
|
+
this.precheckTransfersAppendable();
|
|
2586
|
+
const journal = { txn, ops, ledger2: ledgerEnvelope(rows, this.envelopeExtras), transfers };
|
|
2587
|
+
atomicWriteFileSync(join(this.controlPlaneRoot, JOURNAL_FILE), JSON.stringify(journal));
|
|
2588
|
+
for (const op of ops) {
|
|
2589
|
+
if (op.kind === "delete") {
|
|
2590
|
+
rmSync(op.target, { force: true });
|
|
2591
|
+
}
|
|
2592
|
+
else if (op.kind === "shadow-delete") {
|
|
2593
|
+
rmSync(this.shadowPath(op.target), { force: true });
|
|
2594
|
+
}
|
|
2595
|
+
}
|
|
2596
|
+
this.ledger = rows;
|
|
2597
|
+
this.saveLedger();
|
|
2598
|
+
this.appendTransfers(transfers);
|
|
2599
|
+
this.closeJournalIfOurs(txn);
|
|
2600
|
+
}
|
|
2601
|
+
const late = captureAndClearLineageForEntries(this.controlPlaneRoot, resolvedIds);
|
|
2602
|
+
for (const e of erased) {
|
|
2603
|
+
const lateSessions = late.get(e.id);
|
|
2604
|
+
if (lateSessions !== undefined && lateSessions.length > 0)
|
|
2605
|
+
e.sessions = [...new Set([...e.sessions, ...lateSessions])].sort();
|
|
2606
|
+
}
|
|
2607
|
+
const resolvedSet = new Set(resolvedIds);
|
|
2608
|
+
const quarantineHits = [];
|
|
2609
|
+
let quarantineOpaque = 0;
|
|
2610
|
+
const qDir = join(this.controlPlaneRoot, QUARANTINE_DIR);
|
|
2611
|
+
let qNames = [];
|
|
2612
|
+
try {
|
|
2613
|
+
qNames = readdirSync(qDir);
|
|
2614
|
+
}
|
|
2615
|
+
catch (err) {
|
|
2616
|
+
if (err.code !== "ENOENT") {
|
|
2617
|
+
throw new ControlPlaneCorruptError(`erasure residual enumeration failed: the quarantine dir could not be read (${err.code ?? "io error"}) at ${qDir} — the attestation must not silently claim an empty residual set; retry after the filesystem fault clears (the deletions themselves are committed and a replay converges)`);
|
|
2618
|
+
}
|
|
2619
|
+
}
|
|
2620
|
+
for (const name of qNames.sort()) {
|
|
2621
|
+
const text = readSafe(join(qDir, name));
|
|
2622
|
+
if (text === undefined) {
|
|
2623
|
+
quarantineOpaque++;
|
|
2624
|
+
continue;
|
|
2625
|
+
}
|
|
2626
|
+
const qid = parseEntryFile(text).id;
|
|
2627
|
+
if (qid === undefined) {
|
|
2628
|
+
quarantineOpaque++;
|
|
2629
|
+
continue;
|
|
2630
|
+
}
|
|
2631
|
+
if (resolvedSet.has(qid))
|
|
2632
|
+
quarantineHits.push(name);
|
|
2633
|
+
}
|
|
2634
|
+
const claimed = new Set([...erased.map((e) => e.id), ...erasedPreviously.map((e) => e.id)]);
|
|
2635
|
+
const notFoundSet = new Set(notFoundRows.map((n) => n.id));
|
|
2636
|
+
const complete = conflicts.length === 0 && resolvedIds.every((id) => claimed.has(id) || (custodyState === "complete" && notFoundSet.has(id)));
|
|
2637
|
+
return {
|
|
2638
|
+
v: 1,
|
|
2639
|
+
requestId,
|
|
2640
|
+
at: now,
|
|
2641
|
+
status: complete ? "complete" : "partial",
|
|
2642
|
+
evidenceCapability: "journal",
|
|
2643
|
+
custodyState,
|
|
2644
|
+
select: cloneErasureSelect(select),
|
|
2645
|
+
selectHash,
|
|
2646
|
+
resolvedIds,
|
|
2647
|
+
erased,
|
|
2648
|
+
notFound: notFoundRows,
|
|
2649
|
+
...(erasedPreviously.length > 0 ? { erasedPreviously } : {}),
|
|
2650
|
+
conflicts,
|
|
2651
|
+
residuals: { quarantineHits, quarantineOpaque, propagation: "local-store-only" },
|
|
2652
|
+
};
|
|
2653
|
+
}
|
|
2654
|
+
catch (err) {
|
|
2655
|
+
this.ledger = undefined;
|
|
2656
|
+
throw err;
|
|
2657
|
+
}
|
|
2658
|
+
finally {
|
|
2659
|
+
lock.release();
|
|
2660
|
+
}
|
|
2661
|
+
}
|
|
2662
|
+
static AUDIT_FENCE_ATTEMPTS = 6;
|
|
2663
|
+
static AUDIT_FENCE_RETRY_MS = 25;
|
|
2664
|
+
auditJournalPending() {
|
|
2665
|
+
return readControlFileOrAbsent(join(this.controlPlaneRoot, JOURNAL_FILE), "transaction journal") !== undefined;
|
|
2666
|
+
}
|
|
2667
|
+
auditReadLedger() {
|
|
2668
|
+
const path = join(this.controlPlaneRoot, LEDGER_FILE);
|
|
2669
|
+
const raw = readControlFileOrAbsent(path, "committed-rev ledger");
|
|
2670
|
+
if (raw === undefined)
|
|
2671
|
+
return { raw, rows: Object.create(null) };
|
|
2672
|
+
const parsed = parseLedgerText(raw, path);
|
|
2673
|
+
if (parsed.form === "v1") {
|
|
2674
|
+
throw new Error(`memory audit read refused: the committed-rev ledger at ${path} is still schema v1 (pre-binding) — ` +
|
|
2675
|
+
`migrate first (the store migrates itself at its next uncontended write/read entry or checkControlPlane); ` +
|
|
2676
|
+
`an audit read never triggers the migration`);
|
|
2677
|
+
}
|
|
2678
|
+
return { raw, rows: parsed.rows };
|
|
2679
|
+
}
|
|
2680
|
+
async auditStable(read) {
|
|
2681
|
+
for (let attempt = 0; attempt < FileMemoryEngineBackend.AUDIT_FENCE_ATTEMPTS; attempt++) {
|
|
2682
|
+
if (attempt > 0)
|
|
2683
|
+
await new Promise((r) => setTimeout(r, FileMemoryEngineBackend.AUDIT_FENCE_RETRY_MS));
|
|
2684
|
+
if (this.auditJournalPending())
|
|
2685
|
+
continue;
|
|
2686
|
+
const before = this.auditReadLedger();
|
|
2687
|
+
let out;
|
|
2688
|
+
try {
|
|
2689
|
+
out = read(before.rows, attempt === FileMemoryEngineBackend.AUDIT_FENCE_ATTEMPTS - 1);
|
|
2690
|
+
}
|
|
2691
|
+
catch (err) {
|
|
2692
|
+
if (err instanceof AuditObservationUnstableError)
|
|
2693
|
+
continue;
|
|
2694
|
+
throw err;
|
|
2695
|
+
}
|
|
2696
|
+
if (this.auditJournalPending())
|
|
2697
|
+
continue;
|
|
2698
|
+
const after = readControlFileOrAbsent(join(this.controlPlaneRoot, LEDGER_FILE), "committed-rev ledger");
|
|
2699
|
+
if (after !== before.raw)
|
|
2700
|
+
continue;
|
|
2701
|
+
return out;
|
|
2702
|
+
}
|
|
2703
|
+
throw new AuditSnapshotContendedError(`memory audit read refused after ${FileMemoryEngineBackend.AUDIT_FENCE_ATTEMPTS} attempts: a transaction journal stayed pending ` +
|
|
2704
|
+
`(or the ledger kept advancing) across every try — the committed account cannot be snapshotted without tearing. ` +
|
|
2705
|
+
`A journal that never clears is a crashed transaction: the next backend construction or locked write recovers it.`);
|
|
2706
|
+
}
|
|
2707
|
+
resolveCommittedContent(id, rev, binding, final) {
|
|
2708
|
+
if (binding === undefined)
|
|
2709
|
+
return { state: "unavailable", reason: "carrier-missing" };
|
|
2710
|
+
if (!isValidEntryId(id))
|
|
2711
|
+
return { state: "unavailable", reason: "carrier-missing" };
|
|
2712
|
+
const shadowRaw = readControlFileOrAbsent(this.shadowPath(id), "committed shadow copy");
|
|
2713
|
+
const absPath = this.bindingAbsPath(binding);
|
|
2714
|
+
if (shadowRaw !== undefined) {
|
|
2715
|
+
const shadowEntry = entryFromFile(shadowRaw, id, binding.slug, binding.scope);
|
|
2716
|
+
if (shadowEntry.rev !== rev) {
|
|
2717
|
+
let projText;
|
|
2718
|
+
try {
|
|
2719
|
+
projText = readFileSync(absPath, "utf8");
|
|
2720
|
+
}
|
|
2721
|
+
catch {
|
|
2722
|
+
projText = undefined;
|
|
2723
|
+
}
|
|
2724
|
+
if (projText !== undefined && parseEntryFile(projText).id === id) {
|
|
2725
|
+
const projEntry = entryFromFile(projText, id, binding.slug, binding.scope);
|
|
2726
|
+
if (projEntry.rev === rev)
|
|
2727
|
+
return { state: "present", entry: projEntry };
|
|
2728
|
+
}
|
|
2729
|
+
if (!final)
|
|
2730
|
+
throw new AuditObservationUnstableError(`shadow bytes for ${id} do not carry the committed rev — possible adoption in flight`);
|
|
2731
|
+
if (this.txnInFlight()) {
|
|
2732
|
+
throw new AuditSnapshotContendedError(`memory audit read refused: shadow bytes for ${id} do not carry the committed rev while a live writer holds the txn lock — an adoption is in flight; retry when the store settles.`);
|
|
2733
|
+
}
|
|
2734
|
+
return { state: "unavailable", reason: "carrier-missing" };
|
|
2735
|
+
}
|
|
2736
|
+
return probeProjection(absPath, id) === "present" ? { state: "present", entry: shadowEntry } : { state: "unavailable", reason: "carrier-missing" };
|
|
2737
|
+
}
|
|
2738
|
+
let text;
|
|
2739
|
+
try {
|
|
2740
|
+
text = readFileSync(absPath, "utf8");
|
|
2741
|
+
}
|
|
2742
|
+
catch {
|
|
2743
|
+
text = undefined;
|
|
2744
|
+
}
|
|
2745
|
+
if (text !== undefined && parseEntryFile(text).id === id) {
|
|
2746
|
+
const diskEntry = entryFromFile(text, id, binding.slug, binding.scope);
|
|
2747
|
+
if (diskEntry.rev === rev)
|
|
2748
|
+
return { state: "present", entry: diskEntry };
|
|
2749
|
+
}
|
|
2750
|
+
return { state: "unavailable", reason: "shadowless-legacy" };
|
|
2751
|
+
}
|
|
2752
|
+
async committedSnapshotOf(id) {
|
|
2753
|
+
return this.auditStable((rows, final) => {
|
|
2754
|
+
const row = Object.prototype.hasOwnProperty.call(rows, id) ? rows[id] : undefined;
|
|
2755
|
+
if (row === undefined)
|
|
2756
|
+
return { state: "absent", id };
|
|
2757
|
+
const binding = rowsBinding(rows, id);
|
|
2758
|
+
return {
|
|
2759
|
+
state: "row",
|
|
2760
|
+
id,
|
|
2761
|
+
rev: row.rev,
|
|
2762
|
+
binding: this.publicBinding(row, binding),
|
|
2763
|
+
content: this.resolveCommittedContent(id, row.rev, binding, final),
|
|
2764
|
+
};
|
|
2765
|
+
});
|
|
2766
|
+
}
|
|
2767
|
+
publicBinding(row, binding) {
|
|
2768
|
+
if (binding === undefined)
|
|
2769
|
+
return { state: "unbound" };
|
|
2770
|
+
return {
|
|
2771
|
+
state: "bound",
|
|
2772
|
+
scope: binding.scope,
|
|
2773
|
+
slug: binding.slug,
|
|
2774
|
+
...(typeof row.at === "number" ? { at: row.at } : {}),
|
|
2775
|
+
...(row.prev !== undefined ? { prev: { ...row.prev } } : {}),
|
|
2776
|
+
};
|
|
2777
|
+
}
|
|
2778
|
+
async committedSnapshotsOfScopes(scopes) {
|
|
2779
|
+
const want = new Set(scopes);
|
|
2780
|
+
try {
|
|
2781
|
+
return await this.auditStable((rows, final) => {
|
|
2782
|
+
const out = { complete: true, rows: [], unbound: [] };
|
|
2783
|
+
for (const [id, row] of rowsEntries(rows)) {
|
|
2784
|
+
const binding = rowsBinding(rows, id);
|
|
2785
|
+
if (binding === undefined) {
|
|
2786
|
+
out.unbound.push(id);
|
|
2787
|
+
continue;
|
|
2788
|
+
}
|
|
2789
|
+
if (!want.has(binding.scope))
|
|
2790
|
+
continue;
|
|
2791
|
+
out.rows.push({
|
|
2792
|
+
state: "row",
|
|
2793
|
+
id,
|
|
2794
|
+
rev: row.rev,
|
|
2795
|
+
binding: this.publicBinding(row, binding),
|
|
2796
|
+
content: this.resolveCommittedContent(id, row.rev, binding, final),
|
|
2797
|
+
});
|
|
2798
|
+
}
|
|
2799
|
+
return out;
|
|
2800
|
+
});
|
|
2801
|
+
}
|
|
2802
|
+
catch (err) {
|
|
2803
|
+
if (err instanceof AuditSnapshotContendedError)
|
|
2804
|
+
return { complete: false };
|
|
2805
|
+
throw err;
|
|
2806
|
+
}
|
|
2807
|
+
}
|
|
2808
|
+
async custodyOf(id) {
|
|
2809
|
+
const transfersPath = join(this.controlPlaneRoot, TRANSFERS_FILE);
|
|
2810
|
+
for (let attempt = 0; attempt < FileMemoryEngineBackend.AUDIT_FENCE_ATTEMPTS; attempt++) {
|
|
2811
|
+
if (attempt > 0)
|
|
2812
|
+
await new Promise((r) => setTimeout(r, FileMemoryEngineBackend.AUDIT_FENCE_RETRY_MS));
|
|
2813
|
+
if (this.auditJournalPending())
|
|
2814
|
+
continue;
|
|
2815
|
+
const rawBefore = readControlFileOrAbsent(transfersPath, "transfer evidence log");
|
|
2816
|
+
const out = this.custodyReadPure(id, rawBefore);
|
|
2817
|
+
if (out.state === "complete" && out.events.length === 0) {
|
|
2818
|
+
const ledgerRaw = readControlFileOrAbsent(join(this.controlPlaneRoot, LEDGER_FILE), "committed-rev ledger");
|
|
2819
|
+
if (ledgerRaw !== undefined) {
|
|
2820
|
+
const parsed = parseLedgerText(ledgerRaw, join(this.controlPlaneRoot, LEDGER_FILE));
|
|
2821
|
+
if (parsed.form === "v2" && Object.prototype.hasOwnProperty.call(parsed.rows, id) && parsed.rows[id]?.prev !== undefined) {
|
|
2822
|
+
out.state = "damaged";
|
|
2823
|
+
}
|
|
2824
|
+
}
|
|
2825
|
+
}
|
|
2826
|
+
if (this.auditJournalPending())
|
|
2827
|
+
continue;
|
|
2828
|
+
if (readControlFileOrAbsent(transfersPath, "transfer evidence log") !== rawBefore)
|
|
2829
|
+
continue;
|
|
2830
|
+
return out;
|
|
2831
|
+
}
|
|
2832
|
+
throw new AuditSnapshotContendedError(`memory custody read refused after ${FileMemoryEngineBackend.AUDIT_FENCE_ATTEMPTS} attempts: a transaction journal stayed pending across ` +
|
|
2833
|
+
`every try — the evidence chain may hold a committed transaction's rows not yet appended. A journal that never ` +
|
|
2834
|
+
`clears is a crashed transaction: the next backend construction or locked write recovers it.`);
|
|
2835
|
+
}
|
|
2836
|
+
custodyReadPure(id, raw) {
|
|
2837
|
+
const path = join(this.controlPlaneRoot, TRANSFERS_FILE);
|
|
2838
|
+
const out = { state: "complete", events: [] };
|
|
2839
|
+
if (raw === undefined)
|
|
2840
|
+
return out;
|
|
2841
|
+
const byEv = new Map();
|
|
2842
|
+
const lines = raw.split("\n");
|
|
2843
|
+
for (let i = 0; i < lines.length; i++) {
|
|
2844
|
+
const line = lines[i];
|
|
2845
|
+
if (line === undefined || line === "")
|
|
2846
|
+
continue;
|
|
2847
|
+
let parsed;
|
|
2848
|
+
try {
|
|
2849
|
+
parsed = JSON.parse(line);
|
|
2850
|
+
}
|
|
2851
|
+
catch {
|
|
2852
|
+
if (lines.slice(i + 1).every((l) => l === "")) {
|
|
2853
|
+
out.state = "damaged";
|
|
2854
|
+
break;
|
|
2855
|
+
}
|
|
2856
|
+
throw new ControlPlaneCorruptError(`transfer evidence log line ${i + 1} is unparseable (not a torn tail — fail-closed): ${path}`);
|
|
2857
|
+
}
|
|
2858
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
2859
|
+
throw new ControlPlaneCorruptError(`transfer evidence log line ${i + 1} is not an object (fail-closed): ${path}`);
|
|
2860
|
+
}
|
|
2861
|
+
const e = parsed;
|
|
2862
|
+
if (typeof e.ev !== "string" || e.ev.length === 0 || typeof e.channel !== "string" || e.channel.length === 0 || typeof e.at !== "number" || !Number.isInteger(e.at)) {
|
|
2863
|
+
throw new ControlPlaneCorruptError(`transfer evidence log line ${i + 1} is missing its base keys (ev/channel/at) — fail-closed: ${path}`);
|
|
2864
|
+
}
|
|
2865
|
+
if (KNOWN_TRANSFER_CHANNELS.has(e.channel)) {
|
|
2866
|
+
const invalid = transferEventInvalid(parsed);
|
|
2867
|
+
if (invalid !== undefined)
|
|
2868
|
+
throw new ControlPlaneCorruptError(`transfer evidence log line ${i + 1} is invalid (${invalid}): ${path}`);
|
|
2869
|
+
}
|
|
2870
|
+
const canonical = canonicalJsonStringify(e);
|
|
2871
|
+
const seen = byEv.get(e.ev);
|
|
2872
|
+
if (seen !== undefined) {
|
|
2873
|
+
if (seen.has(canonical))
|
|
2874
|
+
continue;
|
|
2875
|
+
out.state = "damaged";
|
|
2876
|
+
out.reason ??= `evidence chain carries event id ${JSON.stringify(e.ev)} twice with DIFFERENT payloads (line ${i + 1}) — one ev is one identity; the chain was spliced or corrupted out of band and this store's own write path refuses to operate on it`;
|
|
2877
|
+
seen.add(canonical);
|
|
2878
|
+
}
|
|
2879
|
+
else {
|
|
2880
|
+
byEv.set(e.ev, new Set([canonical]));
|
|
2881
|
+
}
|
|
2882
|
+
if (e.id === id)
|
|
2883
|
+
out.events.push(e);
|
|
2884
|
+
}
|
|
2885
|
+
return out;
|
|
2886
|
+
}
|
|
1828
2887
|
async getConsolidationCursor(scope) {
|
|
1829
2888
|
const cursors = this.readCursors();
|
|
1830
2889
|
return cursors[scope];
|
|
@@ -1838,12 +2897,16 @@ export class FileMemoryEngineBackend {
|
|
|
1838
2897
|
try {
|
|
1839
2898
|
const parsed = JSON.parse(readFileSync(join(this.controlPlaneRoot, CURSORS_FILE), "utf8"));
|
|
1840
2899
|
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
|
|
1841
|
-
|
|
2900
|
+
const out = Object.create(null);
|
|
2901
|
+
for (const [k, v] of Object.entries(parsed))
|
|
2902
|
+
if (typeof v === "string")
|
|
2903
|
+
out[k] = v;
|
|
2904
|
+
return out;
|
|
1842
2905
|
}
|
|
1843
2906
|
}
|
|
1844
2907
|
catch {
|
|
1845
2908
|
}
|
|
1846
|
-
return
|
|
2909
|
+
return Object.create(null);
|
|
1847
2910
|
}
|
|
1848
2911
|
}
|
|
1849
2912
|
function headerOf(e, path) {
|