dsh-compaction-weave 0.0.0-stage → 0.3.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/CHANGELOG.md +103 -0
  2. package/LICENSE +21 -0
  3. package/README.en.md +145 -0
  4. package/README.md +148 -2
  5. package/cordis.patch.yml +19 -0
  6. package/docs/architecture.md +84 -0
  7. package/docs/benchmarks/published-0.3.2-rc.1-tests.json +576 -0
  8. package/docs/benchmarks/rc2-documentation.json +41 -0
  9. package/docs/benchmarks/rc2-linux-tests.json +590 -0
  10. package/docs/benchmarks/rc2-live-and-cost.json +746 -0
  11. package/docs/benchmarks/rc3-host-compatibility.json +2314 -0
  12. package/docs/benchmarks/rc4-tests.json +617 -0
  13. package/docs/benchmarks/review35-compression.json +117 -0
  14. package/docs/benchmarks/v0.3.2-live-four-way.json +3495 -0
  15. package/docs/benchmarks/v0.3.2-tests.json +590 -0
  16. package/docs/configuration.md +131 -0
  17. package/docs/development.md +95 -0
  18. package/docs/en/architecture.md +75 -0
  19. package/docs/en/configuration.md +131 -0
  20. package/docs/en/development.md +92 -0
  21. package/docs/en/feedback.md +47 -0
  22. package/docs/en/principles.md +235 -0
  23. package/docs/en/testing.md +368 -0
  24. package/docs/en/troubleshooting.md +60 -0
  25. package/docs/feedback.md +47 -0
  26. package/docs/images/principles-context.en.svg +77 -0
  27. package/docs/images/principles-context.zh.svg +77 -0
  28. package/docs/images/principles-pipeline.en.svg +93 -0
  29. package/docs/images/principles-pipeline.zh.svg +93 -0
  30. package/docs/images/principles-recovery.en.svg +46 -0
  31. package/docs/images/principles-recovery.zh.svg +46 -0
  32. package/docs/principles.md +235 -0
  33. package/docs/test-results.json +634 -0
  34. package/docs/testing.md +358 -0
  35. package/docs/troubleshooting.md +60 -0
  36. package/lib/abort.js +13 -0
  37. package/lib/action-state.js +60 -0
  38. package/lib/atomic-replace.js +18 -0
  39. package/lib/attachments.js +76 -0
  40. package/lib/audit-reject.js +161 -0
  41. package/lib/audit.js +939 -0
  42. package/lib/cache-policy.js +71 -0
  43. package/lib/client.js +696 -0
  44. package/lib/command-compact.js +38 -0
  45. package/lib/command-heartbeat.js +61 -0
  46. package/lib/config.js +596 -0
  47. package/lib/constraint-scope.js +37 -0
  48. package/lib/continuation-budget.js +69 -0
  49. package/lib/continuation-pending.js +57 -0
  50. package/lib/continuation-progress.js +46 -0
  51. package/lib/continuation.js +287 -0
  52. package/lib/engine-entry.js +103 -0
  53. package/lib/engine.js +2485 -0
  54. package/lib/failure-diagnostics.js +220 -0
  55. package/lib/history-source.js +74 -0
  56. package/lib/history.js +712 -0
  57. package/lib/idle-compaction.js +64 -0
  58. package/lib/index.js +611 -0
  59. package/lib/instruction-source.js +163 -0
  60. package/lib/journal.js +164 -0
  61. package/lib/manual-error.js +179 -0
  62. package/lib/memory-policy.js +383 -0
  63. package/lib/meter.js +387 -0
  64. package/lib/observation-mask.js +390 -0
  65. package/lib/proactive.js +108 -0
  66. package/lib/progress-facts.js +347 -0
  67. package/lib/progress-state.js +30 -0
  68. package/lib/prompt.js +194 -0
  69. package/lib/reconstruct.js +72 -0
  70. package/lib/reference-input.js +107 -0
  71. package/lib/region.js +972 -0
  72. package/lib/source-provenance.js +58 -0
  73. package/lib/source-recovery.js +22 -0
  74. package/lib/source-refs.js +137 -0
  75. package/lib/state.js +46 -0
  76. package/lib/storage-path.js +42 -0
  77. package/lib/summary-budget.js +187 -0
  78. package/lib/summary-config.js +308 -0
  79. package/lib/summary-progress.js +56 -0
  80. package/lib/summary-retry.js +27 -0
  81. package/lib/summary-timeout.js +68 -0
  82. package/lib/summary.js +476 -0
  83. package/lib/tools.js +341 -0
  84. package/lib/transaction-host.js +252 -0
  85. package/lib/transaction.js +285 -0
  86. package/lib/validate.js +503 -0
  87. package/lib/version.js +3 -0
  88. package/lib/window.js +120 -0
  89. package/lib/work-report.js +55 -0
  90. package/locale/en.json +6 -0
  91. package/locale/zh.json +6 -0
  92. package/package.json +71 -4
@@ -0,0 +1,285 @@
1
+ // Stock-host transaction adapter inspired by reply-forge's instance-scoped
2
+ // persistence adaptation. No host file is changed and no custom Session event
3
+ // is emitted. Private JS seams are fingerprint-gated; unknown hosts refuse.
4
+ import * as fs from 'node:fs';
5
+ import { createHash, randomUUID } from 'node:crypto';
6
+ import { dirname, resolve } from 'node:path';
7
+ import { HOST_FINGERPRINTS, HOST_NORMALIZED_FINGERPRINTS, normalizeHostMethodSource } from './transaction-host.js';
8
+ import { CompactionJournal } from './journal.js';
9
+ import { SessionStateStore } from './state.js';
10
+
11
+ const digest = bytes => createHash('sha256').update(bytes).digest('hex');
12
+ const SAME = (a, b) => JSON.stringify(a) === JSON.stringify(b);
13
+ const installed = new WeakMap();
14
+ function refuse(message) { const e = new Error(`Weave transaction adapter: ${message}`); e.code = 'RECOVERY_REQUIRED'; return e; }
15
+ function prototypeMethod(object, name) { return Object.getPrototypeOf(object)?.[name]; }
16
+ function checkMethods(object, group) {
17
+ for (const [name, expected] of Object.entries(HOST_FINGERPRINTS[group])) {
18
+ const fn = prototypeMethod(object, name);
19
+ if (typeof fn !== 'function') throw refuse(`unsupported ${group}.${name}; host compatibility must be reviewed before compaction`);
20
+ const src = Function.prototype.toString.call(fn);
21
+ const allowedRaw = Array.isArray(expected) ? expected : [expected];
22
+ if (allowedRaw.includes(digest(src))) continue;
23
+ const normExpected = HOST_NORMALIZED_FINGERPRINTS[group]?.[name];
24
+ const allowedNorm = normExpected ? (Array.isArray(normExpected) ? normExpected : [normExpected]) : [];
25
+ if (allowedNorm.includes(digest(normalizeHostMethodSource(src)))) continue;
26
+ throw refuse(`unsupported ${group}.${name}; host compatibility must be reviewed before compaction`);
27
+ }
28
+ }
29
+ function fileDigest(file, limit = Infinity) {
30
+ const fd = fs.openSync(file, 'r'), hash = createHash('sha256'), chunk = Buffer.alloc(1024 * 1024);
31
+ let position = 0;
32
+ try {
33
+ while (position < limit) {
34
+ const size = fs.readSync(fd, chunk, 0, Math.min(chunk.length, limit - position), position);
35
+ if (!size) break;
36
+ hash.update(chunk.subarray(0, size)); position += size;
37
+ }
38
+ } finally { fs.closeSync(fd); }
39
+ return { bytes: position, sha256: hash.digest('hex') };
40
+ }
41
+ function writeNew(file, bytes) {
42
+ const fd = fs.openSync(file, 'wx', 0o600);
43
+ try { fs.writeFileSync(fd, bytes); fs.fsyncSync(fd); } finally { fs.closeSync(fd); }
44
+ }
45
+ function copySynced(source, destination) {
46
+ const input = fs.openSync(source, 'r'), buffer = Buffer.alloc(1024 * 1024), hash = createHash('sha256');
47
+ let output, position = 0;
48
+ try {
49
+ output = fs.openSync(destination, 'wx', 0o600);
50
+ for (;;) {
51
+ const size = fs.readSync(input, buffer, 0, buffer.length, position);
52
+ if (!size) break;
53
+ hash.update(buffer.subarray(0, size));
54
+ let written = 0;
55
+ while (written < size) written += fs.writeSync(output, buffer, written, size - written, position + written);
56
+ position += size;
57
+ }
58
+ fs.fsyncSync(output);
59
+ } finally { if (output !== undefined) fs.closeSync(output); fs.closeSync(input); }
60
+ return { bytes: position, sha256: hash.digest('hex') };
61
+ }
62
+ function atomicRestore(target, backup) {
63
+ const temp = `${target}.weave-restore-${randomUUID()}.tmp`;
64
+ try { copySynced(backup, temp); fs.renameSync(temp, target); }
65
+ finally { try { fs.unlinkSync(temp); } catch {} }
66
+ }
67
+ function verifyFiles(target, record, journal) {
68
+ const file = journal.fileFor(record.sessionId, record.compactionId), backup = file + '.rollback', tailFile = file + '.tail', meta = record.atomic;
69
+ if (meta?.version !== 1 || !Number.isSafeInteger(meta.beforeBytes) || meta.beforeBytes < 1 || !Number.isSafeInteger(meta.beforeEvents) || meta.beforeEvents < 0 || !Number.isSafeInteger(meta.tailBytes) || meta.tailBytes < 1 || !/^[a-f0-9]{64}$/.test(meta.beforeHash ?? '') || !/^[a-f0-9]{64}$/.test(meta.tailHash ?? '')) throw refuse('invalid recovery manifest; preserving all files');
70
+ const actual = fileDigest(backup), tail = fs.readFileSync(tailFile);
71
+ if (actual.sha256 !== meta.beforeHash || actual.bytes !== meta.beforeBytes || digest(tail) !== meta.tailHash || tail.length !== meta.tailBytes) throw refuse('backup/tail integrity check failed');
72
+ const size = fs.statSync(target).size;
73
+ if (fileDigest(target, meta.beforeBytes).sha256 !== meta.beforeHash || size < meta.beforeBytes) throw refuse('stored prefix diverged; refusing to overwrite unrelated history');
74
+ if (meta.restored === true) return { backup, tail, meta, size };
75
+ if (meta.restored !== undefined) throw refuse('invalid restored marker');
76
+ if (record.phase !== 'committed' && size > meta.beforeBytes + tail.length) throw refuse('newer history exists after the transaction; refusing to discard it');
77
+ const trailing = Math.min(tail.length, size - meta.beforeBytes), fd = fs.openSync(target, 'r'), check = Buffer.alloc(trailing);
78
+ try { let read = 0; while (read < check.length) { const n = fs.readSync(fd, check, read, check.length-read, meta.beforeBytes+read); if (!n) throw refuse('unexpected EOF'); read += n; } }
79
+ finally { fs.closeSync(fd); }
80
+ if (!check.equals(tail.subarray(0, trailing))) throw refuse('stored tail diverged; preserving unrelated changes');
81
+ if (record.phase === 'committed' && trailing !== tail.length) throw refuse('durable success decision has an incomplete event tail');
82
+ return { backup, tail, meta, size };
83
+ }
84
+
85
+ export class StockTransactionAdapter {
86
+ constructor(backend, sessions, logger) {
87
+ checkMethods(backend, 'backend');
88
+ if (!(backend.tracker?.writers instanceof Map)) throw refuse('JSONL writer registry unavailable');
89
+ this.backend = backend; this.sessions = sessions; this.logger = logger;
90
+ this.roots = new Map(); this.blocked = new Set(); this.openOriginal = backend.open;
91
+ this.openDescriptor = Object.getOwnPropertyDescriptor(backend, 'open');
92
+ const adapter = this;
93
+ this.openWrapper = async function(id, access, options) {
94
+ if (access === 'write') await adapter.recoverBeforeOpen(id, options?.signal);
95
+ return adapter.openOriginal.call(this, id, access, options);
96
+ };
97
+ backend.open = this.openWrapper;
98
+ }
99
+ register(journal, state, failureThreshold) {
100
+ this.roots.set(resolve(journal.root), { journal, state, failureThreshold });
101
+ }
102
+ assertSupported(session) {
103
+ checkMethods(session, 'session'); checkMethods(session.surface, 'surface');
104
+ if (this.sessions.get(session.id) !== session) throw refuse('Session is not the current live owner');
105
+ if (session.header.version !== 4 || typeof session.constructor.create !== 'function') throw refuse('unsupported Session format');
106
+ const h = this.backend.tracker.writers.get(session.id);
107
+ if (!h || h.id !== session.id || h.access !== 'write' || h.closing || !h.lease || !h.state?.materialized) throw refuse('live durable writer/lease unavailable');
108
+ checkMethods(h, 'handle');
109
+ if (session.commit !== prototypeMethod(session, 'commit') || session.publicationEntry !== prototypeMethod(session, 'publicationEntry')) throw refuse('Session publication is wrapped by another extension');
110
+ if (this.blocked.has(session.id)) throw refuse('writer is quarantined until cold recovery');
111
+ return h;
112
+ }
113
+ probeSupport(session) {
114
+ if (this.blocked.has(session?.id)) throw refuse('writer is quarantined until cold recovery');
115
+ try {
116
+ return { ok: true, handle: this.assertSupported(session) };
117
+ } catch (error) {
118
+ if (error?.code === 'RECOVERY_REQUIRED') return { ok: false, reason: error.message, error };
119
+ throw error;
120
+ }
121
+ }
122
+ stage(session, build) {
123
+ this.assertSupported(session);
124
+ // Replaying via native commit avoids Session.create(seed)'s end-seed marker.
125
+ const shadow = session.constructor.create(session.id);
126
+ shadow.header = session.header; shadow.inheritedEventCount = session.inheritedEventCount;
127
+ shadow.surfaceManager.projections = session.surface.projections;
128
+ for (const event of session.snapshotEvents()) shadow.commit(event, undefined);
129
+ const before = session.seq, result = build(shadow), events = shadow.snapshotEvents(before);
130
+ if (events.length !== 2 || events[0].type !== 'user/message' || events[0].data.source?.kind !== 'compact-checkpoint' || events[1].type !== 'compaction/end') throw refuse('unexpected staged event batch');
131
+ return { before, events, result, nodes: [...shadow.surface.nodes] };
132
+ }
133
+ // All fallible asynchronous work finishes before entering handle.run's short
134
+ // synchronous section. No live Session event can interleave inside that block.
135
+ async commit(engine, session, record, staged, signal) {
136
+ const h = this.assertSupported(session), journal = engine.journal;
137
+ const encoded = Buffer.from(await this.backend.encodeEventBatch(staged.events));
138
+ await engine._flushSession(session);
139
+ signal?.throwIfAborted();
140
+ return h.run('weave-commit', () => {
141
+ signal?.throwIfAborted(); this.assertSupported(session);
142
+ if (session.seq !== staged.before || h.state.cursor !== staged.before || h.buffered.length || h.state.tornTruncateTo !== undefined || h.state.recoveredTail?.length) throw refuse('session changed during commit preparation; retry after it is idle');
143
+ const location = this.backend.locate(session.header);
144
+ if (location?.kind !== 'jsonl') throw refuse('unsupported persistence location');
145
+ const target = location.path, file = journal.fileFor(session.id, record.compactionId), backup = file+'.rollback', tailFile = file+'.tail';
146
+ fs.mkdirSync(dirname(file), { recursive: true });
147
+ // A transaction id is unique. Existing backup material is never overwritten.
148
+ const statBefore = fs.statSync(target);
149
+ const copied = copySynced(target, backup); writeNew(tailFile, encoded);
150
+ const statAfter = fs.statSync(target), backupStat = fs.statSync(backup);
151
+ const unchanged = statBefore.size === copied.bytes && statAfter.size === copied.bytes && statBefore.mtimeMs === statAfter.mtimeMs && backupStat.size === copied.bytes;
152
+ const before = unchanged ? copied : fileDigest(target);
153
+ if (!unchanged && !SAME(before, fileDigest(backup))) throw refuse('backup differs from the held writer file');
154
+ const atomic = { version: 1, beforeBytes: before.bytes, beforeHash: before.sha256, beforeEvents: staged.before, tailBytes: encoded.length, tailHash: digest(encoded) };
155
+ let current = { ...record, sessionId: session.id, atomic, phase: 'committing' }, decided = false;
156
+ journal.recordSync(current);
157
+ try {
158
+ this.checkpoint?.('before-append', { session, record: current });
159
+ const fd = fs.openSync(target, 'a');
160
+ try { fs.writeFileSync(fd, encoded); fs.fsyncSync(fd); } finally { fs.closeSync(fd); }
161
+ this.checkpoint?.('after-append', { session, record: current });
162
+ engine.sessionState.update(session.id, state => ({ ...state, failures: 0, stopped: false }));
163
+ this.checkpoint?.('after-state', { session, record: current });
164
+ signal?.throwIfAborted();
165
+ journal.recordSync({ ...current, phase: 'committed' });
166
+ decided = true;
167
+ this.checkpoint?.('after-decision', { session, record: current });
168
+ h.state.cursor += staged.events.length; h.state.primed = undefined; h.observedLength = h.state.cursor;
169
+ this.backend.coldLogMemo?.delete(session.id);
170
+ // Publish exactly the already-durable native events. Suppress only their
171
+ // duplicate routing; other synchronous listener events retain normal routing.
172
+ const originalEnqueue = h.enqueueLive, expected = new Map(staged.events.map(e => [e.seq, e]));
173
+ h.enqueueLive = function(event, ...args) {
174
+ if (expected.has(event.seq)) {
175
+ if (!SAME(event, expected.get(event.seq))) throw refuse('publication differs from durable transaction');
176
+ expected.delete(event.seq); return;
177
+ }
178
+ return originalEnqueue.call(this, event, ...args);
179
+ };
180
+ try {
181
+ for (const event of staged.events) session.commit(event, session.publicationEntry());
182
+ if (expected.size) throw refuse('durable events did not pass the live persistence router');
183
+ if (!SAME(session.surface.nodes, staged.nodes)) throw refuse('published surface differs from staged surface');
184
+ } finally { h.enqueueLive = originalEnqueue; }
185
+ this.checkpoint?.('after-publish', { session, record: current });
186
+ return staged.result;
187
+ } catch (error) {
188
+ if (decided) {
189
+ this.quarantine(session.id, h);
190
+ throw refuse(`commit is durable but publication failed; reopen the session: ${error.message}`);
191
+ }
192
+ try {
193
+ verifyFiles(target, current, journal); atomicRestore(target, backup);
194
+ if (!SAME(fileDigest(target), before)) throw refuse('restored backup differs');
195
+ this.backend.coldLogMemo?.delete(session.id);
196
+ const reason = signal?.aborted ? 'cancelled' : 'failed';
197
+ current = { ...current, failureKind: reason, atomic: { ...atomic, restored: true } };
198
+ journal.recordSync(current);
199
+ engine.sessionState.update(session.id, state => {
200
+ const failures = Math.max(state.failures, record.failuresBefore ?? 0);
201
+ return { ...state, failures, stopped: failures >= engine.config.failureThreshold };
202
+ });
203
+ error.atomicRolledBack = true;
204
+ } catch (rollbackError) {
205
+ this.quarantine(session.id, h);
206
+ throw refuse(`backup restoration failed; journal retained: ${rollbackError.message}`);
207
+ }
208
+ throw error;
209
+ }
210
+ });
211
+ }
212
+ quarantine(id, handle) {
213
+ this.blocked.add(id);
214
+ // Prevent buffered closers/inbox events being appended behind an uncertain tail.
215
+ handle.persistContiguous = async () => { throw refuse('writer quarantined; restart for backup recovery'); };
216
+ }
217
+ async recoverBeforeOpen(id, signal) {
218
+ const records = [];
219
+ for (const entry of this.roots.values()) for (const record of await entry.journal.pending(id)) {
220
+ if (record.corrupt) throw refuse('corrupt recovery journal');
221
+ if (record.atomic) records.push({ ...entry, record: { ...record, sessionId: id } });
222
+ }
223
+ if (!records.length) return;
224
+ if (this.sessions.get(id) || this.backend.tracker.writers.has(id)) throw refuse('refusing disk recovery while a session/writer is live');
225
+ const backend = this.backend; backend.tracker.claimWrite(id);
226
+ let lease;
227
+ try {
228
+ const selected = await backend.findLog(id, signal);
229
+ if (!selected) throw refuse('session log is missing');
230
+ lease = await backend.acquireLease(id, undefined, dirname(selected.currentPath));
231
+ // The lock remains held across every check and replacement. Original open
232
+ // acquires it again afterwards; a racing writer then causes normal refusal.
233
+ for (const {journal, state, failureThreshold, record} of records) {
234
+ signal?.throwIfAborted();
235
+ const verified = verifyFiles(selected.currentPath, record, journal);
236
+ if (record.phase === 'committed') { await journal.clear(id, record.compactionId); continue; }
237
+ if (verified.meta.restored !== true) {
238
+ atomicRestore(selected.currentPath, verified.backup);
239
+ if (fileDigest(selected.currentPath).sha256 !== verified.meta.beforeHash) throw refuse('cold recovery verification failed');
240
+ // Persist this before the host can append end-seed/turn closers or
241
+ // newly queued input. A second crash must not rewind those events.
242
+ record.atomic = { ...record.atomic, restored: true };
243
+ journal.recordSync(record);
244
+ }
245
+ backend.coldLogMemo?.delete(id);
246
+ // Close the restored lifecycle BEFORE Session's constructor emits an
247
+ // end-seed marker. Closing it afterwards makes the next disk replay fail.
248
+ const restoredLog = await backend.readStoredLog(selected.currentPath, id, signal);
249
+ const start = restoredLog.events.find(e => e.type === 'compaction/start' && e.data.compactionId === record.compactionId);
250
+ const end = restoredLog.events.find(e => e.type === 'compaction/end' && e.data.compactionId === record.compactionId);
251
+ if (!start) throw refuse('restored log has no matching compaction start');
252
+ if (!end) {
253
+ if (restoredLog.events.some(e => e.type === 'session/end-seed' && e.seq > start.seq)) throw refuse('restored lifecycle crosses an end-seed marker');
254
+ const closer = { type: 'compaction/end', seq: restoredLog.events.length, time: Date.now(), data: { ...start.data, error: 'Weave restored the failed transaction backup before loading this session.' } };
255
+ await backend.persistBatch(restoredLog.meta, [closer], true, restoredLog.inheritedEventCount);
256
+ backend.coldLogMemo?.delete(id);
257
+ await backend.readStoredLog(selected.currentPath, id, signal);
258
+ }
259
+ const reason = record.failureKind ?? 'interrupted';
260
+ state.update(id, current => {
261
+ const failures = Math.max(current.failures, (record.failuresBefore ?? 0) + (reason === 'cancelled' ? 0 : 1));
262
+ return { ...current, failures, stopped: failures >= failureThreshold };
263
+ });
264
+ journal.recordSync({ ...record, failureKind: reason });
265
+ this.checkpoint?.('after-cold-restore', { record });
266
+ this.blocked.delete(id);
267
+ this.logger?.warn?.(`[compaction-weave] Restored failed transaction ${record.compactionId} from verified backup before loading ${id}.`);
268
+ }
269
+ } finally { try { await lease?.release(); } finally { backend.tracker.releaseClaim(id); } }
270
+ }
271
+ dispose() {
272
+ // Cordis traces callable property reads, so compare the raw descriptor.
273
+ if (Object.getOwnPropertyDescriptor(this.backend, 'open')?.value === this.openWrapper) {
274
+ if (this.openDescriptor) Object.defineProperty(this.backend, 'open', this.openDescriptor);
275
+ else delete this.backend.open;
276
+ }
277
+ installed.delete(this.backend);
278
+ }
279
+ }
280
+
281
+ export function installStockTransactionAdapter(backend, sessions, logger) {
282
+ let adapter = installed.get(backend);
283
+ if (!adapter) { adapter = new StockTransactionAdapter(backend, sessions, logger); installed.set(backend, adapter); }
284
+ return adapter;
285
+ }