amalgm 0.1.238 → 0.1.240

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/lib/shared-realtime-tunnel.js +22 -14
  2. package/package.json +1 -1
  3. package/runtime/scripts/amalgm-mcp/browser/cookie-jar.js +68 -20
  4. package/runtime/scripts/amalgm-mcp/project-context/store.js +87 -19
  5. package/runtime/scripts/amalgm-mcp/state/attachments.js +40 -12
  6. package/runtime/scripts/amalgm-mcp/state/db.js +216 -17
  7. package/runtime/scripts/amalgm-mcp/state/doc-disk.js +550 -0
  8. package/runtime/scripts/amalgm-mcp/state/docs.js +1101 -301
  9. package/runtime/scripts/amalgm-mcp/state/mutation-contracts.js +99 -0
  10. package/runtime/scripts/amalgm-mcp/state/mutations.js +507 -130
  11. package/runtime/scripts/amalgm-mcp/state/promotions.js +21 -1
  12. package/runtime/scripts/amalgm-mcp/state/replicas.js +134 -13
  13. package/runtime/scripts/amalgm-mcp/state/shared-rest.js +14 -8
  14. package/runtime/scripts/amalgm-mcp/tests/browser-cookie-cloud.test.js +193 -0
  15. package/runtime/scripts/amalgm-mcp/tests/code-project-stream.test.js +313 -94
  16. package/runtime/scripts/amalgm-mcp/tests/fixtures/code-project-stream-v2.json +1 -0
  17. package/runtime/scripts/amalgm-mcp/tests/project-context.test.js +32 -17
  18. package/runtime/scripts/amalgm-mcp/tests/shared-tunnel-rehydration.test.js +84 -0
  19. package/runtime/scripts/amalgm-mcp/tests/state-docs.test.js +376 -13
  20. package/runtime/scripts/amalgm-mcp/tests/state-mutations.test.js +309 -0
  21. package/runtime/scripts/amalgm-mcp/tests/state-shared-replica.test.js +758 -30
  22. package/runtime/scripts/amalgm-mcp/tests/workspace-cards-store.test.js +72 -3
  23. package/runtime/scripts/amalgm-mcp/tests/workspace-cards.test.js +33 -6
  24. package/runtime/scripts/amalgm-mcp/tests/workspace-checkpoint-reducer.test.js +27 -0
  25. package/runtime/scripts/amalgm-mcp/tests/workspace-checkpoint.test.js +334 -191
  26. package/runtime/scripts/amalgm-mcp/tests/workspace-durable.test.js +24 -0
  27. package/runtime/scripts/amalgm-mcp/tests/workspace-local-intent.test.js +1999 -0
  28. package/runtime/scripts/amalgm-mcp/tests/workspace-native-file-transaction.test.js +364 -0
  29. package/runtime/scripts/amalgm-mcp/tests/workspace-object-tunnel.test.js +81 -3
  30. package/runtime/scripts/amalgm-mcp/tests/workspace-objects.test.js +114 -8
  31. package/runtime/scripts/amalgm-mcp/tests/workspace-provenance.test.js +185 -0
  32. package/runtime/scripts/amalgm-mcp/tests/workspace-publication-invariants.test.js +174 -269
  33. package/runtime/scripts/amalgm-mcp/tests/workspace-reducer-liveness.test.js +26 -0
  34. package/runtime/scripts/amalgm-mcp/tests/workspace-reducer.test.js +2060 -407
  35. package/runtime/scripts/amalgm-mcp/tests/workspace-sync-stats.test.js +119 -0
  36. package/runtime/scripts/amalgm-mcp/tests/workspace-transition.test.js +1446 -0
  37. package/runtime/scripts/amalgm-mcp/tests/workspace-tree-cloud.test.js +5319 -3
  38. package/runtime/scripts/amalgm-mcp/tests/workspace-tree-core.test.js +191 -1
  39. package/runtime/scripts/amalgm-mcp/tests/workspace-tree-store.test.js +3140 -17
  40. package/runtime/scripts/amalgm-mcp/tests/workspace-wall.test.js +135 -637
  41. package/runtime/scripts/amalgm-mcp/workspace/access-store.js +5 -1
  42. package/runtime/scripts/amalgm-mcp/workspace/card.js +7 -7
  43. package/runtime/scripts/amalgm-mcp/workspace/cards.js +36 -12
  44. package/runtime/scripts/amalgm-mcp/workspace/checkpoint.js +436 -173
  45. package/runtime/scripts/amalgm-mcp/workspace/content.js +32 -0
  46. package/runtime/scripts/amalgm-mcp/workspace/durable.js +45 -0
  47. package/runtime/scripts/amalgm-mcp/workspace/git.js +391 -37
  48. package/runtime/scripts/amalgm-mcp/workspace/intent.js +535 -0
  49. package/runtime/scripts/amalgm-mcp/workspace/merge.js +89 -6
  50. package/runtime/scripts/amalgm-mcp/workspace/native-file-transaction/BUILD.md +31 -0
  51. package/runtime/scripts/amalgm-mcp/workspace/native-file-transaction/bin/darwin-arm64/amalgm-file-transaction +0 -0
  52. package/runtime/scripts/amalgm-mcp/workspace/native-file-transaction/bin/linux-x64/amalgm-file-transaction +0 -0
  53. package/runtime/scripts/amalgm-mcp/workspace/native-file-transaction/build.mjs +72 -0
  54. package/runtime/scripts/amalgm-mcp/workspace/native-file-transaction/client.js +165 -0
  55. package/runtime/scripts/amalgm-mcp/workspace/native-file-transaction/go.mod +5 -0
  56. package/runtime/scripts/amalgm-mcp/workspace/native-file-transaction/go.sum +2 -0
  57. package/runtime/scripts/amalgm-mcp/workspace/native-file-transaction/main.go +2201 -0
  58. package/runtime/scripts/amalgm-mcp/workspace/native-file-transaction/manifest.json +19 -0
  59. package/runtime/scripts/amalgm-mcp/workspace/native-file-transaction/move_darwin.go +18 -0
  60. package/runtime/scripts/amalgm-mcp/workspace/native-file-transaction/move_linux.go +18 -0
  61. package/runtime/scripts/amalgm-mcp/workspace/native-file-transaction/process_darwin.go +33 -0
  62. package/runtime/scripts/amalgm-mcp/workspace/native-file-transaction/process_linux.go +47 -0
  63. package/runtime/scripts/amalgm-mcp/workspace/native-file-transaction/verify.mjs +43 -0
  64. package/runtime/scripts/amalgm-mcp/workspace/native-file-transaction/writers_darwin.go +114 -0
  65. package/runtime/scripts/amalgm-mcp/workspace/native-file-transaction/writers_linux.go +84 -0
  66. package/runtime/scripts/amalgm-mcp/workspace/objects.js +119 -54
  67. package/runtime/scripts/amalgm-mcp/workspace/project-stream.js +616 -168
  68. package/runtime/scripts/amalgm-mcp/workspace/provenance.js +240 -0
  69. package/runtime/scripts/amalgm-mcp/workspace/reducer.js +1979 -570
  70. package/runtime/scripts/amalgm-mcp/workspace/rest.js +49 -28
  71. package/runtime/scripts/amalgm-mcp/workspace/store.js +77 -4
  72. package/runtime/scripts/amalgm-mcp/workspace/sync-stats.js +240 -0
  73. package/runtime/scripts/amalgm-mcp/workspace/transition.js +1729 -192
  74. package/runtime/scripts/amalgm-mcp/workspace/tree/floor.js +100 -0
  75. package/runtime/scripts/amalgm-mcp/workspace/tree/inclusion.js +7 -1
  76. package/runtime/scripts/amalgm-mcp/workspace/tree/local-state.js +73 -61
  77. package/runtime/scripts/amalgm-mcp/workspace/tree/observation-holds.js +127 -0
  78. package/runtime/scripts/amalgm-mcp/workspace/tree/path-state.js +12 -0
  79. package/runtime/scripts/amalgm-mcp/workspace/tree/reducer.js +51 -6
  80. package/runtime/scripts/amalgm-mcp/workspace/tree/root-binding.js +125 -0
  81. package/runtime/scripts/amalgm-mcp/workspace/tree/snapshot.js +79 -8
  82. package/runtime/scripts/amalgm-mcp/workspace/tree/symlinks.js +76 -0
  83. package/runtime/scripts/amalgm-mcp/workspace/tree-cloud.js +3480 -165
  84. package/runtime/scripts/amalgm-mcp/workspace/tree-materialization.js +3766 -0
  85. package/runtime/scripts/amalgm-mcp/workspace/tree-store.js +899 -71
  86. package/runtime/scripts/amalgm-mcp/workspace/wall.js +352 -497
@@ -6,8 +6,8 @@
6
6
  * A document is a text file (markdown, code — anything in the fs layer's
7
7
  * text-extension set) whose contents are held in a Y.Doc while open. Edits
8
8
  * travel as Yjs updates in the event log's `patch` slot; the file on disk
9
- * remains the artifact (debounced write-back), so everything that reads the
10
- * file — agents, exports, git — keeps working unchanged.
9
+ * remains the artifact, so everything that reads the file — agents, exports,
10
+ * git — keeps working unchanged.
11
11
  *
12
12
  * Wire contract:
13
13
  * POST /state/docs/open {path} → {resource, path, state, seq}
@@ -30,22 +30,28 @@
30
30
  * (entry.lastDiskText) — our own write-backs and stale watcher events can
31
31
  * never splice, so they can never re-emit.
32
32
  *
33
- * When the doc has ALSO diverged from lastDiskText (live edits inside the
34
- * write-back debounce window), splicing disk text wholesale would delete
35
- * those edits. Instead the disk change three-way merges against
33
+ * When the doc has ALSO diverged from lastDiskText (live edits arrived since
34
+ * the filesystem writer's apparent base), splicing disk text wholesale would
35
+ * delete those edits. Instead the disk change three-way merges against
36
36
  * lastDiskText as the base (see ./merge3): non-overlapping hunks from both
37
37
  * sides land automatically; overlapping hunks keep the live doc's text, and
38
38
  * the full incoming disk text is preserved in a `<name>.conflicted-<stamp>`
39
39
  * sidecar next to the file, with a `conflict` field on the emitted doc
40
- * event. The next write-back then puts the merged text on disk — the
41
- * incoming version it replaces lives on in the sidecar.
40
+ * event. The journal-owned native file transaction then puts the merged text
41
+ * on disk only if that exact observed predecessor is still there.
42
42
  */
43
43
 
44
44
  const crypto = require('crypto');
45
45
  const fs = require('fs');
46
46
  const path = require('path');
47
47
  const { cachedStatement, ensureDocStatesSchema, openLocalDb } = require('./db');
48
- const { appendStateEvent, currentSeq } = require('./events');
48
+ const docDisk = require('./doc-disk');
49
+ const {
50
+ appendStateEvent,
51
+ currentSeq,
52
+ insertStateEvent,
53
+ publishStateEvent,
54
+ } = require('./events');
49
55
  const { merge3 } = require('./merge3');
50
56
  const { trace } = require('./trace');
51
57
 
@@ -53,8 +59,8 @@ const DOC_RESOURCE_PREFIX = 'doc:';
53
59
  const TEXT_KEY = 'content';
54
60
  const MAX_DOC_BYTES = Number.parseInt(process.env.AMALGM_DOC_MAX_BYTES || '', 10) || 2 * 1024 * 1024;
55
61
  const MAX_OPEN_DOCS = 64;
56
- const DISK_WRITE_DEBOUNCE_MS = 300;
57
62
  const WATCH_RECONCILE_DEBOUNCE_MS = 200;
63
+ const WATCH_VERIFY_MS = 5_000;
58
64
  const MAX_UPDATES_PER_CALL = 200;
59
65
 
60
66
  // Bounds on the per-doc disk-sync history ring (see pushDiskHistory): a small
@@ -78,8 +84,8 @@ const WRITE_EXAMPLE = '{"path": "notes.md", "text": "full new file contents", '
78
84
  // doc or feed the reconcile loop.
79
85
  const SIDECAR_NAME_RE = /\.conflicted-\d{8}-\d{6}(?:-\d+)?(?:\.[^./\\]*)?$/;
80
86
 
81
- // resolved path -> { doc, resource, reconcileTimer, diskWriteTimer,
82
- // lastAccess, lastDiskText, lastDiskHash, pathAliases }
87
+ // resolved path -> { doc, resource, reconcileTimer, lastAccess,
88
+ // lastDiskText, lastDiskHash, pathAliases }
83
89
  const openDocs = new Map();
84
90
 
85
91
  // pathInput (as the client sent it) -> resolved path, held only while the doc
@@ -92,6 +98,9 @@ const resolvedPathCache = new Map();
92
98
  // per directory instead of one per doc: N docs in a folder cost one fd, and
93
99
  // the watcher dies when its last doc closes (docs.size is the refcount).
94
100
  const dirWatchers = new Map();
101
+ let watchVerificationTimer = null;
102
+ let watchVerificationPasses = 0;
103
+ let diskTransactionTestHooks = {};
95
104
 
96
105
  function fsPrivate() {
97
106
  return require('../fs/rest')._private;
@@ -113,6 +122,10 @@ function invalid(message, statusCode = 400) {
113
122
  return Object.assign(new Error(message), { statusCode });
114
123
  }
115
124
 
125
+ function isJournalCursor(value) {
126
+ return Number.isSafeInteger(value) && value >= 0;
127
+ }
128
+
116
129
  function hashText(text) {
117
130
  return crypto.createHash('sha256').update(text, 'utf8').digest('hex');
118
131
  }
@@ -133,15 +146,16 @@ function newMutationId() {
133
146
  }
134
147
 
135
148
  // The only way lastDiskText is ever assigned: the hash is what persistence
136
- // writes, and hashing at assignment (rare: open, reconcile, flush) keeps the
149
+ // writes, and hashing at assignment (rare: open, reconcile, materialize) keeps the
137
150
  // full-text SHA-256 off the per-batch persist path. `origin` says how the
138
151
  // text got onto disk — 'disk' (external write folded in), 'flush' (the live
139
- // doc's own write-back), 'open' (seed of an unchanged file) — and is what the
152
+ // doc's own native materialization), 'open' (seed of an unchanged file) — and is what the
140
153
  // overwrite guard reads: a 'flush' head means the last disk state was
141
154
  // produced through the live channel.
142
- function setLastDiskText(entry, text, origin) {
155
+ function setLastDiskText(entry, text, origin, observation = null) {
143
156
  entry.lastDiskText = text;
144
157
  entry.lastDiskHash = text == null ? null : hashText(text);
158
+ entry.lastDiskObservation = observation;
145
159
  if (text != null) pushDiskHistory(entry, text, entry.lastDiskHash, origin || 'disk');
146
160
  }
147
161
 
@@ -154,7 +168,6 @@ function pushDiskHistory(entry, text, sha256, origin) {
154
168
  const history = entry.diskHistory;
155
169
  const head = history[history.length - 1];
156
170
  if (head && head.sha256 === sha256) {
157
- head.seq = currentSeq();
158
171
  head.at = Date.now();
159
172
  return;
160
173
  }
@@ -286,21 +299,48 @@ function isLowSurrogate(code) {
286
299
  return code >= 0xdc00 && code <= 0xdfff;
287
300
  }
288
301
 
302
+ function resetMutationAuthor(Y, entry) {
303
+ entry.mutationAuthor?.destroy();
304
+ entry.mutationAuthor = new Y.Doc();
305
+ Y.applyUpdate(
306
+ entry.mutationAuthor,
307
+ Y.encodeStateAsUpdate(entry.doc),
308
+ { amalgmMutation: true, source: 'doc:author-reset', patch: null },
309
+ );
310
+ }
311
+
289
312
  /**
290
- * Splice `nextText` into the live doc and hand back the exact update it
291
- * produced, ready for commitDocMutation. The live doc's own clientID keeps
292
- * the Yjs state vector at one entry per open session — a fresh doc per splice
293
- * would grow it forever. The in-memory apply preceding the journal insert is
294
- * safe: nothing persists before materialization, so a death in the gap leaves
295
- * disk ahead of the persisted state and reopen simply re-reconciles (a write
296
- * API caller got no acknowledgement and retries). Returns null when the text
297
- * is already current.
313
+ * Produce the exact update for a whole-text change without touching the live
314
+ * document. One stable author is mirrored from the live document before each
315
+ * splice, so its Yjs client clock stays bounded. commitDocMutation is the only
316
+ * place that applies the bytes, after the durable mutation row exists.
298
317
  */
299
- function spliceDocText(Y, entry, nextText, source, mutationId = newMutationId()) {
318
+ function spliceDocText(
319
+ Y,
320
+ entry,
321
+ nextText,
322
+ source,
323
+ mutationId = newMutationId(),
324
+ diskObservation = null,
325
+ ) {
326
+ if (!entry.mutationAuthor) resetMutationAuthor(Y, entry);
327
+ Y.applyUpdate(
328
+ entry.mutationAuthor,
329
+ Y.encodeStateAsUpdate(entry.doc),
330
+ { amalgmMutation: true, source: 'doc:author-mirror', patch: null },
331
+ );
332
+ const ytext = entry.mutationAuthor.getText(TEXT_KEY);
333
+ if (ytext.toString() === nextText) return null;
334
+ const before = Y.encodeStateVector(entry.mutationAuthor);
300
335
  const origin = { amalgmMutation: true, source, patch: null };
301
- const changed = spliceTextInto(Y, entry.doc, entry.doc.getText(TEXT_KEY), nextText, origin);
302
- if (!changed) return null;
303
- return { mutationId, update: origin.patch.yjs, bytes: null };
336
+ spliceTextInto(Y, entry.mutationAuthor, ytext, nextText, origin);
337
+ const bytes = Buffer.from(Y.encodeStateAsUpdate(entry.mutationAuthor, before));
338
+ return {
339
+ mutationId,
340
+ update: bytes.toString('base64'),
341
+ bytes,
342
+ diskObservation,
343
+ };
304
344
  }
305
345
 
306
346
  function canCurrentUserEditDoc(entry) {
@@ -313,23 +353,123 @@ function requireCurrentUserDocEdit(entry) {
313
353
  if (!canCurrentUserEditDoc(entry)) throw invalid('document edit is forbidden', 403);
314
354
  }
315
355
 
356
+ function localMutationInput(entry, replica, edit) {
357
+ return {
358
+ channelId: replica ? `resource:${replica.resourceId}` : entry.resource,
359
+ localResource: entry.resource,
360
+ sharedResourceId: replica?.resourceId,
361
+ mutationId: edit.mutationId,
362
+ deviceId: replica ? require('../workspace/identity').machineId() : null,
363
+ authorityEpoch: replica?.authorityEpoch || 0,
364
+ baseCloudVersion: replica?.appliedVersion || 0,
365
+ contract: replica?.contract || 'text-yjs@1',
366
+ schemaVersion: replica?.schemaVersion || 1,
367
+ operationKind: 'yjs.update',
368
+ operation: { kind: 'yjs.update', update: edit.update },
369
+ };
370
+ }
371
+
372
+ function materializeDocDiskMutation(
373
+ Y,
374
+ entry,
375
+ mutation,
376
+ testHooks = {},
377
+ ) {
378
+ const targetText = entry.doc.getText(TEXT_KEY).toString();
379
+ let plan = docDisk.prepare(mutation.channelId, mutation.mutationId, targetText);
380
+ for (let attempt = 0; attempt < 8; attempt += 1) {
381
+ const result = docDisk.cross(plan, openLocalDb(), testHooks);
382
+ trace('doc-disk-cross', {
383
+ mutationId: mutation.mutationId,
384
+ attempt,
385
+ status: result.status,
386
+ reason: result.reason,
387
+ target: hashText(targetText),
388
+ });
389
+ if (result.status === 'applied') {
390
+ const installed = docDisk.observe(entry.path);
391
+ if (installed.text !== targetText) {
392
+ // The receipt proves this mutation crossed. A different current file
393
+ // is therefore causally later local reality, including the narrow
394
+ // helper-return/process-crash window. Journal it as the successor
395
+ // before allowing the older mutation to complete.
396
+ const successor = stageDiskObservation(Y, entry, installed, {
397
+ predecessorChannelId: mutation.channelId,
398
+ predecessorMutationId: mutation.mutationId,
399
+ predecessorCrossed: true,
400
+ });
401
+ setLastDiskText(entry, installed.text, 'disk', installed);
402
+ persistDocState(Y, entry);
403
+ return {
404
+ crossed: true,
405
+ superseded: false,
406
+ successor,
407
+ };
408
+ }
409
+ setLastDiskText(
410
+ entry,
411
+ installed.text,
412
+ plan.mutationSource === 'doc:disk' ? 'disk' : 'flush',
413
+ installed,
414
+ );
415
+ persistDocState(Y, entry);
416
+ return { crossed: true, superseded: false };
417
+ }
418
+ if (result.status === 'pending') {
419
+ plan = docDisk.get(mutation.channelId, mutation.mutationId);
420
+ if (attempt < 7) continue;
421
+ throw new Error(
422
+ `document disk writer is still active: ${entry.path}: ${result.reason || 'pending'}`,
423
+ );
424
+ }
425
+ if (result.status !== 'changed') {
426
+ throw new Error(`unexpected document disk transaction state: ${result.status}`);
427
+ }
428
+ const newer = docDisk.observe(entry.path);
429
+ if (newer.text === targetText) {
430
+ plan = docDisk.rebaseExpected(mutation.channelId, mutation.mutationId, newer);
431
+ continue;
432
+ }
433
+ const successor = stageDiskObservation(Y, entry, newer, {
434
+ predecessorChannelId: mutation.channelId,
435
+ predecessorMutationId: mutation.mutationId,
436
+ });
437
+ if (!successor) {
438
+ plan = docDisk.rebaseExpected(mutation.channelId, mutation.mutationId, newer);
439
+ continue;
440
+ }
441
+ return { crossed: false, superseded: true, successor };
442
+ }
443
+ throw new Error(`document disk kept changing during materialization: ${entry.path}`);
444
+ }
445
+
316
446
  /**
317
447
  * The one door. Every locally produced document edit — a client Yjs update,
318
448
  * a whole-text API write, a disk reconcile — becomes exactly one journal
319
449
  * mutation: durable row, then materialization, then the projected event;
320
- * shared docs land in the cloud outbox by the same row. `bytes` is null when
321
- * the update came from spliceDocText — the live doc already holds it and
322
- * materialization only persists; client updates arrive as bytes and are
323
- * applied here. `entry.pendingConflict` (set by the merge that produced this
324
- * edit) rides the projected event and is consumed here.
450
+ * shared docs land in the cloud outbox by the same row. Updates arrive as
451
+ * bytes and are applied here only after that row exists. `entry.pendingConflict`
452
+ * (set by the merge that produced this edit) rides the projected event and
453
+ * is consumed here.
325
454
  */
326
- function commitDocMutation(Y, entry, { mutationId, update, bytes }, source) {
327
- // Finish before starting: a journal row whose materialization failed (a
328
- // full disk, a permission flip) is completed here — original mutation id,
329
- // event, cloud-outbox eligibility — before any new edit is journaled. This
330
- // is what keeps journal order and the cloud stream gap-free; if the disk is
331
- // still broken, the new edit fails too instead of shipping past a hole.
332
- recoverPendingDocMutations(Y, entry);
455
+ function commitDocMutation(
456
+ Y,
457
+ entry,
458
+ {
459
+ mutationId,
460
+ update,
461
+ bytes,
462
+ diskObservation = null,
463
+ materializationBase = null,
464
+ },
465
+ source,
466
+ ) {
467
+ // A local file save is full local reality before the API or cloud edit that
468
+ // races it. Capture it durably first. The disk-origin path already carries
469
+ // that exact observation and must not recurse through capture.
470
+ const capturedBase = diskObservation
471
+ || materializationBase
472
+ || (source === 'doc:disk' ? null : captureDiskBeforeMutation(Y, entry));
333
473
  const replica = require('./replicas').getReplicaByLocalResource(entry.resource);
334
474
  trace('doc-commit', {
335
475
  mutationId,
@@ -338,39 +478,53 @@ function commitDocMutation(Y, entry, { mutationId, update, bytes }, source) {
338
478
  shared: replica?.resourceId,
339
479
  });
340
480
  const origin = { amalgmMutation: true, source, patch: null };
341
- return require('./mutations').commitLocalMutation({
342
- channelId: replica ? `resource:${replica.resourceId}` : entry.resource,
343
- localResource: entry.resource,
344
- sharedResourceId: replica?.resourceId,
345
- mutationId,
346
- deviceId: replica ? require('../workspace/identity').machineId() : null,
347
- authorityEpoch: replica?.authorityEpoch || 0,
348
- baseCloudVersion: replica?.appliedVersion || 0,
349
- contract: replica?.contract || 'text-yjs@1',
350
- schemaVersion: replica?.schemaVersion || 1,
351
- operationKind: 'yjs.update',
352
- operation: { kind: 'yjs.update', update },
353
- }, {
354
- materialize() {
355
- if (bytes) Y.applyUpdate(entry.doc, new Uint8Array(bytes), origin);
356
- persistDocState(Y, entry);
357
- if (replica) {
358
- // Shared files have a strict saved-local boundary: the user-visible
359
- // file is current before the event/HTTP acknowledgement.
360
- flushDocToDisk(Y, entry);
361
- } else {
362
- scheduleDiskWrite(Y, entry);
363
- }
364
- },
365
- event: () => {
366
- const patch = origin.patch || { yjs: update };
367
- if (entry.pendingConflict) {
368
- patch.conflict = entry.pendingConflict;
369
- entry.pendingConflict = null;
370
- }
371
- return { resource: entry.resource, op: 'update', id: entry.path, patch, source };
372
- },
373
- });
481
+ const edit = { mutationId, update, bytes, diskObservation };
482
+ const input = localMutationInput(entry, replica, edit);
483
+ const base = capturedBase || docDisk.observe(entry.path);
484
+ let diskOutcome = null;
485
+ try {
486
+ const committed = require('./mutations').commitLocalMutation(input, {
487
+ persist(database, mutation) {
488
+ docDisk.insertBase(
489
+ database,
490
+ mutation,
491
+ base,
492
+ source,
493
+ );
494
+ },
495
+ materialize(mutation) {
496
+ if (bytes) Y.applyUpdate(entry.doc, new Uint8Array(bytes), origin);
497
+ persistDocState(Y, entry);
498
+ diskOutcome = materializeDocDiskMutation(
499
+ Y,
500
+ entry,
501
+ mutation,
502
+ diskTransactionTestHooks,
503
+ );
504
+ },
505
+ event: () => {
506
+ const patch = origin.patch || { yjs: update };
507
+ if (entry.pendingConflict) {
508
+ patch.conflict = entry.pendingConflict;
509
+ entry.pendingConflict = null;
510
+ }
511
+ return { resource: entry.resource, op: 'update', id: entry.path, patch, source };
512
+ },
513
+ });
514
+ docDisk.finish(input.channelId, input.mutationId);
515
+ if (diskOutcome?.superseded || diskOutcome?.successor) {
516
+ recoverPendingDocMutations(Y, entry);
517
+ }
518
+ return committed;
519
+ } catch (error) {
520
+ // The journal row, when one exists, owns recovery. If no row exists, the
521
+ // unchanged live document (or disk observation) is the retry source. In
522
+ // either case an author that ran ahead must not make a retry look like a
523
+ // no-op.
524
+ entry.pendingConflict = null;
525
+ resetMutationAuthor(Y, entry);
526
+ throw error;
527
+ }
374
528
  }
375
529
 
376
530
  // A merge where every incoming hunk conflicted leaves the doc text unchanged:
@@ -407,58 +561,6 @@ function persistDocState(Y, entry) {
407
561
  );
408
562
  }
409
563
 
410
- function scheduleDiskWrite(Y, entry) {
411
- if (entry.diskWriteTimer) return;
412
- entry.diskWriteTimer = setTimeout(() => {
413
- entry.diskWriteTimer = null;
414
- // The debounced write-back has no caller to answer to. A failure here
415
- // loses nothing durable — the acknowledged state lives in SQLite — so it
416
- // is logged and the next flush (edit, close, reopen) retries.
417
- try {
418
- flushDocToDisk(Y, entry);
419
- } catch (error) {
420
- console.warn(`[LiveDoc] Debounced write-back failed for "${entry.path}":`, error?.message || error);
421
- }
422
- }, DISK_WRITE_DEBOUNCE_MS);
423
- if (typeof entry.diskWriteTimer.unref === 'function') entry.diskWriteTimer.unref();
424
- }
425
-
426
- // Replace-by-rename so the file is always either the old text or the new
427
- // text, never a torn partial write. The rename replaces the inode, so the
428
- // original's permission bits must be carried over explicitly — chmod, not an
429
- // open-mode, because open(2) modes are masked by the umask. An executable
430
- // script stays executable through a live edit.
431
- function writeFileAtomic(filePath, text) {
432
- const tmpPath = path.join(path.dirname(filePath), `.${path.basename(filePath)}.amalgm-write`);
433
- let mode = null;
434
- try {
435
- mode = fs.statSync(filePath).mode;
436
- } catch (error) {
437
- if (error.code !== 'ENOENT') throw error;
438
- }
439
- fs.writeFileSync(tmpPath, text, 'utf8');
440
- if (mode != null) fs.chmodSync(tmpPath, mode);
441
- fs.renameSync(tmpPath, filePath);
442
- }
443
-
444
- // Throws on failure: "saved locally" must mean the file is really current.
445
- // Callers that acknowledge an edit (mutation materialization) propagate the
446
- // failure so the journal row is never completed and no event is projected;
447
- // only fire-and-forget callers (debounce timer, close) may catch and log.
448
- function flushDocToDisk(Y, entry) {
449
- const text = entry.doc.getText(TEXT_KEY).toString();
450
- let onDisk = null;
451
- try {
452
- onDisk = fs.readFileSync(entry.path, 'utf8');
453
- } catch (error) {
454
- // A deleted file is not a failed save — the write below recreates it.
455
- if (error.code !== 'ENOENT') throw error;
456
- }
457
- if (onDisk !== text) writeFileAtomic(entry.path, text);
458
- setLastDiskText(entry, text, 'flush');
459
- persistDocState(Y, entry);
460
- }
461
-
462
564
  function isConflictSidecar(name) {
463
565
  return SIDECAR_NAME_RE.test(name);
464
566
  }
@@ -500,31 +602,59 @@ function writeConflictSidecar(entry, theirsText, hunkCount) {
500
602
  );
501
603
  }
502
604
 
503
- function reconcileFromDisk(Y, entry) {
504
- try {
505
- const onDisk = fs.readFileSync(entry.path, 'utf8');
506
- // Only fold in disk content that actually CHANGED since we last read or
507
- // wrote it. Without this, a watcher event firing while the doc is ahead
508
- // of disk (write-back still debounced) would "reconcile" the stale disk
509
- // text back over live edits. (This is also the THEIRS === BASE no-op.)
510
- if (onDisk === entry.lastDiskText) return;
511
- if (!canCurrentUserEditDoc(entry)) {
512
- entry.heldDiskChange = { reason: 'forbidden', at: new Date().toISOString() };
513
- return;
514
- }
515
- entry.heldDiskChange = null;
516
- const ytext = entry.doc.getText(TEXT_KEY);
517
- const ours = ytext.toString();
518
- const base = entry.lastDiskText;
519
- let nextText = onDisk;
520
- // OURS === BASE (no live divergence) or no base at all (first seed):
521
- // disk text applies wholesale, exactly as before. Otherwise both sides
522
- // moved since the last sync — three-way merge against BASE so a writer
523
- // that read the file before our live edits cannot delete them.
524
- if (base != null && ours !== base && ours !== onDisk) {
525
- const merged = merge3(base, ours, onDisk);
526
- nextText = merged.text;
527
- if (merged.conflicts.length > 0) {
605
+ function diskEditForObservation(Y, entry, observation) {
606
+ const onDisk = observation.text;
607
+ if (onDisk === null) return null;
608
+ if (
609
+ onDisk === entry.lastDiskText
610
+ && (
611
+ !entry.lastDiskObservation
612
+ || docDisk.sameObservation(observation, entry.lastDiskObservation)
613
+ )
614
+ ) return null;
615
+ const ytext = entry.doc.getText(TEXT_KEY);
616
+ const ours = ytext.toString();
617
+ // Synchronous materialization means the newest history value may be a
618
+ // local Yjs edit that a filesystem writer never read. When that writer
619
+ // replaces the file from the immediately preceding disk version, merge
620
+ // against that predecessor just as the former debounce window did.
621
+ const history = entry.diskHistory;
622
+ const prior = history.length > 1 ? history[history.length - 2] : null;
623
+ const base = liveEditsSinceLastDiskState(entry) && prior
624
+ ? prior.text
625
+ : entry.lastDiskText;
626
+ let nextText = onDisk;
627
+ if (
628
+ base != null
629
+ && ours !== base
630
+ && ours !== onDisk
631
+ && overwriteDeletesText(ours, onDisk)
632
+ ) {
633
+ const merged = merge3(base, ours, onDisk);
634
+ nextText = merged.text;
635
+ if (merged.conflicts.length > 0) {
636
+ const conflictedOursBytes = merged.conflicts.reduce(
637
+ (total, conflict) => total + Buffer.byteLength(conflict.ours, 'utf8'),
638
+ 0,
639
+ );
640
+ const oursBytes = Math.max(1, Buffer.byteLength(ours, 'utf8'));
641
+ if (conflictedOursBytes / oursBytes >= 0.8) {
642
+ // A raw filesystem writer is allowed to replace the whole file. Keep
643
+ // its rewrite in the main artifact while preserving the displaced
644
+ // live value and surfacing the loss boundary.
645
+ nextText = onDisk;
646
+ const sidecarPath = writePreservationSidecar(
647
+ entry,
648
+ ours,
649
+ 'unversioned disk write replaced flushed live edits; '
650
+ + 'replaced text preserved in the sidecar',
651
+ );
652
+ entry.pendingConflict = {
653
+ reason: 'unversioned-overwrite',
654
+ sidecarPath,
655
+ at: new Date().toISOString(),
656
+ };
657
+ } else {
528
658
  const sidecarPath = writeConflictSidecar(entry, onDisk, merged.conflicts.length);
529
659
  entry.pendingConflict = {
530
660
  reason: 'overlapping-hunks',
@@ -533,52 +663,181 @@ function reconcileFromDisk(Y, entry) {
533
663
  at: new Date().toISOString(),
534
664
  };
535
665
  }
536
- } else if (liveEditsSinceLastDiskState(entry) && overwriteDeletesText(ours, onDisk)) {
537
- // Wholesale accept, but the incoming write DELETES text that arrived
538
- // through the live channel and was already flushed (ours ===
539
- // lastDiskText, so the merge gate above sees no divergence — this was
540
- // the silent-loss window). A raw fs write carries no base, so lineage
541
- // is genuinely ambiguous: the writer keeps the freedom to rewrite the
542
- // file, but the replaced live text is preserved first and the
543
- // overwrite is surfaced, never silent. An insertion-only write (a vim
544
- // append, a new paragraph) destroys nothing and applies silently.
545
- const sidecarPath = writePreservationSidecar(
546
- entry,
547
- ours,
548
- 'unversioned disk write replaced flushed live edits; '
549
- + 'replaced text preserved in the sidecar',
550
- );
551
- entry.pendingConflict = {
552
- reason: 'unversioned-overwrite',
553
- sidecarPath,
554
- at: new Date().toISOString(),
555
- };
556
666
  }
557
- setLastDiskText(entry, onDisk, 'disk');
558
- const edit = spliceDocText(Y, entry, nextText, 'doc:disk');
559
- if (edit) {
560
- // The external edit walks through the same door as a client mutation:
561
- // journal row, materialization (which writes any merged text back — the
562
- // file artifact follows the doc), event, and the cloud outbox when the
563
- // doc is shared.
564
- commitDocMutation(Y, entry, edit, 'doc:disk');
565
- } else {
667
+ } else if (liveEditsSinceLastDiskState(entry) && overwriteDeletesText(ours, onDisk)) {
668
+ const sidecarPath = writePreservationSidecar(
669
+ entry,
670
+ ours,
671
+ 'unversioned disk write replaced flushed live edits; '
672
+ + 'replaced text preserved in the sidecar',
673
+ );
674
+ entry.pendingConflict = {
675
+ reason: 'unversioned-overwrite',
676
+ sidecarPath,
677
+ at: new Date().toISOString(),
678
+ };
679
+ }
680
+ return spliceDocText(
681
+ Y,
682
+ entry,
683
+ nextText,
684
+ 'doc:disk',
685
+ newMutationId(),
686
+ observation,
687
+ );
688
+ }
689
+
690
+ function isKnownDiskObservation(entry, observation) {
691
+ return Boolean(
692
+ observation.text !== null
693
+ && observation.text === entry.lastDiskText
694
+ && (
695
+ !entry.lastDiskObservation
696
+ || docDisk.sameObservation(observation, entry.lastDiskObservation)
697
+ )
698
+ );
699
+ }
700
+
701
+ function holdDiskObservation(entry, observation, reason) {
702
+ entry.heldDiskChange = {
703
+ reason,
704
+ identity: observation.identity,
705
+ at: new Date().toISOString(),
706
+ };
707
+ throw invalid(`document disk change is held: ${reason}`, 409);
708
+ }
709
+
710
+ function stageDiskObservation(Y, entry, observation, options = {}) {
711
+ if (isKnownDiskObservation(entry, observation)) {
712
+ entry.heldDiskChange = null;
713
+ return null;
714
+ }
715
+ if (observation.text === null) {
716
+ return holdDiskObservation(entry, observation, 'missing');
717
+ }
718
+ if (!canCurrentUserEditDoc(entry)) {
719
+ return holdDiskObservation(entry, observation, 'forbidden');
720
+ }
721
+ entry.heldDiskChange = null;
722
+ let edit = diskEditForObservation(Y, entry, observation);
723
+ if (!edit) {
724
+ if (entry.pendingConflict) {
725
+ const bytes = Buffer.from(Y.encodeStateAsUpdate(entry.doc));
726
+ edit = {
727
+ mutationId: newMutationId(),
728
+ update: bytes.toString('base64'),
729
+ bytes,
730
+ diskObservation: observation,
731
+ };
732
+ } else if (observation.text === entry.doc.getText(TEXT_KEY).toString()) {
733
+ setLastDiskText(entry, observation.text, 'disk', observation);
566
734
  publishConflictOnly(entry, 'doc:disk');
567
- // Even a no-op still refreshed lastDiskText; record its hash.
568
735
  persistDocState(Y, entry);
736
+ } else {
737
+ return null;
738
+ }
739
+ if (!edit) return null;
740
+ }
741
+ const replica = require('./replicas').getReplicaByLocalResource(entry.resource);
742
+ const input = localMutationInput(entry, replica, edit);
743
+ const persist = (database, mutation) => {
744
+ docDisk.insertBase(database, mutation, observation, 'doc:disk');
745
+ if (options.predecessorChannelId && options.predecessorMutationId) {
746
+ if (options.predecessorCrossed) {
747
+ docDisk.linkCrossedSuccessor(
748
+ database,
749
+ options.predecessorChannelId,
750
+ options.predecessorMutationId,
751
+ mutation.channelId,
752
+ mutation.mutationId,
753
+ );
754
+ } else {
755
+ docDisk.markSuperseded(
756
+ database,
757
+ options.predecessorChannelId,
758
+ options.predecessorMutationId,
759
+ mutation.channelId,
760
+ mutation.mutationId,
761
+ );
762
+ }
569
763
  }
764
+ };
765
+ if (!options.predecessorChannelId && !options.predecessorMutationId) {
766
+ return {
767
+ edit,
768
+ input,
769
+ committed: commitDocMutation(Y, entry, edit, 'doc:disk'),
770
+ };
771
+ }
772
+ const started = require('./mutations').journalMutation(input, {
773
+ persist(database, mutation) {
774
+ persist(database, mutation);
775
+ },
776
+ });
777
+ return { edit, input, mutation: started.mutation };
778
+ }
779
+
780
+ function captureDiskBeforeMutation(Y, entry) {
781
+ // A snapshot helper can die after installing official bytes but before the
782
+ // in-memory document and cursor commit. Recover that physical receipt first;
783
+ // otherwise a cached entry can mistake those official bytes for a new local
784
+ // save and publish an echo before the snapshot is replayed.
785
+ recoverPendingDocSnapshots(Y, entry);
786
+ recoverPendingDocMutations(Y, entry);
787
+ for (let attempt = 0; attempt < 8; attempt += 1) {
788
+ const observation = docDisk.observe(entry.path);
789
+ const staged = stageDiskObservation(Y, entry, observation);
790
+ if (!staged) {
791
+ // Return the exact observation whose provenance was just proved. A
792
+ // later reread could contain an unjournaled save and must never become
793
+ // overwrite permission for the caller's native CAS.
794
+ return observation;
795
+ }
796
+ recoverPendingDocMutations(Y, entry);
797
+ }
798
+ throw new Error(`document disk kept changing during capture: ${entry.path}`);
799
+ }
800
+
801
+ function reconcileFromDisk(Y, entry, strict = false) {
802
+ try {
803
+ captureDiskBeforeMutation(Y, entry);
570
804
  } catch (error) {
571
- // File deleted or unreadable — the doc simply stops following the disk.
805
+ if (strict) throw error;
572
806
  console.warn(`[LiveDoc] Reconcile from disk failed for "${entry.path}":`, error?.message || error);
573
807
  }
574
808
  }
575
809
 
810
+ function restoreDurableDiskBase(entry, persisted) {
811
+ const plan = docDisk.firstBase(entry.resource);
812
+ if (plan?.baseText !== null && plan?.baseText !== undefined) {
813
+ // Every destructive writer seals its exact semantic predecessor beside
814
+ // the physical CAS. That text remains the merge base even when a crash
815
+ // happens after the desired Y.Doc state was persisted but before the
816
+ // file crossed.
817
+ setLastDiskText(entry, plan.baseText, 'open');
818
+ return true;
819
+ }
820
+
821
+ const knownHash = persisted?.last_disk_hash;
822
+ if (!knownHash) return false;
823
+ const persistedText = entry.doc.getText(TEXT_KEY).toString();
824
+ if (hashText(persistedText) !== knownHash) return false;
825
+
826
+ // The persisted Y.Doc and last-disk hash together prove the semantic base
827
+ // that was on disk before the process stopped. The file may have changed
828
+ // since then, so this is merge provenance—not permission to replace its
829
+ // current inode. Physical permission still comes only from the durable
830
+ // plan's expected identity and the native CAS.
831
+ setLastDiskText(entry, persistedText, 'open');
832
+ return true;
833
+ }
834
+
576
835
  /**
577
836
  * First look at the disk after (re)opening a doc. Three cases:
578
837
  * - no persisted history → the disk seeds the doc (first ever open);
579
838
  * - disk matches the hash we recorded before dying → disk is unchanged;
580
- * if the persisted doc is ahead (edits inside the unflushed debounce
581
- * window), write the doc back out instead of splicing stale disk text
839
+ * if the persisted doc is ahead (an interrupted journal-owned file
840
+ * transaction), finish that durable mutation instead of splicing stale disk text
582
841
  * over those edits;
583
842
  * - disk differs from the recorded hash → a genuine external edit while we
584
843
  * were closed; fold it in like any other disk change.
@@ -588,10 +847,11 @@ function seedFromDisk(Y, entry, persisted) {
588
847
  if (!knownHash) {
589
848
  if (!canCurrentUserEditDoc(entry)) {
590
849
  try {
591
- const onDisk = fs.readFileSync(entry.path, 'utf8');
850
+ const observation = docDisk.observe(entry.path);
851
+ const onDisk = observation.text;
592
852
  const origin = { amalgmMutation: true, source: 'doc:read-only-seed', patch: null };
593
853
  spliceTextInto(Y, entry.doc, entry.doc.getText(TEXT_KEY), onDisk, origin);
594
- setLastDiskText(entry, onDisk, 'open');
854
+ setLastDiskText(entry, onDisk, 'open', observation);
595
855
  persistDocState(Y, entry);
596
856
  } catch {
597
857
  // Unreadable/deleted: same posture as reconcile — stop following disk.
@@ -602,8 +862,10 @@ function seedFromDisk(Y, entry, persisted) {
602
862
  return;
603
863
  }
604
864
  let onDisk = null;
865
+ let observation = null;
605
866
  try {
606
- onDisk = fs.readFileSync(entry.path, 'utf8');
867
+ observation = docDisk.observe(entry.path);
868
+ onDisk = observation.text;
607
869
  } catch {
608
870
  // Unreadable/deleted: same posture as reconcile — stop following disk.
609
871
  return;
@@ -612,9 +874,15 @@ function seedFromDisk(Y, entry, persisted) {
612
874
  reconcileFromDisk(Y, entry);
613
875
  return;
614
876
  }
615
- setLastDiskText(entry, onDisk, 'open');
877
+ setLastDiskText(entry, onDisk, 'open', observation);
616
878
  if (entry.doc.getText(TEXT_KEY).toString() !== onDisk) {
617
- scheduleDiskWrite(Y, entry);
879
+ const bytes = Buffer.from(Y.encodeStateAsUpdate(entry.doc));
880
+ commitDocMutation(Y, entry, {
881
+ mutationId: newMutationId(),
882
+ update: bytes.toString('base64'),
883
+ bytes,
884
+ diskObservation: observation,
885
+ }, 'doc:recovery');
618
886
  }
619
887
  }
620
888
 
@@ -627,6 +895,35 @@ function scheduleReconcile(Y, entry) {
627
895
  if (typeof entry.reconcileTimer.unref === 'function') entry.reconcileTimer.unref();
628
896
  }
629
897
 
898
+ /**
899
+ * Filesystem notifications make ordinary edits fast, but they are not
900
+ * durable evidence. This bounded pass makes every open document converge
901
+ * even when the operating system coalesces or drops a notification.
902
+ */
903
+ function verifyOpenDocs() {
904
+ watchVerificationPasses += 1;
905
+ const Y = loadYjs();
906
+ for (const entry of openDocs.values()) {
907
+ if (isInternalStateDoc(entry.path)) continue;
908
+ reconcileFromDisk(Y, entry);
909
+ if (!dirWatchers.has(path.dirname(entry.path))) watchDocFile(Y, entry);
910
+ }
911
+ }
912
+
913
+ function ensureWatchVerification() {
914
+ if (watchVerificationTimer || openDocs.size === 0) return;
915
+ watchVerificationTimer = setInterval(verifyOpenDocs, WATCH_VERIFY_MS);
916
+ if (typeof watchVerificationTimer.unref === 'function') {
917
+ watchVerificationTimer.unref();
918
+ }
919
+ }
920
+
921
+ function stopWatchVerificationIfIdle() {
922
+ if (openDocs.size > 0 || !watchVerificationTimer) return;
923
+ clearInterval(watchVerificationTimer);
924
+ watchVerificationTimer = null;
925
+ }
926
+
630
927
  function watchDocFile(Y, entry) {
631
928
  const dir = path.dirname(entry.path);
632
929
  let shared = dirWatchers.get(dir);
@@ -674,36 +971,36 @@ function unwatchDocFile(entry) {
674
971
  }
675
972
 
676
973
  function closeDocEntry(Y, entry) {
677
- if (entry.diskWriteTimer) {
678
- clearTimeout(entry.diskWriteTimer);
679
- entry.diskWriteTimer = null;
680
- }
681
974
  if (entry.reconcileTimer) {
682
975
  clearTimeout(entry.reconcileTimer);
683
976
  entry.reconcileTimer = null;
684
977
  }
685
- // Close must complete even when the disk is unwritable: every acknowledged
686
- // edit is already durable in SQLite, and reopen flushes the file current.
687
- try {
688
- flushDocToDisk(Y, entry);
689
- } catch (error) {
690
- console.warn(`[LiveDoc] Close flush failed for "${entry.path}":`, error?.message || error);
691
- }
978
+ // Capture a last-moment external save before closing. If journaling fails,
979
+ // leave that disk value untouched; reopen sees it as unknown input and
980
+ // retries. Close is still allowed to release memory.
981
+ if (!isInternalStateDoc(entry.path)) reconcileFromDisk(Y, entry);
692
982
  unwatchDocFile(entry);
693
983
  for (const alias of entry.pathAliases) resolvedPathCache.delete(alias);
984
+ entry.mutationAuthor?.destroy();
694
985
  entry.doc.destroy();
695
986
  openDocs.delete(entry.path);
987
+ stopWatchVerificationIfIdle();
696
988
  }
697
989
 
698
990
  function recoverPendingDocMutations(Y, entry) {
699
- const pending = require('./mutations').listMutations({ state: 'saving-local', limit: 1000 })
700
- .filter((mutation) => mutation.localResource === entry.resource);
701
- for (const mutation of pending) {
991
+ const mutations = require('./mutations');
992
+ for (let pass = 0; pass < 1000; pass += 1) {
993
+ const [mutation] = mutations.listMutations({
994
+ state: 'saving-local',
995
+ localResource: entry.resource,
996
+ limit: 1,
997
+ });
998
+ if (!mutation) return;
702
999
  if (
703
1000
  mutation.operationKind !== 'yjs.update'
704
1001
  || typeof mutation.operation?.update !== 'string'
705
1002
  ) {
706
- continue;
1003
+ throw new Error(`document mutation ${mutation.mutationId} has an unsupported operation`);
707
1004
  }
708
1005
  const bytes = Buffer.from(mutation.operation.update, 'base64');
709
1006
  try {
@@ -712,11 +1009,52 @@ function recoverPendingDocMutations(Y, entry) {
712
1009
  throw invalid(`cannot recover mutation ${mutation.mutationId}: ${error?.message || error}`, 500);
713
1010
  }
714
1011
  const origin = { amalgmMutation: true, source: 'recovery', patch: null };
1012
+ const beforeMutationText = entry.doc.getText(TEXT_KEY).toString();
715
1013
  Y.applyUpdate(entry.doc, new Uint8Array(bytes), origin);
716
1014
  persistDocState(Y, entry);
717
- if (mutation.sharedResourceId) flushDocToDisk(Y, entry);
718
- else scheduleDiskWrite(Y, entry);
719
- require('./mutations').completeMaterializedMutation(
1015
+ let plan = docDisk.get(mutation.channelId, mutation.mutationId);
1016
+ if (!plan) {
1017
+ // Legacy saving-local rows predate physical plans. The current file may
1018
+ // be a newer save, so make it the old plan's base and immediately ask
1019
+ // the ordinary disk detector whether it is a successor. Only an
1020
+ // unchanged predecessor may then be replaced.
1021
+ const base = docDisk.observe(entry.path);
1022
+ if (entry.lastDiskText === null) {
1023
+ setLastDiskText(entry, beforeMutationText, 'open');
1024
+ }
1025
+ docDisk.insertBase(openLocalDb(), mutation, base, 'doc:recovery');
1026
+ stageDiskObservation(Y, entry, base, {
1027
+ predecessorChannelId: mutation.channelId,
1028
+ predecessorMutationId: mutation.mutationId,
1029
+ });
1030
+ plan = docDisk.get(mutation.channelId, mutation.mutationId);
1031
+ }
1032
+ const ownsSuccessor = (
1033
+ plan.successorChannelId !== null
1034
+ && plan.successorMutationId !== null
1035
+ );
1036
+ if (plan.state === 'superseded' || (plan.state === 'crossed' && ownsSuccessor)) {
1037
+ const successor = mutations.getMutation(
1038
+ plan.successorChannelId,
1039
+ plan.successorMutationId,
1040
+ { revealOperation: true },
1041
+ );
1042
+ if (!successor) {
1043
+ throw new Error(`document mutation ${mutation.mutationId} lost its disk successor`);
1044
+ }
1045
+ // A crossed predecessor with a linked successor has already proved both
1046
+ // sides of the handoff. Re-running its physical materializer would
1047
+ // observe the successor as a new edit and assign the same save a second
1048
+ // birth certificate.
1049
+ } else {
1050
+ materializeDocDiskMutation(
1051
+ Y,
1052
+ entry,
1053
+ mutation,
1054
+ diskTransactionTestHooks,
1055
+ );
1056
+ }
1057
+ mutations.completeMaterializedMutation(
720
1058
  mutation.channelId,
721
1059
  mutation.mutationId,
722
1060
  {
@@ -724,10 +1062,37 @@ function recoverPendingDocMutations(Y, entry) {
724
1062
  op: 'update',
725
1063
  id: entry.path,
726
1064
  patch: origin.patch || { yjs: mutation.operation.update },
727
- source: mutation.origin === 'cloud' ? 'doc:cloud-recovery' : 'doc:recovery',
1065
+ source: mutation.origin === 'cloud'
1066
+ ? 'doc:cloud-recovery'
1067
+ : plan.mutationSource === 'doc:disk'
1068
+ ? 'doc:disk'
1069
+ : 'doc:recovery',
728
1070
  resourceVersion: mutation.officialCloudVersion,
729
1071
  },
730
1072
  );
1073
+ docDisk.finish(mutation.channelId, mutation.mutationId);
1074
+ }
1075
+ throw new Error(`document mutation recovery exceeded its finite bound: ${entry.path}`);
1076
+ }
1077
+
1078
+ function releaseCompletedDocPlans(entry) {
1079
+ const mutations = require('./mutations');
1080
+ for (const plan of docDisk.listMutations(entry.resource)) {
1081
+ const mutation = mutations.getMutation(
1082
+ plan.channelId,
1083
+ plan.mutationId,
1084
+ { revealOperation: true },
1085
+ );
1086
+ if (!mutation) {
1087
+ // Retention may prune a completed mutation after its event is durable
1088
+ // but before a crash-left receipt is released. The plan itself proves
1089
+ // whether it crossed; finish still rejects an incomplete orphan.
1090
+ docDisk.finish(plan.channelId, plan.mutationId);
1091
+ continue;
1092
+ }
1093
+ if (mutation.state !== 'saving-local') {
1094
+ docDisk.finish(plan.channelId, plan.mutationId);
1095
+ }
731
1096
  }
732
1097
  }
733
1098
 
@@ -783,13 +1148,13 @@ function loadDocEntry(pathInput) {
783
1148
  nodeId,
784
1149
  doc,
785
1150
  reconcileTimer: null,
786
- diskWriteTimer: null,
787
1151
  lastAccess: Date.now(),
788
1152
  // The disk text as of our last read/write; reconcile skips anything that
789
1153
  // hasn't changed past it (see reconcileFromDisk). Assigned only through
790
1154
  // setLastDiskText so the hash that persistence writes stays in step.
791
1155
  lastDiskText: null,
792
1156
  lastDiskHash: null,
1157
+ lastDiskObservation: null,
793
1158
  // Bounded ring of recent disk-sync states {text, sha256, seq, at, origin}
794
1159
  // (see pushDiskHistory): the base-resolution window for versioned writes
795
1160
  // and the live-edit detector for unversioned overwrites. Newest last.
@@ -798,19 +1163,26 @@ function loadDocEntry(pathInput) {
798
1163
  // consumed by the very next disk-origin update event (the merge splice is
799
1164
  // synchronous) so the conflict rides the doc's own event patch.
800
1165
  pendingConflict: null,
1166
+ // Whole-text changes are generated by one stable Yjs writer but do not
1167
+ // touch the live doc until their mutation row exists.
1168
+ mutationAuthor: null,
801
1169
  // Every pathInput form seen for this doc, so close can clear its
802
1170
  // resolvedPathCache entries.
803
1171
  pathAliases: new Set([pathInput]),
804
1172
  };
805
1173
 
806
- let persisted = nodeId
807
- ? cachedStatement(database, `
1174
+ const readPersisted = () => (
1175
+ nodeId
1176
+ ? cachedStatement(database, `
808
1177
  SELECT state, last_disk_hash FROM doc_states
809
1178
  WHERE node_id = ? OR path = ?
810
1179
  ORDER BY CASE WHEN node_id = ? THEN 0 ELSE 1 END
811
1180
  LIMIT 1
812
1181
  `).get(nodeId, resolved, nodeId) || null
813
- : cachedStatement(database, 'SELECT state, last_disk_hash FROM doc_states WHERE path = ?').get(resolved) || null;
1182
+ : cachedStatement(database, 'SELECT state, last_disk_hash FROM doc_states WHERE path = ?')
1183
+ .get(resolved) || null
1184
+ );
1185
+ let persisted = readPersisted();
814
1186
  if (persisted?.state) {
815
1187
  try {
816
1188
  Y.applyUpdate(doc, new Uint8Array(persisted.state), 'persisted');
@@ -833,33 +1205,52 @@ function loadDocEntry(pathInput) {
833
1205
  origin.patch = { yjs: Buffer.from(update).toString('base64') };
834
1206
  });
835
1207
 
836
- // Disk at open: an externally changed file folds in as an edit, but a file
837
- // that merely lags our persisted state (the process died inside the disk
838
- // write debounce) gets the doc written back out — never spliced over it.
839
- seedFromDisk(Y, entry, persisted);
840
- if (nodeId) {
841
- cachedStatement(database, 'UPDATE doc_states SET node_id = ?, path = ? WHERE node_id = ? OR path = ?')
842
- .run(nodeId, resolved, nodeId, resolved);
843
- }
1208
+ // Journal completion and native-receipt release are separately durable.
1209
+ // A crash between them leaves a harmless receipt, which is released here
1210
+ // before any new plan can accumulate beside it.
1211
+ releaseCompletedDocPlans(entry);
1212
+
1213
+ // Restore the last proven semantic disk base before any durable recovery.
1214
+ // A pending official row and a disjoint save made while this process was
1215
+ // down must both derive from the exact text sealed in its physical plan;
1216
+ // the persisted hash is only the legacy fallback.
1217
+ restoreDurableDiskBase(entry, persisted);
1218
+
1219
+ // A crash may leave an official snapshot physically crossed but not yet
1220
+ // registered. Finish that recorded plan before ordinary disk seeding can
1221
+ // misclassify its bytes as a new local edit.
1222
+ recoverPendingDocSnapshots(Y, entry);
844
1223
 
845
1224
  // A process can die after the durable journal insert but before Y.Doc/file
846
1225
  // materialization or event projection. Replay those exact operation IDs
847
- // before this open is acknowledged, so a restart cannot bury an edit.
1226
+ // before ordinary disk seeding. Otherwise a successor already owned by a
1227
+ // pending row can look like a fresh save and receive a second mutation ID.
848
1228
  try {
849
1229
  recoverPendingDocMutations(Y, entry);
850
1230
  } catch (error) {
851
- if (entry.diskWriteTimer) clearTimeout(entry.diskWriteTimer);
852
1231
  if (entry.reconcileTimer) clearTimeout(entry.reconcileTimer);
853
1232
  doc.destroy();
854
1233
  throw error;
855
1234
  }
856
1235
 
1236
+ // Disk at open: an externally changed file folds in as an edit, but a file
1237
+ // that merely lags our persisted state is recovered from the journal first.
1238
+ // Re-read the hash because recovery above may have advanced both the live
1239
+ // document and its exact physical provenance.
1240
+ persisted = readPersisted();
1241
+ seedFromDisk(Y, entry, persisted);
1242
+ if (nodeId) {
1243
+ cachedStatement(database, 'UPDATE doc_states SET node_id = ?, path = ? WHERE node_id = ? OR path = ?')
1244
+ .run(nodeId, resolved, nodeId, resolved);
1245
+ }
1246
+
857
1247
  evictIfNeeded(Y);
858
1248
  openDocs.set(resolved, entry);
859
1249
  // Protocol documents are written only through this module. Watching their
860
- // own atomic flush as if it were an external user edit creates a second
1250
+ // own native materialization as if it were an external user edit creates a second
861
1251
  // concurrent mutation of the same value.
862
1252
  if (!isInternalStateDoc(resolved)) watchDocFile(Y, entry);
1253
+ ensureWatchVerification();
863
1254
  resolvedPathCache.set(pathInput, resolved);
864
1255
  return entry;
865
1256
  }
@@ -950,12 +1341,10 @@ function writeDocText(pathInput, textInput, options = {}) {
950
1341
  const entry = loadDocEntry(pathInput);
951
1342
  entry.lastAccess = Date.now();
952
1343
  requireCurrentUserDocEdit(entry);
953
- // Every respond() below acknowledges saved state, so it must stand on a
954
- // settled journal — most pointedly the noop return: a retry of a write
955
- // whose disk flush failed finds the text already in memory, and without
956
- // this the original mutation would stay stuck while the caller is told all
957
- // is well. (Edits that DO reach commitDocMutation settle there.)
958
- recoverPendingDocMutations(Y, entry);
1344
+ // A dropped watcher cannot make a direct disk save disappear under this
1345
+ // API write. Capture and materialize every unexplained byte first, then
1346
+ // derive the merge from the resulting live state.
1347
+ const materializationBase = captureDiskBeforeMutation(Y, entry);
959
1348
  const ytext = entry.doc.getText(TEXT_KEY);
960
1349
  const ours = ytext.toString();
961
1350
 
@@ -1021,9 +1410,29 @@ function writeDocText(pathInput, textInput, options = {}) {
1021
1410
 
1022
1411
  entry.pendingConflict = conflict;
1023
1412
  const edit = spliceDocText(Y, entry, nextText, 'doc:write', mutationId);
1024
- if (edit) commitDocMutation(Y, entry, edit, 'doc:write');
1413
+ if (edit) {
1414
+ if (diskTransactionTestHooks.pauseAfterEditBeforeJournal > 0) {
1415
+ Atomics.wait(
1416
+ new Int32Array(new SharedArrayBuffer(4)),
1417
+ 0,
1418
+ 0,
1419
+ diskTransactionTestHooks.pauseAfterEditBeforeJournal,
1420
+ );
1421
+ }
1422
+ commitDocMutation(
1423
+ Y,
1424
+ entry,
1425
+ { ...edit, materializationBase },
1426
+ 'doc:write',
1427
+ );
1428
+ }
1025
1429
  else publishConflictOnly(entry, 'doc:write');
1026
- return respond(merge, nextText, conflict, edit?.mutationId || null);
1430
+ return respond(
1431
+ merge,
1432
+ entry.doc.getText(TEXT_KEY).toString(),
1433
+ conflict,
1434
+ edit?.mutationId || null,
1435
+ );
1027
1436
  }
1028
1437
 
1029
1438
  function normalizeDocMutations(Y, mutationsInput) {
@@ -1059,6 +1468,7 @@ function applyDocMutations(pathInput, mutationsInput) {
1059
1468
  const entry = loadDocEntry(pathInput);
1060
1469
  entry.lastAccess = Date.now();
1061
1470
  requireCurrentUserDocEdit(entry);
1471
+ captureDiskBeforeMutation(Y, entry);
1062
1472
  const results = mutations.map((mutation) => commitDocMutation(Y, entry, mutation, 'doc:update'));
1063
1473
 
1064
1474
  return {
@@ -1093,18 +1503,30 @@ function applyCloudDocMutation(input) {
1093
1503
  if (entry.resource !== replica.localResource) {
1094
1504
  throw invalid(`shared resource ${envelope.resourceId} local binding changed`, 409);
1095
1505
  }
1506
+ // A save whose watcher was delayed is causally before the incoming cloud
1507
+ // event and must acquire durable local provenance before cloud ingest can
1508
+ // advance the official cursor.
1509
+ const base = captureDiskBeforeMutation(Y, entry);
1096
1510
  const origin = { amalgmMutation: true, source: 'cloud', patch: null };
1097
- return require('./mutations').ingestCloudMutation({
1511
+ let diskOutcome = null;
1512
+ const result = require('./mutations').ingestCloudMutation({
1098
1513
  envelope,
1099
1514
  version: input?.version,
1100
1515
  committedAt: input?.committedAt,
1101
1516
  localResource: entry.resource,
1102
1517
  }, {
1103
- materialize() {
1518
+ persist(database, journaled) {
1519
+ docDisk.insertBase(database, journaled, base, 'doc:cloud');
1520
+ },
1521
+ materialize(journaled) {
1104
1522
  Y.applyUpdate(entry.doc, new Uint8Array(mutation.bytes), origin);
1105
1523
  persistDocState(Y, entry);
1106
- // Recipient file first; the projected event that wakes its UI follows.
1107
- flushDocToDisk(Y, entry);
1524
+ diskOutcome = materializeDocDiskMutation(
1525
+ Y,
1526
+ entry,
1527
+ journaled,
1528
+ diskTransactionTestHooks,
1529
+ );
1108
1530
  },
1109
1531
  event: () => ({
1110
1532
  resource: entry.resource,
@@ -1115,10 +1537,233 @@ function applyCloudDocMutation(input) {
1115
1537
  resourceVersion: Number(input.version),
1116
1538
  }),
1117
1539
  });
1540
+ docDisk.finish(`resource:${envelope.resourceId}`, envelope.mutationId);
1541
+ if (diskOutcome?.superseded || diskOutcome?.successor) {
1542
+ recoverPendingDocMutations(Y, entry);
1543
+ }
1544
+ require('../workspace/tree-cloud').reconcileContentOverlay(envelope.resourceId);
1545
+ return result;
1546
+ }
1547
+
1548
+ function normalizedSnapshotPayload(entry, input, snapshotBase64, checksum) {
1549
+ const envelope = require('./replicas').validateSnapshotEnvelope(input, {
1550
+ contract: 'text-yjs@1',
1551
+ schemaVersion: 1,
1552
+ });
1553
+ return {
1554
+ resourceId: envelope.resourceId,
1555
+ localResource: entry.resource,
1556
+ localPath: entry.path,
1557
+ snapshotBase64,
1558
+ snapshotChecksum: checksum,
1559
+ snapshotVersion: envelope.snapshotVersion,
1560
+ headVersion: envelope.headVersion,
1561
+ authorityEpoch: envelope.authorityEpoch,
1562
+ schemaVersion: envelope.schemaVersion,
1563
+ contract: envelope.contract,
1564
+ journalCursor: isJournalCursor(input?.attachmentJournalCursor)
1565
+ ? input.attachmentJournalCursor
1566
+ : isJournalCursor(input?.journalCursor)
1567
+ ? input.journalCursor
1568
+ : Number(
1569
+ cachedStatement(
1570
+ openLocalDb(),
1571
+ 'SELECT COALESCE(MAX(journal_id), 0) AS cursor FROM mutation_journal',
1572
+ ).get().cursor,
1573
+ ),
1574
+ };
1575
+ }
1576
+
1577
+ function snapshotTargetText(Y, entry, snapshotBase64) {
1578
+ const candidate = new Y.Doc();
1579
+ try {
1580
+ Y.applyUpdate(
1581
+ candidate,
1582
+ Y.encodeStateAsUpdate(entry.doc),
1583
+ { amalgmMutation: true, source: 'snapshot-candidate-base', patch: null },
1584
+ );
1585
+ Y.applyUpdate(
1586
+ candidate,
1587
+ new Uint8Array(Buffer.from(snapshotBase64, 'base64')),
1588
+ { amalgmMutation: true, source: 'snapshot-candidate-official', patch: null },
1589
+ );
1590
+ return candidate.getText(TEXT_KEY).toString();
1591
+ } finally {
1592
+ candidate.destroy();
1593
+ }
1594
+ }
1595
+
1596
+ function commitSnapshotCursor(Y, entry, plan) {
1597
+ const payload = plan.snapshotPayload;
1598
+ const replicas = require('./replicas');
1599
+ const database = openLocalDb();
1600
+ let event = null;
1601
+ let replica = null;
1602
+ database.transaction(() => {
1603
+ const held = docDisk.get(plan.channelId, plan.mutationId, database);
1604
+ if (!held || held.state !== 'crossed') {
1605
+ throw new Error(`snapshot ${payload.resourceId}@${payload.snapshotVersion} did not cross`);
1606
+ }
1607
+ const existing = replicas.getReplicaByResourceId(payload.resourceId, { database });
1608
+ if (existing && existing.localResource !== entry.resource) {
1609
+ throw invalid(`shared resource ${payload.resourceId} is bound to a different local file`, 409);
1610
+ }
1611
+ if (existing) {
1612
+ replica = replicas.advanceReplicaSnapshot({
1613
+ resourceId: payload.resourceId,
1614
+ snapshotVersion: payload.snapshotVersion,
1615
+ snapshotChecksum: payload.snapshotChecksum,
1616
+ authorityEpoch: payload.authorityEpoch,
1617
+ }, { database });
1618
+ } else {
1619
+ replica = replicas.registerReplica({
1620
+ resourceId: payload.resourceId,
1621
+ localResource: entry.resource,
1622
+ localPath: entry.path,
1623
+ contract: payload.contract,
1624
+ schemaVersion: payload.schemaVersion,
1625
+ authorityEpoch: payload.authorityEpoch,
1626
+ appliedVersion: payload.snapshotVersion,
1627
+ snapshotVersion: payload.snapshotVersion,
1628
+ snapshotChecksum: payload.snapshotChecksum,
1629
+ }, { database }).replica;
1630
+ }
1631
+ // Close the attachment fence. Local edits born after the snapshot plan
1632
+ // began were not in the official snapshot; re-home them onto the newly
1633
+ // installed cloud channel without changing their mutation IDs or local
1634
+ // event order.
1635
+ cachedStatement(database, `
1636
+ UPDATE mutation_journal
1637
+ SET channel_id = ?,
1638
+ shared_resource_id = ?,
1639
+ device_id = COALESCE(device_id, ?),
1640
+ authority_epoch = ?,
1641
+ base_cloud_version = ?,
1642
+ updated_at = ?
1643
+ WHERE local_resource = ?
1644
+ AND shared_resource_id IS NULL
1645
+ AND mutation_origin = 'local'
1646
+ AND journal_id > ?
1647
+ `).run(
1648
+ `resource:${replica.resourceId}`,
1649
+ replica.resourceId,
1650
+ require('../workspace/identity').machineId(),
1651
+ replica.authorityEpoch,
1652
+ replica.snapshotVersion,
1653
+ new Date().toISOString(),
1654
+ entry.resource,
1655
+ payload.journalCursor,
1656
+ );
1657
+ if (held.snapshotEventSeq === null) {
1658
+ event = insertStateEvent(database, {
1659
+ resource: entry.resource,
1660
+ op: 'update',
1661
+ id: entry.path,
1662
+ patch: { yjs: payload.snapshotBase64 },
1663
+ mutationId: plan.mutationId,
1664
+ sharedResourceId: payload.resourceId,
1665
+ source: 'doc:cloud-snapshot',
1666
+ resourceVersion: payload.snapshotVersion,
1667
+ });
1668
+ docDisk.markSnapshotCommitted(
1669
+ plan.channelId,
1670
+ plan.mutationId,
1671
+ event.seq,
1672
+ database,
1673
+ );
1674
+ }
1675
+ })();
1676
+ if (event) publishStateEvent(event);
1677
+ return { replica, event };
1678
+ }
1679
+
1680
+ function materializeCloudDocSnapshot(Y, entry, initialPlan) {
1681
+ let plan = initialPlan;
1682
+ const payload = plan.snapshotPayload;
1683
+
1684
+ for (let attempt = 0; attempt < 8; attempt += 1) {
1685
+ const targetText = plan.targetContent
1686
+ ? plan.targetContent.toString('utf8')
1687
+ : snapshotTargetText(Y, entry, payload.snapshotBase64);
1688
+ if (!plan.target) {
1689
+ plan = docDisk.prepare(plan.channelId, plan.mutationId, targetText);
1690
+ } else if (plan.targetContent.toString('utf8') !== targetText) {
1691
+ throw new Error(`snapshot ${payload.resourceId}@${payload.snapshotVersion} changed its target`);
1692
+ }
1693
+ const result = docDisk.cross(plan, openLocalDb(), diskTransactionTestHooks);
1694
+ if (result.status === 'pending') {
1695
+ plan = docDisk.get(plan.channelId, plan.mutationId);
1696
+ if (attempt < 7) continue;
1697
+ throw new Error(
1698
+ `snapshot disk writer is still active: ${entry.path}: ${result.reason || 'pending'}`,
1699
+ );
1700
+ }
1701
+ if (result.status === 'changed') {
1702
+ const newer = docDisk.observe(entry.path);
1703
+ if (newer.text === targetText) {
1704
+ plan = docDisk.rebaseExpected(plan.channelId, plan.mutationId, newer);
1705
+ continue;
1706
+ }
1707
+ // No byte was replaced. Retire this physical attempt, capture the newer
1708
+ // save through the ordinary mutation journal, then derive a new target
1709
+ // for the same official snapshot over that overlay.
1710
+ docDisk.cancel(plan.channelId, plan.mutationId);
1711
+ if (attempt === 7) break;
1712
+ plan = planCloudDocSnapshot(Y, entry, payload);
1713
+ continue;
1714
+ }
1715
+ if (result.status !== 'applied') {
1716
+ throw new Error(`unexpected snapshot disk transaction state: ${result.status}`);
1717
+ }
1718
+
1719
+ const origin = { amalgmMutation: true, source: 'cloud-snapshot', patch: null };
1720
+ Y.applyUpdate(
1721
+ entry.doc,
1722
+ new Uint8Array(Buffer.from(payload.snapshotBase64, 'base64')),
1723
+ origin,
1724
+ );
1725
+ // The native receipt proves targetText was installed at the crossing.
1726
+ // A later external save is causally after the snapshot and is captured by
1727
+ // the immediately following reconcile or the restart seed.
1728
+ setLastDiskText(entry, targetText, 'flush');
1729
+ persistDocState(Y, entry);
1730
+ const committed = commitSnapshotCursor(Y, entry, plan);
1731
+ docDisk.finish(plan.channelId, plan.mutationId);
1732
+ reconcileFromDisk(Y, entry, true);
1733
+ require('../workspace/tree-cloud').reconcileContentOverlay(payload.resourceId);
1734
+ return committed;
1735
+ }
1736
+ throw new Error(`document disk kept changing during snapshot install: ${entry.path}`);
1737
+ }
1738
+
1739
+ function recoverPendingDocSnapshots(Y, entry) {
1740
+ for (let pass = 0; pass < 100; pass += 1) {
1741
+ const plan = docDisk.listSnapshots(entry.resource)[0];
1742
+ if (!plan) return;
1743
+ materializeCloudDocSnapshot(Y, entry, plan);
1744
+ }
1745
+ throw new Error(`snapshot recovery exceeded its finite bound: ${entry.path}`);
1746
+ }
1747
+
1748
+ function planCloudDocSnapshot(Y, entry, payload) {
1749
+ const base = captureDiskBeforeMutation(Y, entry);
1750
+ if (diskTransactionTestHooks.pauseAfterSnapshotCapture > 0) {
1751
+ Atomics.wait(
1752
+ new Int32Array(new SharedArrayBuffer(4)),
1753
+ 0,
1754
+ 0,
1755
+ diskTransactionTestHooks.pauseAfterSnapshotCapture,
1756
+ );
1757
+ }
1758
+ return docDisk.insertSnapshotBase(openLocalDb(), payload, base);
1118
1759
  }
1119
1760
 
1120
1761
  function installCloudDocSnapshot(pathInput, input) {
1121
1762
  const Y = loadYjs();
1763
+ const envelope = require('./replicas').validateSnapshotEnvelope(input, {
1764
+ contract: 'text-yjs@1',
1765
+ schemaVersion: 1,
1766
+ });
1122
1767
  const snapshotBase64 = input?.snapshotBase64;
1123
1768
  if (typeof snapshotBase64 !== 'string' || !snapshotBase64) {
1124
1769
  throw invalid('snapshotBase64 is required');
@@ -1131,8 +1776,9 @@ function installCloudDocSnapshot(pathInput, input) {
1131
1776
  } catch (error) {
1132
1777
  throw invalid(`invalid Yjs snapshot: ${error?.message || error}`);
1133
1778
  }
1134
- const replicas = require('./replicas');
1135
1779
  const entry = loadDocEntry(pathInput);
1780
+ recoverPendingDocSnapshots(Y, entry);
1781
+ const replicas = require('./replicas');
1136
1782
  const existingReplica = replicas.getReplicaByResourceId(input?.resourceId);
1137
1783
  if (existingReplica) {
1138
1784
  // Rehydration: this replica fell behind the pruned official tail, so the
@@ -1142,46 +1788,36 @@ function installCloudDocSnapshot(pathInput, input) {
1142
1788
  if (existingReplica.localResource !== entry.resource) {
1143
1789
  throw invalid(`shared resource ${input?.resourceId} is bound to a different local file`, 409);
1144
1790
  }
1145
- } else if (entry.doc.getText(TEXT_KEY).toString().length > 0) {
1146
- throw invalid('cloud snapshot attachment requires an empty local text file', 409);
1147
- }
1148
- const origin = { amalgmMutation: true, source: 'cloud-snapshot', patch: null };
1149
- Y.applyUpdate(entry.doc, new Uint8Array(bytes), origin);
1150
- persistDocState(Y, entry);
1151
- flushDocToDisk(Y, entry);
1152
- const registered = existingReplica
1153
- ? {
1154
- replica: replicas.advanceReplicaSnapshot({
1155
- resourceId: input?.resourceId,
1156
- snapshotVersion: input?.snapshotVersion,
1791
+ const incomingVersion = envelope.snapshotVersion;
1792
+ if (incomingVersion <= existingReplica.appliedVersion) {
1793
+ const checked = replicas.advanceReplicaSnapshot({
1794
+ resourceId: envelope.resourceId,
1795
+ snapshotVersion: incomingVersion,
1157
1796
  snapshotChecksum: checksum,
1158
- authorityEpoch: input?.authorityEpoch,
1159
- }),
1797
+ authorityEpoch: envelope.authorityEpoch,
1798
+ });
1799
+ if (
1800
+ incomingVersion === existingReplica.snapshotVersion
1801
+ && existingReplica.snapshotChecksum === checksum
1802
+ ) {
1803
+ return { replica: checked, event: null, duplicate: true };
1804
+ }
1805
+ return { replica: checked, event: null, duplicate: true, stale: true };
1160
1806
  }
1161
- : replicas.registerReplica({
1162
- resourceId: input?.resourceId,
1163
- localResource: entry.resource,
1164
- localPath: entry.path,
1165
- contract: input?.contract,
1166
- schemaVersion: input?.schemaVersion,
1167
- authorityEpoch: input?.authorityEpoch,
1168
- appliedVersion: input?.snapshotVersion,
1169
- snapshotVersion: input?.snapshotVersion,
1170
- snapshotChecksum: checksum,
1171
- });
1172
- // File and SQLite binding are current before a mounted UI can observe the
1173
- // snapshot. Applying this full Yjs state is idempotent for an already-open
1174
- // recipient surface.
1175
- const event = appendStateEvent({
1176
- resource: entry.resource,
1177
- op: 'update',
1178
- id: entry.path,
1179
- patch: origin.patch || { yjs: snapshotBase64 },
1180
- sharedResourceId: input.resourceId,
1181
- source: 'doc:cloud-snapshot',
1182
- resourceVersion: Number(input.snapshotVersion),
1183
- });
1184
- return { replica: registered.replica, event };
1807
+ } else if (
1808
+ entry.doc.getText(TEXT_KEY).toString().length > 0
1809
+ && !isJournalCursor(input?.attachmentJournalCursor)
1810
+ ) {
1811
+ throw invalid('cloud snapshot attachment requires an empty local text file', 409);
1812
+ }
1813
+ const payload = normalizedSnapshotPayload(entry, {
1814
+ ...input,
1815
+ ...envelope,
1816
+ contract: input?.contract ?? existingReplica?.contract,
1817
+ schemaVersion: input?.schemaVersion ?? existingReplica?.schemaVersion,
1818
+ }, snapshotBase64, checksum);
1819
+ const plan = planCloudDocSnapshot(Y, entry, payload);
1820
+ return materializeCloudDocSnapshot(Y, entry, plan);
1185
1821
  }
1186
1822
 
1187
1823
  /**
@@ -1216,36 +1852,165 @@ function getDocText(pathInput) {
1216
1852
  * Repoint persisted and open document materializations after tree identity
1217
1853
  * moved. The resource remains doc:<node UUID>; only the local path changes.
1218
1854
  */
1219
- function rebindNodePaths(changes) {
1220
- const database = openLocalDb();
1855
+ function rebindNodePaths(changes, options = {}) {
1856
+ const database = options.database || openLocalDb();
1221
1857
  ensureDocStatesSchema(database);
1222
- for (const change of changes || []) {
1223
- if (!change?.nodeId || !change?.newPath || change.oldPath === change.newPath) continue;
1224
- database.transaction(() => {
1225
- database.prepare('UPDATE doc_states SET path = ? WHERE node_id = ? OR path = ?')
1226
- .run(change.newPath, change.nodeId, change.oldPath);
1227
- database.prepare('UPDATE shared_resource_replicas SET local_path = ?, updated_at = ? WHERE local_resource = ? OR local_path = ?')
1228
- .run(change.newPath, new Date().toISOString(), `${DOC_RESOURCE_PREFIX}${change.nodeId}`, change.oldPath);
1229
- database.prepare('UPDATE shared_resource_promotions SET local_path = ?, updated_at = ? WHERE local_resource = ? OR local_path = ?')
1230
- .run(change.newPath, new Date().toISOString(), `${DOC_RESOURCE_PREFIX}${change.nodeId}`, change.oldPath);
1231
- database.prepare('UPDATE shared_resource_attachments SET local_path = ?, updated_at = ? WHERE local_path = ?')
1232
- .run(change.newPath, new Date().toISOString(), change.oldPath);
1233
- })();
1234
-
1235
- const entry = Array.from(openDocs.values()).find((candidate) => (
1858
+ const token = crypto.randomBytes(16).toString('hex');
1859
+ const nodeIds = new Set();
1860
+ const newPaths = new Set();
1861
+ const normalized = [];
1862
+ for (const [index, value] of Array.from(changes || []).entries()) {
1863
+ if (!value?.nodeId || !value?.newPath) continue;
1864
+ const oldPath = path.resolve(value.oldPath);
1865
+ const newPath = path.resolve(value.newPath);
1866
+ if (oldPath === newPath) continue;
1867
+ if (nodeIds.has(value.nodeId) || newPaths.has(newPath)) {
1868
+ throw new Error('document path rebind is not one-to-one');
1869
+ }
1870
+ nodeIds.add(value.nodeId);
1871
+ newPaths.add(newPath);
1872
+ normalized.push({
1873
+ nodeId: value.nodeId,
1874
+ oldPath,
1875
+ newPath,
1876
+ newResource: `${DOC_RESOURCE_PREFIX}${value.nodeId}`,
1877
+ oldLocalResource: `${DOC_RESOURCE_PREFIX}${
1878
+ fsPrivate().base64UrlEncode(oldPath)
1879
+ }`,
1880
+ temporaryPath: path.join(
1881
+ path.dirname(oldPath),
1882
+ `.amalgm-rebind-${token}-${index}`,
1883
+ ),
1884
+ });
1885
+ }
1886
+ if (normalized.length === 0) return options.deferOpenDocs === true ? () => {} : undefined;
1887
+
1888
+ const currentEntries = Array.from(new Set(openDocs.values()));
1889
+ const assignments = normalized.map((change) => ({
1890
+ change,
1891
+ entry: currentEntries.find((candidate) => (
1236
1892
  candidate.nodeId === change.nodeId || candidate.path === change.oldPath
1237
- ));
1238
- if (!entry) continue;
1239
- unwatchDocFile(entry);
1240
- openDocs.delete(entry.path);
1241
- entry.path = change.newPath;
1242
- entry.nodeId = change.nodeId;
1243
- entry.resource = `${DOC_RESOURCE_PREFIX}${change.nodeId}`;
1244
- entry.pathAliases.add(change.newPath);
1245
- resolvedPathCache.set(change.newPath, change.newPath);
1246
- openDocs.set(change.newPath, entry);
1247
- if (!isInternalStateDoc(change.newPath)) watchDocFile(loadYjs(), entry);
1893
+ )) || null,
1894
+ })).filter((assignment) => assignment.entry);
1895
+ const affectedEntries = new Set(assignments.map((assignment) => assignment.entry));
1896
+ for (const { change } of assignments) {
1897
+ const occupant = currentEntries.find((entry) => entry.path === change.newPath);
1898
+ if (occupant && !affectedEntries.has(occupant)) {
1899
+ throw new Error(`document path rebind destination is already open: ${change.newPath}`);
1900
+ }
1248
1901
  }
1902
+
1903
+ // Every path-bearing table moves through a private temporary namespace
1904
+ // before any final path is installed. Swaps and longer cycles are therefore
1905
+ // one set-wise transition instead of a sequence that can collide with
1906
+ // UNIQUE(local_path) or doc_states' path primary key.
1907
+ database.transaction(() => {
1908
+ const now = new Date().toISOString();
1909
+ for (const change of normalized) {
1910
+ database.prepare('UPDATE doc_states SET path = ? WHERE node_id = ? OR path = ?')
1911
+ .run(change.temporaryPath, change.nodeId, change.oldPath);
1912
+ database.prepare(`
1913
+ UPDATE shared_resource_replicas
1914
+ SET local_resource = ?, local_path = ?, updated_at = ?
1915
+ WHERE resource_id = ? OR local_resource = ? OR local_resource = ? OR local_path = ?
1916
+ `).run(
1917
+ change.newResource,
1918
+ change.temporaryPath,
1919
+ now,
1920
+ change.nodeId,
1921
+ change.oldLocalResource,
1922
+ change.newResource,
1923
+ change.oldPath,
1924
+ );
1925
+ database.prepare(`
1926
+ UPDATE shared_resource_promotions
1927
+ SET local_resource = ?, local_path = ?, updated_at = ?
1928
+ WHERE resource_id = ? OR local_resource = ? OR local_resource = ? OR local_path = ?
1929
+ `).run(
1930
+ change.newResource,
1931
+ change.temporaryPath,
1932
+ now,
1933
+ change.nodeId,
1934
+ change.oldLocalResource,
1935
+ change.newResource,
1936
+ change.oldPath,
1937
+ );
1938
+ database.prepare(`
1939
+ UPDATE shared_resource_attachments
1940
+ SET local_path = ?, updated_at = ?
1941
+ WHERE resource_id = ? OR local_path = ?
1942
+ `).run(change.temporaryPath, now, change.nodeId, change.oldPath);
1943
+ }
1944
+ for (const change of normalized) {
1945
+ database.prepare('UPDATE doc_states SET path = ? WHERE node_id = ? OR path = ?')
1946
+ .run(change.newPath, change.nodeId, change.temporaryPath);
1947
+ database.prepare(`
1948
+ UPDATE mutation_journal
1949
+ SET local_resource = ?, updated_at = ?
1950
+ WHERE local_resource = ? OR local_resource = ?
1951
+ `).run(
1952
+ change.newResource,
1953
+ now,
1954
+ change.oldLocalResource,
1955
+ change.newResource,
1956
+ );
1957
+ docDisk.rebindLocalResource(database, {
1958
+ oldLocalResource: change.oldLocalResource,
1959
+ newLocalResource: change.newResource,
1960
+ newPath: change.newPath,
1961
+ });
1962
+ database.prepare(`
1963
+ UPDATE shared_resource_replicas
1964
+ SET local_resource = ?, local_path = ?, updated_at = ?
1965
+ WHERE resource_id = ? OR local_resource = ? OR local_path = ?
1966
+ `).run(
1967
+ change.newResource,
1968
+ change.newPath,
1969
+ now,
1970
+ change.nodeId,
1971
+ change.newResource,
1972
+ change.temporaryPath,
1973
+ );
1974
+ database.prepare(`
1975
+ UPDATE shared_resource_promotions
1976
+ SET local_resource = ?, local_path = ?, updated_at = ?
1977
+ WHERE resource_id = ? OR local_resource = ? OR local_path = ?
1978
+ `).run(
1979
+ change.newResource,
1980
+ change.newPath,
1981
+ now,
1982
+ change.nodeId,
1983
+ change.newResource,
1984
+ change.temporaryPath,
1985
+ );
1986
+ database.prepare(`
1987
+ UPDATE shared_resource_attachments
1988
+ SET local_path = ?, updated_at = ?
1989
+ WHERE resource_id = ? OR local_path = ?
1990
+ `).run(change.newPath, now, change.nodeId, change.temporaryPath);
1991
+ }
1992
+ })();
1993
+
1994
+ const finishOpenDocs = () => {
1995
+ for (const entry of affectedEntries) unwatchDocFile(entry);
1996
+ for (const [key, entry] of Array.from(openDocs.entries())) {
1997
+ if (affectedEntries.has(entry)) openDocs.delete(key);
1998
+ }
1999
+ for (const { change, entry } of assignments) {
2000
+ entry.path = change.newPath;
2001
+ entry.nodeId = change.nodeId;
2002
+ entry.resource = change.newResource;
2003
+ entry.pathAliases.add(change.newPath);
2004
+ resolvedPathCache.set(change.newPath, change.newPath);
2005
+ openDocs.set(change.newPath, entry);
2006
+ }
2007
+ for (const entry of affectedEntries) {
2008
+ if (!isInternalStateDoc(entry.path)) watchDocFile(loadYjs(), entry);
2009
+ }
2010
+ };
2011
+ if (options.deferOpenDocs === true) return finishOpenDocs;
2012
+ finishOpenDocs();
2013
+ return undefined;
1249
2014
  }
1250
2015
 
1251
2016
  function closeDocByNode(nodeId) {
@@ -1266,7 +2031,28 @@ function closeDocsByNodeIds(nodeIds) {
1266
2031
  return closed;
1267
2032
  }
1268
2033
 
1269
- /** Test/shutdown helper: flush and close every open document. */
2034
+ function captureDocsByNodeIds(nodeIds, options = {}) {
2035
+ const ids = Array.from(nodeIds instanceof Set ? nodeIds : new Set(nodeIds || []));
2036
+ if (!ids.length) return 0;
2037
+ const database = options.database || openLocalDb();
2038
+ const placeholders = ids.map(() => '?').join(',');
2039
+ const replicas = database.prepare(`
2040
+ SELECT resource_id, local_path
2041
+ FROM shared_resource_replicas
2042
+ WHERE resource_id IN (${placeholders})
2043
+ AND contract = 'text-yjs@1'
2044
+ AND access_status = 'active'
2045
+ `).all(...ids);
2046
+ if (!replicas.length) return 0;
2047
+ const Y = loadYjs();
2048
+ for (const replica of replicas) {
2049
+ const entry = loadDocEntry(replica.local_path);
2050
+ captureDiskBeforeMutation(Y, entry);
2051
+ }
2052
+ return replicas.length;
2053
+ }
2054
+
2055
+ /** Test/shutdown helper: capture final disk observations and close every open document. */
1270
2056
  function closeAllDocs() {
1271
2057
  let Y;
1272
2058
  try {
@@ -1282,6 +2068,7 @@ module.exports = {
1282
2068
  applyCloudDocMutation,
1283
2069
  applyDocMutations,
1284
2070
  applyDocUpdates,
2071
+ captureDocsByNodeIds,
1285
2072
  closeDocByNode,
1286
2073
  closeDocsByNodeIds,
1287
2074
  closeAllDocs,
@@ -1294,5 +2081,18 @@ module.exports = {
1294
2081
  rebindNodePaths,
1295
2082
  writeDocText,
1296
2083
  // Test hooks only — not part of the module contract.
1297
- _private: { isInternalStateDoc, openDocs },
2084
+ _private: {
2085
+ isInternalStateDoc,
2086
+ openDocs,
2087
+ setDiskTransactionTestHooks(hooks = {}) {
2088
+ diskTransactionTestHooks = { ...hooks };
2089
+ },
2090
+ verifyOpenDocs,
2091
+ watchVerificationStats() {
2092
+ return {
2093
+ active: watchVerificationTimer !== null,
2094
+ passes: watchVerificationPasses,
2095
+ };
2096
+ },
2097
+ },
1298
2098
  };