amalgm 0.1.238 → 0.1.240

package/runtime/scripts/amalgm-mcp/tests/state-docs.test.js

@@ -2,9 +2,11 @@
 
 const test = require('node:test');
 const assert = require('node:assert/strict');
+const crypto = require('node:crypto');
 const fs = require('fs');
 const os = require('os');
 const path = require('path');
+const { spawn } = require('node:child_process');
 
 const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'amalgm-live-docs-test-'));
 process.env.AMALGM_DIR = path.join(tempRoot, '.amalgm');
@@ -28,6 +30,19 @@ function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
+function sha256(text) {
+  return crypto.createHash('sha256').update(text, 'utf8').digest('hex');
+}
+
+function writeFromAnotherProcessAfter(file, text, delayMs) {
+  return spawn(process.execPath, [
+    '-e',
+    'const fs=require("fs");'
+      + `setTimeout(()=>fs.writeFileSync(${JSON.stringify(file)},`
+      + `${JSON.stringify(text)},"utf8"),${delayMs});`,
+  ], { stdio: 'ignore' });
+}
+
 function clientFromState(stateBase64) {
   const doc = new Y.Doc();
   Y.applyUpdate(doc, new Uint8Array(Buffer.from(stateBase64, 'base64')));
@@ -70,7 +85,7 @@ test('open seeds the document from disk and returns usable state', () => {
   assert.equal(client.getText('content').toString(), '# hello\n');
 });
 
-test('client updates apply, log as patch events, and flush back to disk', async () => {
+test('client updates apply, log as patch events, and materialize to disk', async () => {
   const target = writeDoc('edit.md', 'hello world\n');
   const opened = docs.openDoc(target);
   const client = clientFromState(opened.state);
@@ -89,7 +104,7 @@ test('client updates apply, log as patch events, and flush back to disk', async
   assert.ok(events[0].patch?.yjs);
   assert.equal(events[0].value, undefined);
 
-  // Debounced write-back lands on disk.
+  // The saved-local response already stands on a crossed file transaction.
   await sleep(500);
   assert.equal(fs.readFileSync(target, 'utf8'), 'hello, world\n');
 });
@@ -119,14 +134,14 @@ test('two clients editing concurrently converge to the same text', () => {
   assert.equal(clientB.getText('content').toString(), engineText);
 });
 
-test('external disk edits reconcile into the live doc via the watcher', async () => {
+test('bounded verification reconciles a dropped external disk notification', () => {
   const target = writeDoc('agent.md', 'agent line 1\n');
   const opened = docs.openDoc(target);
   const before = currentSeq();
 
   // An "agent" writes the file directly, bypassing the doc API.
   fs.writeFileSync(target, 'agent line 1\nagent line 2\n', 'utf8');
-  await sleep(700);
+  docs._private.verifyOpenDocs();
 
   assert.equal(docs.getDocText(target), 'agent line 1\nagent line 2\n');
   const events = listEventsAfter(before).filter((e) => e.resource === opened.resource);
@@ -134,6 +149,243 @@ test('external disk edits reconcile into the live doc via the watcher', async ()
   assert.equal(events[events.length - 1].source, 'doc:disk');
 });
 
+test('a dropped watcher cannot let a later versioned API write erase the disk save', () => {
+  const base = 'left\nright\n';
+  const target = writeDoc('dropped-watcher-api.md', base);
+  docs.openDoc(target);
+
+  fs.writeFileSync(target, 'left from disk\nright\n', 'utf8');
+  const written = docs.writeDocText(target, 'left\nright from api\n', {
+    baseHash: sha256(base),
+    mutationId: 'dropped_watcher_api_001',
+  });
+
+  assert.equal(written.merge, 'merged');
+  assert.equal(docs.getDocText(target), 'left from disk\nright from api\n');
+  assert.equal(fs.readFileSync(target, 'utf8'), 'left from disk\nright from api\n');
+});
+
+test('the final native CAS retains a save that lands during an API materialization', () => {
+  const base = 'first\nsecond\n';
+  const target = writeDoc('api-final-cas.md', base);
+  docs.openDoc(target);
+  const child = writeFromAnotherProcessAfter(
+    target,
+    'first from disk\nsecond\n',
+    75,
+  );
+  docs._private.setDiskTransactionTestHooks({ pauseBeforeInstall: 350 });
+  try {
+    docs.writeDocText(target, 'first\nsecond from api\n', {
+      baseHash: sha256(base),
+      mutationId: 'api_final_cas_race_001',
+    });
+  } finally {
+    docs._private.setDiskTransactionTestHooks();
+  }
+  child.unref();
+  assert.equal(docs.getDocText(target), 'first from disk\nsecond from api\n');
+  assert.equal(fs.readFileSync(target, 'utf8'), 'first from disk\nsecond from api\n');
+});
+
+test('the API plan stays pinned when disk changes after merge derivation', () => {
+  const base = 'first\nsecond\n';
+  const target = writeDoc('api-derived-base-cas.md', base);
+  docs.openDoc(target);
+  const child = writeFromAnotherProcessAfter(
+    target,
+    'first from disk\nsecond\n',
+    75,
+  );
+  docs._private.setDiskTransactionTestHooks({ pauseAfterEditBeforeJournal: 350 });
+  let written;
+  try {
+    written = docs.writeDocText(target, 'first\nsecond from api\n', {
+      baseHash: sha256(base),
+      mutationId: 'api_derived_base_cas_001',
+    });
+  } finally {
+    docs._private.setDiskTransactionTestHooks();
+  }
+  child.unref();
+  assert.equal(docs.getDocText(target), 'first from disk\nsecond from api\n');
+  assert.equal(fs.readFileSync(target, 'utf8'), 'first from disk\nsecond from api\n');
+  assert.equal(written.sha256, sha256('first from disk\nsecond from api\n'));
+});
+
+test('a save after native install becomes the successor during receipt recovery', () => {
+  const base = 'first\nsecond\n';
+  const target = writeDoc('api-post-install-successor.md', base);
+  docs.openDoc(target);
+
+  docs._private.setDiskTransactionTestHooks({ crashAfterInstall: true });
+  try {
+    assert.throws(
+      () => docs.writeDocText(target, 'first\nsecond from api\n', {
+        baseHash: sha256(base),
+        mutationId: 'api_post_install_successor_001',
+      }),
+      /native file transaction failed/,
+    );
+  } finally {
+    docs._private.setDiskTransactionTestHooks();
+  }
+  assert.equal(fs.readFileSync(target, 'utf8'), 'first\nsecond from api\n');
+
+  // The helper's receipt proves the API target crossed. This later save is B,
+  // not evidence that A failed, and must become a durable successor.
+  fs.writeFileSync(target, 'first from disk\nsecond\n', 'utf8');
+  docs._private.verifyOpenDocs();
+
+  assert.equal(docs.getDocText(target), 'first from disk\nsecond from api\n');
+  assert.equal(fs.readFileSync(target, 'utf8'), 'first from disk\nsecond from api\n');
+});
+
+test('UUID ownership recovers a crossed edit after its parent folder is renamed', () => {
+  const beforeDir = path.join(docDir, 'rename-before');
+  const afterDir = path.join(docDir, 'rename-after');
+  fs.mkdirSync(beforeDir);
+  const beforePath = path.join(beforeDir, 'owned.md');
+  const afterPath = path.join(afterDir, 'owned.md');
+  fs.writeFileSync(beforePath, 'before\n', 'utf8');
+  docs.openDoc(beforePath);
+
+  docs._private.setDiskTransactionTestHooks({ crashAfterInstall: true });
+  try {
+    assert.throws(
+      () => docs.writeDocText(beforePath, 'after API\n', {
+        baseHash: sha256('before\n'),
+        mutationId: 'uuid_parent_rename_recovery_001',
+      }),
+      /native file transaction failed/,
+    );
+  } finally {
+    docs._private.setDiskTransactionTestHooks();
+  }
+
+  fs.renameSync(beforeDir, afterDir);
+  docs.rebindNodePaths([{
+    nodeId: '11111111-1111-4111-8111-111111111111',
+    oldPath: beforePath,
+    newPath: afterPath,
+  }]);
+  docs._private.verifyOpenDocs();
+
+  assert.equal(docs.getDocText(afterPath), 'after API\n');
+  assert.equal(fs.readFileSync(afterPath, 'utf8'), 'after API\n');
+});
+
+test('the bounded verifier schedule reconciles a file whose watcher failed', async () => {
+  const isolatedDir = path.join(docDir, 'scheduled-verifier');
+  fs.mkdirSync(isolatedDir, { recursive: true });
+  const target = path.join(isolatedDir, 'agent.md');
+  fs.writeFileSync(target, 'before\n', 'utf8');
+  const originalWatch = fs.watch;
+  let watchAttempts = 0;
+  fs.watch = (...args) => {
+    watchAttempts += 1;
+    if (watchAttempts === 1) {
+      const error = new Error('simulated dropped watcher');
+      error.code = 'EMFILE';
+      throw error;
+    }
+    return originalWatch.apply(fs, args);
+  };
+
+  try {
+    const scheduledBefore = docs._private.watchVerificationStats();
+    const opened = docs.openDoc(target);
+    const before = currentSeq();
+    fs.writeFileSync(target, 'after\n', 'utf8');
+
+    const deadline = Date.now() + 7_000;
+    while (docs.getDocText(target) !== 'after\n' && Date.now() < deadline) {
+      await sleep(50);
+    }
+    assert.equal(
+      docs.getDocText(target),
+      'after\n',
+      JSON.stringify({
+        scheduledBefore,
+        scheduledAfter: docs._private.watchVerificationStats(),
+        watchAttempts,
+        disk: fs.readFileSync(target, 'utf8'),
+      }),
+    );
+    const events = listEventsAfter(before).filter((event) => event.resource === opened.resource);
+    assert.equal(events.length, 1);
+    assert.equal(events[0].source, 'doc:disk');
+    assert.ok(watchAttempts >= 2, 'the scheduled pass retried the failed watcher');
+    const scheduledAfter = docs._private.watchVerificationStats();
+    assert.equal(scheduledAfter.active, true);
+    assert.ok(scheduledAfter.passes > scheduledBefore.passes);
+  } finally {
+    fs.watch = originalWatch;
+  }
+});
+
+test('a transient journal failure leaves the exact disk observation retryable', () => {
+  const target = writeDoc('agent-retry.md', 'before\n');
+  const opened = docs.openDoc(target);
+  const before = currentSeq();
+  const mutations = require('../state/mutations');
+  const commitLocalMutation = mutations.commitLocalMutation;
+  fs.writeFileSync(target, 'after\n', 'utf8');
+  mutations.commitLocalMutation = () => {
+    throw new Error('simulated journal outage');
+  };
+  try {
+    docs._private.verifyOpenDocs();
+  } finally {
+    mutations.commitLocalMutation = commitLocalMutation;
+  }
+
+  assert.equal(docs.getDocText(target), 'before\n');
+  assert.equal(
+    listEventsAfter(before).filter((event) => event.resource === opened.resource).length,
+    0,
+  );
+
+  docs._private.verifyOpenDocs();
+  assert.equal(docs.getDocText(target), 'after\n');
+  assert.equal(fs.readFileSync(target, 'utf8'), 'after\n');
+  const events = listEventsAfter(before).filter((event) => event.resource === opened.resource);
+  assert.equal(events.length, 1);
+  assert.ok(events.every((event) => event.source === 'doc:disk'));
+});
+
+test('a transient journal failure cannot consume a whole-text API write', () => {
+  const target = writeDoc('api-retry.md', 'before\n');
+  const opened = docs.openDoc(target);
+  const before = currentSeq();
+  const mutations = require('../state/mutations');
+  const commitLocalMutation = mutations.commitLocalMutation;
+  mutations.commitLocalMutation = () => {
+    throw new Error('simulated journal outage');
+  };
+  try {
+    assert.throws(
+      () => docs.writeDocText(target, 'after\n', { mutationId: 'api_retry_write_0001' }),
+      /simulated journal outage/,
+    );
+  } finally {
+    mutations.commitLocalMutation = commitLocalMutation;
+  }
+
+  assert.equal(docs.getDocText(target), 'before\n');
+  assert.equal(fs.readFileSync(target, 'utf8'), 'before\n');
+  assert.equal(
+    listEventsAfter(before).filter((event) => event.resource === opened.resource).length,
+    0,
+  );
+
+  docs.writeDocText(target, 'after\n', { mutationId: 'api_retry_write_0001' });
+  assert.equal(docs.getDocText(target), 'after\n');
+  const events = listEventsAfter(before).filter((event) => event.resource === opened.resource);
+  assert.equal(events.length, 1);
+  assert.equal(events[0].source, 'doc:write');
+});
+
 test('internal protocol documents never treat their own disk flush as another writer', async () => {
   const checkpointDir = path.join(process.env.AMALGM_DIR, 'state', 'repo-checkpoints');
   const projectDir = path.join(process.env.AMALGM_DIR, 'state', 'code-projects');
@@ -185,7 +437,7 @@ test('documents survive close: persisted state + disk agree on reopen', () => {
   const update = encodeLocalEdit(client, (ytext) => ytext.insert(0, 'v2 over '));
   docs.applyDocUpdates(target, [update]);
 
-  docs.closeAllDocs(); // flushes disk + state
+  docs.closeAllDocs(); // captures final disk observations and releases memory
 
   const reopened = docs.openDoc(target);
   const fresh = clientFromState(reopened.state);
@@ -224,8 +476,33 @@ test('guards: binary, missing, oversized, and malformed inputs are rejected', ()
   assert.throws(() => docs.applyDocUpdates(target, ['aGVsbG8=']), /invalid Yjs update/);
 });
 
+test('all managed document writes cross the one native materialization boundary', () => {
+  const source = fs.readFileSync(require.resolve('../state/docs'), 'utf8');
+  const rawWriteTargets = Array.from(
+    source.matchAll(/fs\.writeFileSync\(\s*([^,\n]+)/g),
+    (match) => match[1].trim(),
+  );
+
+  assert.deepEqual(
+    rawWriteTargets,
+    ['sidecarPath'],
+    'only the independent metadata sidecar may bypass document materialization',
+  );
+  assert.doesNotMatch(source, /\bflushDocToDisk\b|\bwriteFileAtomic\b|fs\.renameSync\(/);
+  assert.match(source, /docDisk\.cross\(/);
+  assert.equal(
+    (source.match(/docDisk\.insertSnapshotBase\(/g) || []).length,
+    1,
+    'every snapshot attempt must acquire its physical base through one capture-and-pin helper',
+  );
+  assert.match(
+    source,
+    /function planCloudDocSnapshot[\s\S]*?captureDiskBeforeMutation[\s\S]*?docDisk\.insertSnapshotBase/,
+  );
+});
+
 // ---------------------------------------------------------------------------
-// Durability: a crash (SIGKILL — no flush, no graceful close) must never lose
+// Durability: a crash (SIGKILL — no graceful close) must never lose
 // an acknowledged edit. The contract is per acknowledgment, not per update:
 // state is persisted before applyDocUpdates returns (before the POST response
 // acknowledges); the recorded last-disk hash lets reopen tell "disk is merely
@@ -234,20 +511,20 @@ test('guards: binary, missing, oversized, and malformed inputs are rejected', ()
 
 function crashAndReloadDocs() {
   // Simulate SIGKILL: drop the module (and its open docs, timers untouched)
-  // without any flush. The reloaded module sees only what was persisted.
+  // without graceful cleanup. The reloaded module sees only durable state.
   delete require.cache[require.resolve('../state/docs')];
   return require('../state/docs');
 }
 
-test('a kill inside the disk-write debounce loses nothing: reopen restores the edit', async () => {
+test('an acknowledged edit is on disk before a kill and remains current on reopen', () => {
   const target = writeDoc('crash.md', 'before the crash\n');
   const opened = docs.openDoc(target);
   const client = clientFromState(opened.state);
   const update = encodeLocalEdit(client, (ytext) => ytext.insert(0, 'acknowledged edit / '));
   docs.applyDocUpdates(target, [update]);
 
-  // Crash immediately — the 300ms disk write has NOT run; disk is stale.
-  assert.equal(fs.readFileSync(target, 'utf8'), 'before the crash\n');
+  // Acknowledgment is earned only after the native file transaction crossed.
+  assert.equal(fs.readFileSync(target, 'utf8'), 'acknowledged edit / before the crash\n');
   const reloaded = crashAndReloadDocs();
 
   const reopened = reloaded.openDoc(target);
@@ -258,8 +535,7 @@ test('a kill inside the disk-write debounce loses nothing: reopen restores the e
     'persisted state must win over a disk that is merely behind',
   );
 
-  // The stale disk artifact catches up via the rescheduled write-back.
-  await sleep(500);
+  // No timer or later process lifetime is needed to make the save real.
   assert.equal(fs.readFileSync(target, 'utf8'), 'acknowledged edit / before the crash\n');
   reloaded.closeAllDocs();
 });
@@ -277,7 +553,7 @@ test('a multi-update batch is fully persisted by the time it returns', () => {
   const result = docs.applyDocUpdates(target, updates);
   assert.equal(result.applied, 3);
 
-  // Crash with no flush: the batch persist alone must carry all three
+  // Crash with no graceful close: the journal and crossed files carry all three
   // updates into doc_states.
   const reloaded = crashAndReloadDocs();
   const reopened = reloaded.openDoc(target);
@@ -306,3 +582,90 @@ test('an external edit made while closed still folds in on reopen', () => {
     'a genuinely changed disk is folded in, not overwritten',
   );
 });
+
+test('restart reuses a crossed successor instead of inventing another mutation', () => {
+  const base = 'first\nsecond\n';
+  const target = writeDoc('crossed-successor-restart.md', base);
+  const opened = docs.openDoc(target);
+  const before = currentSeq();
+  const mutations = require('../state/mutations');
+  const baselineMutationIds = new Set(
+    mutations.listMutations({ localResource: opened.resource })
+      .map((mutation) => mutation.mutationId),
+  );
+
+  docs._private.setDiskTransactionTestHooks({ crashAfterInstall: true });
+  try {
+    assert.throws(
+      () => docs.writeDocText(target, 'first\nsecond from api\n', {
+        baseHash: sha256(base),
+        mutationId: 'restart_crossed_original_001',
+      }),
+      /native file transaction failed/,
+    );
+  } finally {
+    docs._private.setDiskTransactionTestHooks();
+  }
+
+  fs.writeFileSync(target, 'first from disk\nsecond\n', 'utf8');
+  const completeMaterializedMutation = mutations.completeMaterializedMutation;
+  let interrupted = false;
+  mutations.completeMaterializedMutation = (...args) => {
+    if (!interrupted) {
+      interrupted = true;
+      throw new Error('simulated process death before predecessor completion');
+    }
+    return completeMaterializedMutation(...args);
+  };
+  try {
+    assert.throws(
+      () => docs.writeDocText(target, 'unreached later API\n'),
+      /simulated process death before predecessor completion/,
+    );
+  } finally {
+    mutations.completeMaterializedMutation = completeMaterializedMutation;
+  }
+
+  const beforeRestart = mutations.listMutations({ localResource: opened.resource });
+  assert.equal(
+    beforeRestart.length,
+    baselineMutationIds.size + 2,
+    `one API edit and one disk successor are durable: ${JSON.stringify(
+      beforeRestart.map((mutation) => ({
+        mutationId: mutation.mutationId,
+        state: mutation.state,
+        operation: mutation.operation,
+      })),
+    )}`,
+  );
+  assert.ok(
+    beforeRestart
+      .filter((mutation) => !baselineMutationIds.has(mutation.mutationId))
+      .every((mutation) => mutation.state === 'saving-local'),
+  );
+
+  const reloaded = crashAndReloadDocs();
+  const reopened = reloaded.openDoc(target);
+  const finalText = fs.readFileSync(target, 'utf8');
+  assert.match(finalText, /^first from disk\n/);
+  assert.match(finalText, /second from api\n/);
+  assert.equal(clientFromState(reopened.state).getText('content').toString(), finalText);
+
+  const afterRestart = mutations.listMutations({ localResource: opened.resource });
+  assert.equal(
+    afterRestart.length,
+    baselineMutationIds.size + 2,
+    'restart must not assign the successor a second ID',
+  );
+  assert.ok(
+    afterRestart
+      .filter((mutation) => !baselineMutationIds.has(mutation.mutationId))
+      .every((mutation) => mutation.state === 'saved-local'),
+  );
+  assert.equal(
+    listEventsAfter(before).filter((event) => event.resource === opened.resource).length,
+    2,
+    'one real API edit plus one real disk save produce exactly two events',
+  );
+  reloaded.closeAllDocs();
+});