amalgm 0.1.152 → 0.1.153

package/runtime/scripts/amalgm-mcp/state/docs.js

@@ -29,13 +29,24 @@
  * when disk content changed past the last text we read or wrote
  * (entry.lastDiskText) — our own write-backs and stale watcher events can
  * never splice, so they can never re-emit.
+ *
+ * When the doc has ALSO diverged from lastDiskText (live edits inside the
+ * write-back debounce window), splicing disk text wholesale would delete
+ * those edits. Instead the disk change three-way merges against
+ * lastDiskText as the base (see ./merge3): non-overlapping hunks from both
+ * sides land automatically; overlapping hunks keep the live doc's text, and
+ * the full incoming disk text is preserved in a `<name>.conflicted-<stamp>`
+ * sidecar next to the file, with a `conflict` field on the emitted doc
+ * event. The next write-back then puts the merged text on disk — the
+ * incoming version it replaces lives on in the sidecar.
  */
 
 const crypto = require('crypto');
 const fs = require('fs');
 const path = require('path');
-const { cachedStatement, openLocalDb } = require('./db');
+const { cachedStatement, ensureDocStatesSchema, openLocalDb } = require('./db');
 const { appendStateEvent, currentSeq } = require('./events');
+const { merge3 } = require('./merge3');
 
 const DOC_RESOURCE_PREFIX = 'doc:';
 const TEXT_KEY = 'content';
@@ -45,6 +56,27 @@ const DISK_WRITE_DEBOUNCE_MS = 300;
 const WATCH_RECONCILE_DEBOUNCE_MS = 200;
 const MAX_UPDATES_PER_CALL = 200;
 
+// Bounds on the per-doc disk-sync history ring (see pushDiskHistory): a small
+// window of recent {text, sha256, seq, origin} states that versioned writers
+// resolve their declared base against, and that the unversioned-overwrite
+// guard uses to tell "live edits since the last disk state" from "the agent
+// just rewrote its own file". In-memory only; dies with the doc entry.
+const MAX_HISTORY_ENTRIES = 8;
+const MAX_HISTORY_BYTES = 4 * 1024 * 1024;
+const MAX_HISTORY_AGE_MS = 10 * 60 * 1000;
+
+// Example payloads embedded in validation errors (example-first: the fastest
+// way to tell a writer what shape the call wants is to show a correct one).
+const WRITE_EXAMPLE = '{"path": "notes.md", "text": "full new file contents", '
+  + '"baseHash": "<sha256 hex of the exact text you read>"}';
+
+// Conflict sidecars: `notes.conflicted-20260713-104501.md` next to `notes.md`
+// (a `-2`, `-3`… suffix on the stamp if the second collides). The pattern is
+// what keeps them inert — the directory watcher skips matching names and
+// resolveDocPath refuses to open them, so a sidecar can never become a live
+// doc or feed the reconcile loop.
+const SIDECAR_NAME_RE = /\.conflicted-\d{8}-\d{6}(?:-\d+)?(?:\.[^./\\]*)?$/;
+
 // resolved path -> { doc, resource, reconcileTimer, diskWriteTimer,
 //                    lastAccess, lastDiskText, lastDiskHash, pathAliases }
 const openDocs = new Map();
@@ -80,36 +112,65 @@ function invalid(message, statusCode = 400) {
   return Object.assign(new Error(message), { statusCode });
 }
 
-function ensureSchema(database) {
-  database.exec(`
-    CREATE TABLE IF NOT EXISTS doc_states (
-      path TEXT PRIMARY KEY,
-      state BLOB NOT NULL,
-      updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
-    );
-  `);
-  // last_disk_hash records the disk content as we last read or wrote it.
-  // On reopen it distinguishes "disk changed externally" (fold it in) from
-  // "disk is merely behind our persisted state" (write the doc back out) —
-  // without it, a restart would revert edits that hadn't hit the debounced
-  // disk write yet.
-  try {
-    database.exec('ALTER TABLE doc_states ADD COLUMN last_disk_hash TEXT');
-  } catch {
-    // Column already exists.
-  }
-}
-
 function hashText(text) {
   return crypto.createHash('sha256').update(text, 'utf8').digest('hex');
 }
 
 // The only way lastDiskText is ever assigned: the hash is what persistence
 // writes, and hashing at assignment (rare: open, reconcile, flush) keeps the
-// full-text SHA-256 off the per-batch persist path.
-function setLastDiskText(entry, text) {
+// full-text SHA-256 off the per-batch persist path. `origin` says how the
+// text got onto disk — 'disk' (external write folded in), 'flush' (the live
+// doc's own write-back), 'open' (seed of an unchanged file) — and is what the
+// overwrite guard reads: a 'flush' head means the last disk state was
+// produced through the live channel.
+function setLastDiskText(entry, text, origin) {
   entry.lastDiskText = text;
   entry.lastDiskHash = text == null ? null : hashText(text);
+  if (text != null) pushDiskHistory(entry, text, entry.lastDiskHash, origin || 'disk');
+}
+
+// Retain the new disk-sync state in the entry's bounded ring. `seq` is the
+// event-log position at record time so versioned writers can name a base by
+// the seq they observed. Identical text refreshes the head (seq/timestamp)
+// instead of duplicating it — which also guarantees a 'flush' head really
+// does differ from the previous entry (i.e. live edits happened).
+function pushDiskHistory(entry, text, sha256, origin) {
+  const history = entry.diskHistory;
+  const head = history[history.length - 1];
+  if (head && head.sha256 === sha256) {
+    head.seq = currentSeq();
+    head.at = Date.now();
+    return;
+  }
+  history.push({ text, sha256, seq: currentSeq(), at: Date.now(), origin });
+  const cutoff = Date.now() - MAX_HISTORY_AGE_MS;
+  let bytes = 0;
+  for (const item of history) bytes += Buffer.byteLength(item.text, 'utf8');
+  // The head (current state) is never trimmed; older entries fall off by
+  // count, age, or total bytes.
+  while (history.length > 1 && (
+    history.length > MAX_HISTORY_ENTRIES
+    || history[0].at < cutoff
+    || bytes > MAX_HISTORY_BYTES
+  )) {
+    bytes -= Buffer.byteLength(history[0].text, 'utf8');
+    history.shift();
+  }
+}
+
+// True when the current head disk state was written by the live channel
+// (flush) — meaning edits arrived through Yjs since the previous disk state,
+// and wholesale-replacing that text would destroy them. A 'disk'/'open' head
+// means disk itself produced the last state: nothing live is at risk and
+// overwrites stay silent (the common agent-rewrites-its-own-file case).
+function liveEditsSinceLastDiskState(entry) {
+  const history = entry.diskHistory;
+  const head = history[history.length - 1];
+  if (!head || head.origin !== 'flush') return false;
+  const previous = history[history.length - 2];
+  // pushDiskHistory dedupes identical text, so a 'flush' head implies a real
+  // live-channel change; the comparison is belt-and-braces for a trimmed ring.
+  return !previous || previous.text !== head.text;
 }
 
 function docResourceName(absolutePath) {
@@ -127,6 +188,9 @@ function pathFromDocResource(resource) {
 
 function resolveDocPath(pathInput) {
   const resolved = fsPrivate().resolveSafePathSync(pathInput);
+  if (isConflictSidecar(path.basename(resolved))) {
+    throw invalid('conflict sidecar files cannot be opened as live documents');
+  }
   const { isText } = fsPrivate().classifyFile(resolved);
   if (!isText) {
     throw invalid(`live documents support text files only (got ${path.extname(resolved) || 'no extension'})`);
@@ -205,25 +269,120 @@ function flushDocToDisk(Y, entry) {
     const text = entry.doc.getText(TEXT_KEY).toString();
     const onDisk = fs.readFileSync(entry.path, 'utf8');
     if (onDisk !== text) fs.writeFileSync(entry.path, text, 'utf8');
-    setLastDiskText(entry, text);
+    setLastDiskText(entry, text, 'flush');
     persistDocState(Y, entry);
   } catch (error) {
     console.warn(`[LiveDoc] Failed to flush "${entry.path}" to disk:`, error?.message || error);
   }
 }
 
+function isConflictSidecar(name) {
+  return SIDECAR_NAME_RE.test(name);
+}
+
+function conflictSidecarPath(docPath) {
+  const dir = path.dirname(docPath);
+  const ext = path.extname(docPath);
+  const stem = path.basename(docPath, ext);
+  const now = new Date();
+  const pad = (value) => String(value).padStart(2, '0');
+  const stamp = `${now.getFullYear()}${pad(now.getMonth() + 1)}${pad(now.getDate())}`
+    + `-${pad(now.getHours())}${pad(now.getMinutes())}${pad(now.getSeconds())}`;
+  let candidate = path.join(dir, `${stem}.conflicted-${stamp}${ext}`);
+  for (let n = 2; fs.existsSync(candidate); n += 1) {
+    candidate = path.join(dir, `${stem}.conflicted-${stamp}-${n}${ext}`);
+  }
+  return candidate;
+}
+
+// Preserve a losing version of the document next to the file. Throws on
+// failure — every caller then aborts before applying, which is the safe
+// posture: nothing is replaced until its predecessor is durably preserved.
+// Applying without the sidecar would be the silent-loss bug.
+function writePreservationSidecar(entry, text, why) {
+  const sidecarPath = conflictSidecarPath(entry.path);
+  fs.writeFileSync(sidecarPath, text, 'utf8');
+  console.warn(`[LiveDoc] CONFLICT path="${entry.path}" sidecar="${sidecarPath}" — ${why}`);
+  return sidecarPath;
+}
+
+// Overlapping-hunk merge: the merged text keeps OURS for conflicted regions,
+// so THEIRS (the full incoming text) is what needs preserving.
+function writeConflictSidecar(entry, theirsText, hunkCount) {
+  return writePreservationSidecar(
+    entry,
+    theirsText,
+    `${hunkCount} overlapping hunk(s); conflicted regions keep the live doc, `
+    + 'full incoming text preserved in the sidecar',
+  );
+}
+
 function reconcileFromDisk(Y, entry) {
   try {
     const onDisk = fs.readFileSync(entry.path, 'utf8');
     // Only fold in disk content that actually CHANGED since we last read or
     // wrote it. Without this, a watcher event firing while the doc is ahead
     // of disk (write-back still debounced) would "reconcile" the stale disk
-    // text back over live edits.
+    // text back over live edits. (This is also the THEIRS === BASE no-op.)
     if (onDisk === entry.lastDiskText) return;
-    setLastDiskText(entry, onDisk);
     const ytext = entry.doc.getText(TEXT_KEY);
+    const ours = ytext.toString();
+    const base = entry.lastDiskText;
+    let nextText = onDisk;
+    // OURS === BASE (no live divergence) or no base at all (first seed):
+    // disk text applies wholesale, exactly as before. Otherwise both sides
+    // moved since the last sync — three-way merge against BASE so a writer
+    // that read the file before our live edits cannot delete them.
+    if (base != null && ours !== base && ours !== onDisk) {
+      const merged = merge3(base, ours, onDisk);
+      nextText = merged.text;
+      if (merged.conflicts.length > 0) {
+        const sidecarPath = writeConflictSidecar(entry, onDisk, merged.conflicts.length);
+        entry.pendingConflict = {
+          reason: 'overlapping-hunks',
+          sidecarPath,
+          hunks: merged.conflicts.length,
+          at: new Date().toISOString(),
+        };
+      }
+    } else if (ours !== onDisk && liveEditsSinceLastDiskState(entry)) {
+      // Wholesale accept, but the text it replaces was produced through the
+      // live channel and already flushed (ours === lastDiskText, so the merge
+      // gate above sees no divergence — this was the silent-loss window). A
+      // raw fs write carries no base, so lineage is genuinely ambiguous: the
+      // writer keeps the freedom to rewrite the file, but the replaced live
+      // text is preserved first and the overwrite is surfaced, never silent.
+      const sidecarPath = writePreservationSidecar(
+        entry,
+        ours,
+        'unversioned disk write replaced flushed live edits; '
+        + 'replaced text preserved in the sidecar',
+      );
+      entry.pendingConflict = {
+        reason: 'unversioned-overwrite',
+        sidecarPath,
+        at: new Date().toISOString(),
+      };
+    }
+    setLastDiskText(entry, onDisk, 'disk');
     // origin 'disk' keeps the update handler from writing the same bytes back.
-    spliceTextInto(Y, entry.doc, ytext, onDisk, 'disk');
+    const spliced = spliceTextInto(Y, entry.doc, ytext, nextText, 'disk');
+    if (!spliced && entry.pendingConflict) {
+      // Every external hunk conflicted, so the doc text is unchanged and no
+      // update event fired — publish the conflict on its own.
+      appendStateEvent({
+        resource: entry.resource,
+        op: 'update',
+        id: entry.path,
+        patch: { conflict: entry.pendingConflict },
+        source: 'doc:disk',
+      });
+      entry.pendingConflict = null;
+    }
+    // A merge that diverged from what's on disk must write back: disk still
+    // holds THEIRS (preserved in the sidecar when it conflicted); the file
+    // artifact follows the doc.
+    if (nextText !== onDisk) scheduleDiskWrite(Y, entry);
     // Persist even for a no-op splice: the refreshed lastDiskText hash still
     // needs recording.
     persistDocState(Y, entry);
@@ -260,7 +419,7 @@ function seedFromDisk(Y, entry, persisted) {
     reconcileFromDisk(Y, entry);
     return;
   }
-  setLastDiskText(entry, onDisk);
+  setLastDiskText(entry, onDisk, 'open');
   if (entry.doc.getText(TEXT_KEY).toString() !== onDisk) {
     scheduleDiskWrite(Y, entry);
   }
@@ -286,6 +445,9 @@ function watchDocFile(Y, entry) {
         // and idempotent, so fanning out to every doc in the directory is
         // safe when we cannot tell which file changed.
         if (filename) {
+          // Conflict sidecars live in the doc's directory but must never feed
+          // the reconcile loop.
+          if (isConflictSidecar(filename)) return;
           const target = docs.get(filename);
           if (target) scheduleReconcile(Y, target);
           return;
@@ -370,7 +532,7 @@ function loadDocEntry(pathInput) {
   }
 
   const database = openLocalDb();
-  ensureSchema(database);
+  ensureDocStatesSchema(database);
 
   const doc = new Y.Doc();
   const entry = {
@@ -385,6 +547,14 @@ function loadDocEntry(pathInput) {
     // setLastDiskText so the hash that persistence writes stays in step.
     lastDiskText: null,
     lastDiskHash: null,
+    // Bounded ring of recent disk-sync states {text, sha256, seq, at, origin}
+    // (see pushDiskHistory): the base-resolution window for versioned writes
+    // and the live-edit detector for unversioned overwrites. Newest last.
+    diskHistory: [],
+    // Set by reconcileFromDisk when a three-way merge hit overlapping hunks;
+    // consumed by the very next disk-origin update event (the merge splice is
+    // synchronous) so the conflict rides the doc's own event patch.
+    pendingConflict: null,
     // Every pathInput form seen for this doc, so close can clear its
     // resolvedPathCache entries.
     pathAliases: new Set([pathInput]),
@@ -411,12 +581,19 @@ function loadDocEntry(pathInput) {
   // acknowledged edit; at worst the disk artifact lags by the debounce window
   // until the doc is next opened.
   doc.on('update', (update, origin) => {
+    const patch = { yjs: Buffer.from(update).toString('base64') };
+    // Disk reconciles and versioned writes both splice synchronously right
+    // after setting pendingConflict, so the conflict rides this event.
+    if ((origin === 'disk' || origin === 'write') && entry.pendingConflict) {
+      patch.conflict = entry.pendingConflict;
+      entry.pendingConflict = null;
+    }
     appendStateEvent({
       resource: entry.resource,
       op: 'update',
       id: entry.path,
-      patch: { yjs: Buffer.from(update).toString('base64') },
-      source: origin === 'disk' ? 'doc:disk' : 'doc:update',
+      patch,
+      source: origin === 'disk' ? 'doc:disk' : origin === 'write' ? 'doc:write' : 'doc:update',
     });
     if (origin !== 'disk') scheduleDiskWrite(Y, entry);
   });
@@ -440,10 +617,150 @@ function openDoc(pathInput) {
     resource: entry.resource,
     path: entry.path,
     state: Buffer.from(Y.encodeStateAsUpdate(entry.doc)).toString('base64'),
+    // The versioned-write handshake: hash of the exact current text; a writer
+    // holds onto it and sends it back as `baseHash` so its later write can be
+    // safely three-way merged no matter what landed in between.
+    sha256: hashText(entry.doc.getText(TEXT_KEY).toString()),
     seq: currentSeq(),
   };
 }
 
+/**
+ * Resolve a writer-declared base to the exact text it read. `baseHash` is the
+ * authoritative claim (hash of the exact text) and is checked against the
+ * current live text first, then the retained disk-sync history. `baseSeq` is
+ * only consulted when no hash was sent: it must exactly match a retained
+ * entry's recorded seq. Returns null when the declared base is unknown or has
+ * aged out of the ring — the caller then degrades to the conservative path.
+ */
+function resolveBaseText(entry, oursText, baseHash, baseSeq) {
+  if (baseHash !== undefined) {
+    if (baseHash === hashText(oursText)) return oursText;
+    const match = entry.diskHistory.find((item) => item.sha256 === baseHash);
+    return match ? match.text : null;
+  }
+  if (baseSeq !== undefined) {
+    const match = entry.diskHistory.find((item) => item.seq === baseSeq);
+    return match ? match.text : null;
+  }
+  return null;
+}
+
+/**
+ * Versioned whole-text write — the SDK write contract (POST /state/docs/write).
+ *
+ * `options.baseHash` (sha256 hex of the exact text the writer read; preferred)
+ * or `options.baseSeq` (the seq at which it read) declare the write's lineage:
+ *  - base == current text        → plain apply            (merge: 'clean')
+ *  - base in retained history    → three-way merge; non-overlapping edits
+ *    both land (merge: 'merged'); overlapping regions keep the live doc,
+ *    incoming text preserved in a sidecar + conflict on the doc event
+ *    (merge: 'conflict', reason 'overlapping-hunks')
+ *  - base unknown / aged out     → conservative: current live text preserved
+ *    to a sidecar, incoming applies, conflict surfaced
+ *    (merge: 'base-unknown', reason 'base-unknown')
+ *  - no base sent                → unversioned overwrite (merge: 'overwrite'):
+ *    silent when nothing arrived through the live channel since the last disk
+ *    sync; otherwise replaced text preserved + conflict surfaced
+ *    (reason 'unversioned-overwrite')
+ * Never a silent winner: any path that discards text preserves it first.
+ */
+function writeDocText(pathInput, textInput, options = {}) {
+  const Y = loadYjs();
+  if (typeof textInput !== 'string') {
+    throw invalid(`"text" must be the full replacement file contents as a string, e.g. ${WRITE_EXAMPLE} (got ${typeof textInput})`);
+  }
+  if (Buffer.byteLength(textInput, 'utf8') > MAX_DOC_BYTES) {
+    throw invalid(`"text" exceeds the ${MAX_DOC_BYTES}-byte live document limit`, 413);
+  }
+  const { baseHash, baseSeq } = options;
+  if (baseHash !== undefined && !(typeof baseHash === 'string' && /^[0-9a-f]{64}$/.test(baseHash))) {
+    throw invalid(`"baseHash" must be the lowercase sha256 hex (64 chars) of the exact text you read — the "sha256" field returned by open/write, e.g. ${WRITE_EXAMPLE} (got ${JSON.stringify(baseHash)})`);
+  }
+  if (baseSeq !== undefined && !(Number.isInteger(baseSeq) && baseSeq >= 0)) {
+    throw invalid(`"baseSeq" must be the non-negative integer event seq at which you read the doc, e.g. {"path": "notes.md", "text": "...", "baseSeq": 42} (got ${JSON.stringify(baseSeq)})`);
+  }
+
+  const entry = loadDocEntry(pathInput);
+  entry.lastAccess = Date.now();
+  const ytext = entry.doc.getText(TEXT_KEY);
+  const ours = ytext.toString();
+
+  const respond = (merge, finalText, conflict) => ({
+    merge,
+    seq: currentSeq(),
+    sha256: hashText(finalText),
+    conflict: conflict || null,
+  });
+
+  if (textInput === ours) return respond('noop', ours, null);
+
+  let merge;
+  let nextText = textInput;
+  let conflict = null;
+  if (baseHash !== undefined || baseSeq !== undefined) {
+    const baseText = resolveBaseText(entry, ours, baseHash, baseSeq);
+    if (baseText != null) {
+      const merged = merge3(baseText, ours, textInput);
+      nextText = merged.text;
+      merge = merged.conflicts.length > 0 ? 'conflict' : baseText === ours ? 'clean' : 'merged';
+      if (merged.conflicts.length > 0) {
+        const sidecarPath = writeConflictSidecar(entry, textInput, merged.conflicts.length);
+        conflict = {
+          reason: 'overlapping-hunks',
+          sidecarPath,
+          hunks: merged.conflicts.length,
+          at: new Date().toISOString(),
+        };
+      }
+    } else {
+      // The declared base is not the current text and not in the retention
+      // window: merging against a guess could mangle both sides, so preserve
+      // the current text durably, let the write land, and surface it.
+      merge = 'base-unknown';
+      const sidecarPath = writePreservationSidecar(
+        entry,
+        ours,
+        'versioned write declared a base outside the retention window; '
+        + 'replaced live text preserved in the sidecar',
+      );
+      conflict = { reason: 'base-unknown', sidecarPath, at: new Date().toISOString() };
+    }
+  } else {
+    // No base declared: same conservative posture as a raw disk write —
+    // silent when the replaced text is just the last disk state, preserved
+    // and surfaced when live-channel edits would be destroyed.
+    merge = 'overwrite';
+    if (ours !== entry.lastDiskText || liveEditsSinceLastDiskState(entry)) {
+      const sidecarPath = writePreservationSidecar(
+        entry,
+        ours,
+        'unversioned API write replaced live edits; replaced text preserved in the sidecar',
+      );
+      conflict = { reason: 'unversioned-overwrite', sidecarPath, at: new Date().toISOString() };
+    }
+  }
+
+  entry.pendingConflict = conflict;
+  // origin 'write' rides the conflict on the emitted doc event and schedules
+  // the disk write-back (unlike 'disk', which must not echo to disk).
+  const spliced = spliceTextInto(Y, entry.doc, ytext, nextText, 'write');
+  if (!spliced && entry.pendingConflict) {
+    // Every incoming hunk conflicted: the doc text is unchanged, no update
+    // event fired — publish the conflict on its own (mirrors reconcile).
+    appendStateEvent({
+      resource: entry.resource,
+      op: 'update',
+      id: entry.path,
+      patch: { conflict: entry.pendingConflict },
+      source: 'doc:write',
+    });
+    entry.pendingConflict = null;
+  }
+  persistDocState(Y, entry);
+  return respond(merge, nextText, conflict);
+}
+
 function applyDocUpdates(pathInput, updatesInput) {
   const Y = loadYjs();
   const inputs = Array.isArray(updatesInput) ? updatesInput : [updatesInput];
@@ -528,4 +845,7 @@ module.exports = {
   openDoc,
   pathFromDocResource,
   readDocResource,
+  writeDocText,
+  // Test hooks only — not part of the module contract.
+  _private: { openDocs },
 };