amalgm 0.1.151 → 0.1.152

package/lib/tunnel-events.js

@@ -325,17 +325,21 @@ function createEventTunnel({ record, foreground = false }) {
   let backoffMs = 1000;
   let connectStartedAt = 0;
   let lastGatewayFrameAt = 0;
+  let malformedFrames = 0;
   const upstreamSockets = new Map();
   const upstreamStreams = new Map();
 
+  // Always emit: daemon mode pipes stdio to daemon.log (cli.js start path),
+  // which is where anyone hunting a dead stream looks first. The tunnel is
+  // the flakiest link in the realtime chain — a mute tunnel means reconnects,
+  // heartbeat timeouts, and close codes leave no trace exactly when they
+  // matter. These fire on lifecycle events, not per frame.
   function log(message) {
-    const line = `[event-tunnel] ${message}`;
-    if (foreground) console.log(line);
+    console.log(`[event-tunnel] ${message}`);
   }
 
   function warn(message) {
-    const line = `[event-tunnel] ${message}`;
-    if (foreground) console.warn(line);
+    console.warn(`[event-tunnel] ${message}`);
   }
 
   function send(frame) {
@@ -660,8 +664,14 @@ function createEventTunnel({ record, foreground = false }) {
       lastGatewayFrameAt = Date.now();
       try {
         handleFrame(JSON.parse(data.toString()));
-      } catch {
-        // Ignore malformed gateway frames.
+      } catch (error) {
+        // A malformed frame is the silent-drop failure mode the tunnel war is
+        // about — count it and sample-log so it's visible without flooding on
+        // a corrupt burst.
+        malformedFrames += 1;
+        if (malformedFrames === 1 || malformedFrames % 100 === 0) {
+          warn(`malformed gateway frame #${malformedFrames} dropped: ${error?.message || error}`);
+        }
       }
     });
     ws.on('close', (code, reason) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "amalgm",
-  "version": "0.1.151",
+  "version": "0.1.152",
   "description": "Amalgm local computer runtime: login, MCP, chat, events, previews, and tunnels.",
   "license": "UNLICENSED",
   "private": false,

package/runtime/scripts/amalgm-mcp/server/routes/state.js

@@ -4,12 +4,16 @@ const stateRest = require('../../state/rest');
 
 async function handleStateRoutes(ctx) {
   if (ctx.method === 'GET') {
+    // Async handlers must be awaited: a rejection that escapes here is an
+    // unhandled promise, and the client hangs on a response that never
+    // arrives instead of getting the router's 500. handleStream is
+    // synchronous, so its throws already propagate.
     if (ctx.pathname === '/state/snapshot') {
-      stateRest.handleSnapshot(ctx.getQuery(), ctx.sendJson);
+      await stateRest.handleSnapshot(ctx.getQuery(), ctx.sendJson);
       return true;
     }
     if (ctx.pathname === '/state/events') {
-      stateRest.handleEvents(ctx.getQuery(), ctx.sendJson);
+      await stateRest.handleEvents(ctx.getQuery(), ctx.sendJson);
       return true;
     }
     if (ctx.pathname === '/state/stream') {
@@ -17,7 +21,7 @@ async function handleStateRoutes(ctx) {
       return true;
     }
     if (ctx.pathname === '/state/resources') {
-      stateRest.handleResourceList(ctx.sendJson);
+      await stateRest.handleResourceList(ctx.sendJson);
       return true;
     }
     return false;

package/runtime/scripts/amalgm-mcp/state/app-resources.js

@@ -13,7 +13,7 @@
  * them into the in-memory registry on first use.
  */
 
-const { openLocalDb } = require('./db');
+const { cachedStatement, openLocalDb } = require('./db');
 const registry = require('./registry');
 const { insertStateEvent, publishStateEvent } = require('./events');
 
@@ -140,30 +140,42 @@ function normalizeEmitEvent(name, input, index) {
   return { op, id: String(id), value: input.value };
 }
 
-function applyMirror(database, name, event) {
+// event.value is stringified exactly once per event; the mirror rows and the
+// event-log insert both consume this. For replace, JSON.stringify of an array
+// is precisely its element JSON joined by commas, so the whole-array form is
+// composed from the per-row strings instead of serializing twice.
+function serializeEmitEvent(event) {
   if (event.op === 'replace') {
-    database.prepare('DELETE FROM app_resource_rows WHERE resource = ?').run(name);
-    const upsert = database.prepare(`
+    const rowJson = event.value.map((row) => JSON.stringify(row));
+    return { rowJson, valueJson: `[${rowJson.join(',')}]` };
+  }
+  return { valueJson: event.value === undefined ? null : JSON.stringify(event.value) };
+}
+
+function applyMirror(database, name, event, serialized) {
+  if (event.op === 'replace') {
+    cachedStatement(database, 'DELETE FROM app_resource_rows WHERE resource = ?').run(name);
+    const upsert = cachedStatement(database, `
       INSERT INTO app_resource_rows (resource, id, value_json, updated_at)
       VALUES (?, ?, ?, strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
       ON CONFLICT(resource, id) DO UPDATE SET
         value_json = excluded.value_json,
         updated_at = excluded.updated_at
     `);
-    for (const row of event.value) upsert.run(name, String(row.id), JSON.stringify(row));
+    event.value.forEach((row, index) => upsert.run(name, String(row.id), serialized.rowJson[index]));
     return;
   }
   if (event.op === 'delete') {
-    database.prepare('DELETE FROM app_resource_rows WHERE resource = ? AND id = ?').run(name, event.id);
+    cachedStatement(database, 'DELETE FROM app_resource_rows WHERE resource = ? AND id = ?').run(name, event.id);
     return;
   }
-  database.prepare(`
+  cachedStatement(database, `
     INSERT INTO app_resource_rows (resource, id, value_json, updated_at)
     VALUES (?, ?, ?, strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
     ON CONFLICT(resource, id) DO UPDATE SET
       value_json = excluded.value_json,
       updated_at = excluded.updated_at
-  `).run(name, event.id, JSON.stringify(event.value));
+  `).run(name, event.id, serialized.valueJson);
 }
 
 /**
@@ -176,7 +188,7 @@ function emitAppResourceEvents(name, eventsInput, options = {}) {
     throw invalid(`resource name must look like "app:<appId>:<name>" (got ${JSON.stringify(name)})`);
   }
   const database = openLocalDb();
-  const registered = database.prepare('SELECT 1 FROM app_resources WHERE name = ?').get(name);
+  const registered = cachedStatement(database, 'SELECT 1 FROM app_resources WHERE name = ?').get(name);
   if (!registered) {
     const error = new Error(`resource "${name}" is not registered; call /state/resources/register first`);
     error.statusCode = 404;
@@ -194,12 +206,14 @@ function emitAppResourceEvents(name, eventsInput, options = {}) {
   const inserted = database.transaction(() => {
     const out = [];
     for (const event of normalized) {
-      applyMirror(database, name, event);
+      const serialized = serializeEmitEvent(event);
+      applyMirror(database, name, event, serialized);
       out.push(insertStateEvent(database, {
         resource: name,
         op: event.op,
         id: event.id,
         value: event.value,
+        valueJson: serialized.valueJson,
         source,
       }));
     }

package/runtime/scripts/amalgm-mcp/state/db.js

@@ -14,6 +14,26 @@ const { LOCAL_DB_FILE, STORAGE_DIR } = require('../config');
 
 let db = null;
 
+// Prepared statements cached per database handle: preparing is a parse+plan
+// every time, and the realtime hot paths (event insert, replay reads, mirror
+// upserts, doc persistence) run the same fixed SQL for the process lifetime.
+// WeakMap so a closed handle's statements are collected with it.
+const statementCaches = new WeakMap();
+
+function cachedStatement(database, sql) {
+  let cache = statementCaches.get(database);
+  if (!cache) {
+    cache = new Map();
+    statementCaches.set(database, cache);
+  }
+  let statement = cache.get(sql);
+  if (!statement) {
+    statement = database.prepare(sql);
+    cache.set(sql, statement);
+  }
+  return statement;
+}
+
 function getBetterSqlite3() {
   try {
     return require('better-sqlite3');
@@ -52,8 +72,10 @@ function migrate(database = openLocalDb()) {
       resource_version INTEGER
     );
 
-    CREATE INDEX IF NOT EXISTS event_log_resource_seq_idx
-      ON event_log(resource, seq);
+    -- No query filters event_log by resource (replay and pruning walk seq/ts);
+    -- the index only taxed every insert. Dropped on databases that carry it
+    -- from earlier releases.
+    DROP INDEX IF EXISTS event_log_resource_seq_idx;
 
     CREATE TABLE IF NOT EXISTS local_meta (
       key TEXT PRIMARY KEY,
@@ -454,6 +476,7 @@ function closeLocalDb() {
 }
 
 module.exports = {
+  cachedStatement,
   closeLocalDb,
   openLocalDb,
 };

package/runtime/scripts/amalgm-mcp/state/docs.js

@@ -23,17 +23,18 @@
  * realtime layer is affected.
  *
  * External writers (agents editing the file directly) are folded in by a
- * per-document directory watcher: on file change, disk text is diffed into
- * the Y.Doc as a splice, which emits a normal update event. The cycle is
- * loop-safe because reconcile only acts when disk content changed past the
- * last text we read or wrote (entry.lastDiskText) — our own write-backs and
- * stale watcher events can never splice, so they can never re-emit.
+ * per-directory watcher shared by every open doc in that folder: on file
+ * change, disk text is diffed into the Y.Doc as a splice, which emits a
+ * normal update event. The cycle is loop-safe because reconcile only acts
+ * when disk content changed past the last text we read or wrote
+ * (entry.lastDiskText) — our own write-backs and stale watcher events can
+ * never splice, so they can never re-emit.
  */
 
 const crypto = require('crypto');
 const fs = require('fs');
 const path = require('path');
-const { openLocalDb } = require('./db');
+const { cachedStatement, openLocalDb } = require('./db');
 const { appendStateEvent, currentSeq } = require('./events');
 
 const DOC_RESOURCE_PREFIX = 'doc:';
@@ -44,10 +45,21 @@ const DISK_WRITE_DEBOUNCE_MS = 300;
 const WATCH_RECONCILE_DEBOUNCE_MS = 200;
 const MAX_UPDATES_PER_CALL = 200;
 
-// path -> { doc, resource, watcher, reconcileTimer, diskWriteTimer,
-//           lastAccess, suppressDiskWriteOnce }
+// resolved path -> { doc, resource, reconcileTimer, diskWriteTimer,
+//                    lastAccess, lastDiskText, lastDiskHash, pathAliases }
 const openDocs = new Map();
 
+// pathInput (as the client sent it) -> resolved path, held only while the doc
+// is open (entries die with closeDocEntry). The update POST is the hot path,
+// and re-running the realpath/stat/containment chain per call buys nothing
+// for a doc whose guards were enforced at open.
+const resolvedPathCache = new Map();
+
+// parent directory -> { watcher, docs: Map<basename, entry> }. One watcher
+// per directory instead of one per doc: N docs in a folder cost one fd, and
+// the watcher dies when its last doc closes (docs.size is the refcount).
+const dirWatchers = new Map();
+
 function fsPrivate() {
   return require('../fs/rest')._private;
 }
@@ -92,6 +104,14 @@ function hashText(text) {
   return crypto.createHash('sha256').update(text, 'utf8').digest('hex');
 }
 
+// The only way lastDiskText is ever assigned: the hash is what persistence
+// writes, and hashing at assignment (rare: open, reconcile, flush) keeps the
+// full-text SHA-256 off the per-batch persist path.
+function setLastDiskText(entry, text) {
+  entry.lastDiskText = text;
+  entry.lastDiskHash = text == null ? null : hashText(text);
+}
+
 function docResourceName(absolutePath) {
   return `${DOC_RESOURCE_PREFIX}${fsPrivate().base64UrlEncode(absolutePath)}`;
 }
@@ -154,10 +174,10 @@ function isLowSurrogate(code) {
   return code >= 0xdc00 && code <= 0xdfff;
 }
 
+// Schema is ensured once at loadDocEntry; every entry that reaches here came
+// through it.
 function persistDocState(Y, entry) {
-  const database = openLocalDb();
-  ensureSchema(database);
-  database.prepare(`
+  cachedStatement(openLocalDb(), `
     INSERT INTO doc_states (path, state, updated_at, last_disk_hash)
     VALUES (?, ?, strftime('%Y-%m-%dT%H:%M:%fZ', 'now'), ?)
     ON CONFLICT(path) DO UPDATE SET
@@ -167,7 +187,7 @@ function persistDocState(Y, entry) {
   `).run(
     entry.path,
     Buffer.from(Y.encodeStateAsUpdate(entry.doc)),
-    entry.lastDiskText == null ? null : hashText(entry.lastDiskText),
+    entry.lastDiskHash,
   );
 }
 
@@ -185,7 +205,7 @@ function flushDocToDisk(Y, entry) {
     const text = entry.doc.getText(TEXT_KEY).toString();
     const onDisk = fs.readFileSync(entry.path, 'utf8');
     if (onDisk !== text) fs.writeFileSync(entry.path, text, 'utf8');
-    entry.lastDiskText = text;
+    setLastDiskText(entry, text);
     persistDocState(Y, entry);
   } catch (error) {
     console.warn(`[LiveDoc] Failed to flush "${entry.path}" to disk:`, error?.message || error);
@@ -200,13 +220,13 @@ function reconcileFromDisk(Y, entry) {
     // of disk (write-back still debounced) would "reconcile" the stale disk
     // text back over live edits.
     if (onDisk === entry.lastDiskText) return;
-    entry.lastDiskText = onDisk;
+    setLastDiskText(entry, onDisk);
     const ytext = entry.doc.getText(TEXT_KEY);
     // origin 'disk' keeps the update handler from writing the same bytes back.
-    const changed = spliceTextInto(Y, entry.doc, ytext, onDisk, 'disk');
-    // The splice's update handler persists; a no-op splice still needs the
-    // refreshed lastDiskText hash recorded.
-    if (!changed) persistDocState(Y, entry);
+    spliceTextInto(Y, entry.doc, ytext, onDisk, 'disk');
+    // Persist even for a no-op splice: the refreshed lastDiskText hash still
+    // needs recording.
+    persistDocState(Y, entry);
   } catch (error) {
     // File deleted or unreadable — the doc simply stops following the disk.
     console.warn(`[LiveDoc] Reconcile from disk failed for "${entry.path}":`, error?.message || error);
@@ -240,34 +260,61 @@ function seedFromDisk(Y, entry, persisted) {
     reconcileFromDisk(Y, entry);
     return;
   }
-  entry.lastDiskText = onDisk;
+  setLastDiskText(entry, onDisk);
   if (entry.doc.getText(TEXT_KEY).toString() !== onDisk) {
     scheduleDiskWrite(Y, entry);
   }
 }
 
+function scheduleReconcile(Y, entry) {
+  if (entry.reconcileTimer) return;
+  entry.reconcileTimer = setTimeout(() => {
+    entry.reconcileTimer = null;
+    reconcileFromDisk(Y, entry);
+  }, WATCH_RECONCILE_DEBOUNCE_MS);
+  if (typeof entry.reconcileTimer.unref === 'function') entry.reconcileTimer.unref();
+}
+
 function watchDocFile(Y, entry) {
   const dir = path.dirname(entry.path);
-  const base = path.basename(entry.path);
-  try {
-    entry.watcher = fs.watch(dir, { persistent: false }, (eventType, filename) => {
-      // Some platforms omit filename; reconcile is content-compare based and
-      // idempotent, so over-firing is safe.
-      if (filename && filename !== base) return;
-      if (entry.reconcileTimer) return;
-      entry.reconcileTimer = setTimeout(() => {
-        entry.reconcileTimer = null;
-        reconcileFromDisk(Y, entry);
-      }, WATCH_RECONCILE_DEBOUNCE_MS);
-      if (typeof entry.reconcileTimer.unref === 'function') entry.reconcileTimer.unref();
-    });
-    entry.watcher.on('error', () => {
-      try { entry.watcher.close(); } catch {}
-      entry.watcher = null;
-    });
-    if (typeof entry.watcher.unref === 'function') entry.watcher.unref();
-  } catch {
-    // Watching is best-effort; external edits still reconcile on next open.
+  let shared = dirWatchers.get(dir);
+  if (!shared) {
+    const docs = new Map();
+    try {
+      const watcher = fs.watch(dir, { persistent: false }, (eventType, filename) => {
+        // Some platforms omit filename; reconcile is content-compare based
+        // and idempotent, so fanning out to every doc in the directory is
+        // safe when we cannot tell which file changed.
+        if (filename) {
+          const target = docs.get(filename);
+          if (target) scheduleReconcile(Y, target);
+          return;
+        }
+        for (const target of docs.values()) scheduleReconcile(Y, target);
+      });
+      watcher.on('error', () => {
+        try { watcher.close(); } catch {}
+        dirWatchers.delete(dir);
+      });
+      if (typeof watcher.unref === 'function') watcher.unref();
+      shared = { watcher, docs };
+      dirWatchers.set(dir, shared);
+    } catch {
+      // Watching is best-effort; external edits still reconcile on next open.
+      return;
+    }
+  }
+  shared.docs.set(path.basename(entry.path), entry);
+}
+
+function unwatchDocFile(entry) {
+  const dir = path.dirname(entry.path);
+  const shared = dirWatchers.get(dir);
+  if (!shared) return;
+  shared.docs.delete(path.basename(entry.path));
+  if (shared.docs.size === 0) {
+    try { shared.watcher.close(); } catch {}
+    dirWatchers.delete(dir);
   }
 }
 
@@ -281,16 +328,16 @@ function closeDocEntry(Y, entry) {
     entry.reconcileTimer = null;
   }
   flushDocToDisk(Y, entry);
-  if (entry.watcher) {
-    try { entry.watcher.close(); } catch {}
-    entry.watcher = null;
-  }
+  unwatchDocFile(entry);
+  for (const alias of entry.pathAliases) resolvedPathCache.delete(alias);
   entry.doc.destroy();
   openDocs.delete(entry.path);
 }
 
+// Called before inserting a new entry: evicts down to MAX_OPEN_DOCS - 1 so
+// the insert lands exactly at the cap, never past it.
 function evictIfNeeded(Y) {
-  while (openDocs.size > MAX_OPEN_DOCS) {
+  while (openDocs.size >= MAX_OPEN_DOCS) {
     let oldest = null;
     for (const entry of openDocs.values()) {
       if (!oldest || entry.lastAccess < oldest.lastAccess) oldest = entry;
@@ -302,10 +349,23 @@ function evictIfNeeded(Y) {
 
 function loadDocEntry(pathInput) {
   const Y = loadYjs();
+
+  const cachedResolved = resolvedPathCache.get(pathInput);
+  if (cachedResolved !== undefined) {
+    const open = openDocs.get(cachedResolved);
+    if (open) {
+      open.lastAccess = Date.now();
+      return open;
+    }
+    resolvedPathCache.delete(pathInput);
+  }
+
   const resolved = resolveDocPath(pathInput);
   const existing = openDocs.get(resolved);
   if (existing) {
     existing.lastAccess = Date.now();
+    existing.pathAliases.add(pathInput);
+    resolvedPathCache.set(pathInput, resolved);
     return existing;
   }
 
@@ -317,16 +377,20 @@ function loadDocEntry(pathInput) {
     path: resolved,
     resource: docResourceName(resolved),
     doc,
-    watcher: null,
     reconcileTimer: null,
     diskWriteTimer: null,
     lastAccess: Date.now(),
     // The disk text as of our last read/write; reconcile skips anything that
-    // hasn't changed past it (see reconcileFromDisk).
+    // hasn't changed past it (see reconcileFromDisk). Assigned only through
+    // setLastDiskText so the hash that persistence writes stays in step.
     lastDiskText: null,
+    lastDiskHash: null,
+    // Every pathInput form seen for this doc, so close can clear its
+    // resolvedPathCache entries.
+    pathAliases: new Set([pathInput]),
   };
 
-  let persisted = database.prepare('SELECT state, last_disk_hash FROM doc_states WHERE path = ?').get(resolved) || null;
+  let persisted = cachedStatement(database, 'SELECT state, last_disk_hash FROM doc_states WHERE path = ?').get(resolved) || null;
   if (persisted?.state) {
     try {
       Y.applyUpdate(doc, new Uint8Array(persisted.state), 'persisted');
@@ -339,10 +403,13 @@ function loadDocEntry(pathInput) {
   }
 
   // Every applied change — client update, disk reconcile, or seed — flows
-  // through this one pipeline: event out, state persisted, then (except
-  // disk-origin) disk out. Persisting on every update means a kill — even
-  // SIGKILL — can never lose an acknowledged edit; at worst the disk artifact
-  // lags by the debounce window until the doc is next opened.
+  // through this one pipeline: event out, then (except disk-origin) disk out.
+  // State persistence is NOT per update: the durability contract only needs
+  // the state durable before the caller acknowledges, so each mutation entry
+  // point (applyDocUpdates, reconcileFromDisk, flushDocToDisk) persists once
+  // when it is done. A kill — even SIGKILL — still never loses an
+  // acknowledged edit; at worst the disk artifact lags by the debounce window
+  // until the doc is next opened.
   doc.on('update', (update, origin) => {
     appendStateEvent({
       resource: entry.resource,
@@ -351,7 +418,6 @@ function loadDocEntry(pathInput) {
       patch: { yjs: Buffer.from(update).toString('base64') },
       source: origin === 'disk' ? 'doc:disk' : 'doc:update',
     });
-    persistDocState(Y, entry);
     if (origin !== 'disk') scheduleDiskWrite(Y, entry);
   });
 
@@ -360,9 +426,10 @@ function loadDocEntry(pathInput) {
   // write debounce) gets the doc written back out — never spliced over it.
   seedFromDisk(Y, entry, persisted);
 
+  evictIfNeeded(Y);
   openDocs.set(resolved, entry);
   watchDocFile(Y, entry);
-  evictIfNeeded(Y);
+  resolvedPathCache.set(pathInput, resolved);
   return entry;
 }
 
@@ -393,12 +460,22 @@ function applyDocUpdates(pathInput, updatesInput) {
 
   const entry = loadDocEntry(pathInput);
   entry.lastAccess = Date.now();
-  for (const update of updates) {
-    try {
-      Y.applyUpdate(entry.doc, new Uint8Array(update), 'client');
-    } catch (error) {
-      throw invalid(`invalid Yjs update: ${error?.message || error}`);
+  let applied = 0;
+  try {
+    for (const update of updates) {
+      try {
+        Y.applyUpdate(entry.doc, new Uint8Array(update), 'client');
+      } catch (error) {
+        throw invalid(`invalid Yjs update: ${error?.message || error}`);
+      }
+      applied += 1;
     }
+  } finally {
+    // One persist per batch, before the HTTP response acknowledges: that is
+    // the whole durability contract. It also runs when a later update in the
+    // batch is rejected — events for the applied prefix are already in the
+    // log, and the persisted state must never lag them.
+    if (applied > 0) persistDocState(Y, entry);
   }
   return { applied: updates.length, seq: currentSeq() };
 }

package/runtime/scripts/amalgm-mcp/state/events.js

@@ -1,7 +1,7 @@
 'use strict';
 
 const { EventEmitter } = require('events');
-const { openLocalDb } = require('./db');
+const { cachedStatement, openLocalDb } = require('./db');
 
 const emitter = new EventEmitter();
 emitter.setMaxListeners(200);
@@ -10,9 +10,13 @@ emitter.setMaxListeners(200);
 // bridge brief stream blips from it, and anything older is served better by a
 // fresh snapshot (the stream sends `reset` when history is gone). Kept just
 // long enough to ride out sleep/wake and flaky-network windows.
-const DEFAULT_EVENT_LOG_RETENTION_HOURS = 0.25;
-const DEFAULT_EVENT_LOG_PRUNE_BATCH_SIZE = 1000;
-const DEFAULT_EVENT_LOG_PRUNE_INTERVAL_MS = 60_000;
+//
+// The env knobs are resolved once at load: they exist for overrides set
+// before the process starts, and re-parsing them on every publish was pure
+// hot-path waste. Explicit options still win per call.
+const EVENT_LOG_RETENTION_HOURS = positiveNumber(process.env.AMALGM_EVENT_LOG_RETENTION_HOURS, 0.25);
+const EVENT_LOG_PRUNE_BATCH_SIZE = Math.max(1, Math.floor(positiveNumber(process.env.AMALGM_EVENT_LOG_PRUNE_BATCH_SIZE, 1000)));
+const EVENT_LOG_PRUNE_INTERVAL_MS = positiveNumber(process.env.AMALGM_EVENT_LOG_PRUNE_INTERVAL_MS, 60_000);
 const EVENT_LOG_PRUNE_DELAY_MS = 1_000;
 
 let lastPruneAttemptAt = 0;
@@ -24,24 +28,15 @@ function positiveNumber(value, fallback) {
 }
 
 function retentionHours(options = {}) {
-  return positiveNumber(
-    options.retentionHours ?? process.env.AMALGM_EVENT_LOG_RETENTION_HOURS,
-    DEFAULT_EVENT_LOG_RETENTION_HOURS,
-  );
+  return positiveNumber(options.retentionHours, EVENT_LOG_RETENTION_HOURS);
 }
 
 function pruneBatchSize(options = {}) {
-  return Math.max(1, Math.floor(positiveNumber(
-    options.batchSize ?? process.env.AMALGM_EVENT_LOG_PRUNE_BATCH_SIZE,
-    DEFAULT_EVENT_LOG_PRUNE_BATCH_SIZE,
-  )));
+  return Math.max(1, Math.floor(positiveNumber(options.batchSize, EVENT_LOG_PRUNE_BATCH_SIZE)));
 }
 
 function pruneIntervalMs(options = {}) {
-  return positiveNumber(
-    options.intervalMs ?? process.env.AMALGM_EVENT_LOG_PRUNE_INTERVAL_MS,
-    DEFAULT_EVENT_LOG_PRUNE_INTERVAL_MS,
-  );
+  return positiveNumber(options.intervalMs, EVENT_LOG_PRUNE_INTERVAL_MS);
 }
 
 function retentionCutoffIso(options = {}) {
@@ -88,8 +83,22 @@ function insertStateEvent(database, input) {
     throw new Error('state event requires resource and op');
   }
 
-  const info = database.prepare(`
+  // ts is generated here (same format sqlite's strftime('%Y-%m-%dT%H:%M:%fZ')
+  // default produces: millisecond precision, Z suffix) and written into the
+  // row, so the event can be constructed in-process instead of SELECTing the
+  // row back out and re-parsing JSON the caller already had in hand.
+  const ts = new Date().toISOString();
+  // input.valueJson lets callers that already serialized the value (e.g. the
+  // app-resource mirror) share that one stringify with the log insert.
+  const valueJson = input.valueJson !== undefined ? input.valueJson : safeJsonStringify(input.value);
+  const id = input.id ?? null;
+  const clientMutationId = input.clientMutationId ?? input.client_mutation_id ?? null;
+  const source = input.source ?? null;
+  const version = input.version ?? input.resourceVersion ?? null;
+
+  const info = cachedStatement(database, `
     INSERT INTO event_log (
+      ts,
       resource,
       op,
       id,
@@ -99,20 +108,36 @@ function insertStateEvent(database, input) {
       source,
       resource_version
     )
-    VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
   `).run(
+    ts,
     input.resource,
     input.op,
-    input.id ?? null,
-    safeJsonStringify(input.value),
+    id,
+    valueJson,
     safeJsonStringify(input.patch),
-    input.clientMutationId ?? input.client_mutation_id ?? null,
-    input.source ?? null,
-    input.version ?? input.resourceVersion ?? null,
-  );
-  return normalizeEventRow(
-    database.prepare('SELECT * FROM event_log WHERE seq = ?').get(info.lastInsertRowid),
+    clientMutationId,
+    source,
+    version,
   );
+
+  // Key order matches normalizeEventRow — JSON.stringify follows insertion
+  // order, so the wire shape stays byte-identical with replayed rows. String
+  // coercion on id/clientMutationId/source mirrors the TEXT column affinity a
+  // read-back would have applied.
+  const event = {
+    seq: Number(info.lastInsertRowid),
+    ts,
+    resource: input.resource,
+    op: input.op,
+  };
+  if (id != null) event.id = String(id);
+  if (clientMutationId != null) event.clientMutationId = String(clientMutationId);
+  if (source != null) event.source = String(source);
+  if (version != null) event.version = Number(version);
+  if (input.value !== undefined) event.value = input.value;
+  if (input.patch !== undefined) event.patch = input.patch;
+  return event;
 }
 
 function publishStateEvent(event) {
@@ -133,8 +158,7 @@ function scheduleEventLogPrune() {
 }
 
 function currentSeq() {
-  const row = openLocalDb()
-    .prepare('SELECT COALESCE(MAX(seq), 0) AS seq FROM event_log')
+  const row = cachedStatement(openLocalDb(), 'SELECT COALESCE(MAX(seq), 0) AS seq FROM event_log')
     .get();
   return Number(row?.seq || 0);
 }
@@ -149,15 +173,14 @@ function appendStateEvent(input) {
 function listEventsAfter(afterSeq, options = {}) {
   const after = Number.isFinite(Number(afterSeq)) ? Number(afterSeq) : 0;
   const limit = Math.max(1, Math.min(Number(options.limit) || 1000, 5000));
-  const rows = openLocalDb()
-    .prepare('SELECT * FROM event_log WHERE seq > ? ORDER BY seq ASC LIMIT ?')
+  const database = openLocalDb();
+  const rows = cachedStatement(database, 'SELECT * FROM event_log WHERE seq > ? ORDER BY seq ASC LIMIT ?')
     .all(after, limit);
   return rows.map(normalizeEventRow).filter(Boolean);
 }
 
 function eventLogBounds(database = openLocalDb()) {
-  const row = database
-    .prepare('SELECT MIN(seq) AS min_seq, MAX(seq) AS max_seq FROM event_log')
+  const row = cachedStatement(database, 'SELECT MIN(seq) AS min_seq, MAX(seq) AS max_seq FROM event_log')
     .get();
   const minSeq = row?.min_seq == null ? 0 : Number(row.min_seq);
   const maxSeq = row?.max_seq == null ? 0 : Number(row.max_seq);
@@ -189,7 +212,7 @@ function pruneEventLog(options = {}) {
   const database = options.database || openLocalDb();
   const cutoffIso = retentionCutoffIso({ ...options, retentionHours: hours });
   const batchSize = pruneBatchSize(options);
-  const result = database.prepare(`
+  const result = cachedStatement(database, `
     DELETE FROM event_log
     WHERE seq IN (
       SELECT seq
@@ -232,11 +255,12 @@ function subscribeStateEvents(callback) {
 module.exports = {
   appendStateEvent,
   currentSeq,
-  eventLogBounds,
   insertStateEvent,
   listEventsAfter,
   maybePruneEventLog,
   publishStateEvent,
+  // Exported as the retention test's deterministic seam: the production path
+  // (debounced timer → maybePruneEventLog) cannot be driven predictably.
   pruneEventLog,
   replayGapAfter,
   subscribeStateEvents,

package/runtime/scripts/amalgm-mcp/state/migrations.js

@@ -64,9 +64,29 @@ function migrateLegacyToolboxTables(database) {
   }
 }
 
+const INHERITED_HARNESS_TOOLS_KEY = 'inherited_harness_tools_removed_v1';
+
+function metaValue(database, key) {
+  return database.prepare('SELECT value FROM local_meta WHERE key = ?').get(key)?.value || null;
+}
+
+function setMetaValue(database, key, value) {
+  database.prepare(`
+    INSERT INTO local_meta (key, value, updated_at)
+    VALUES (?, ?, strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
+    ON CONFLICT(key) DO UPDATE SET
+      value = excluded.value,
+      updated_at = excluded.updated_at
+  `).run(key, value);
+}
+
 // Harness-inherited codex catalog tools were seeded by an older release and
-// are now provided dynamically; stale rows are removed on open.
+// are now provided dynamically. The DELETEs are idempotent, so a machine
+// missing the stamp re-runs them harmlessly — but stamping in local_meta
+// stops two full-table scans on every single boot after the first.
 function removeInheritedHarnessTools(database) {
+  if (metaValue(database, INHERITED_HARNESS_TOOLS_KEY)) return;
+
   database.prepare(`
     DELETE FROM tool_actions
     WHERE tool_id IN (
@@ -81,6 +101,13 @@ function removeInheritedHarnessTools(database) {
     WHERE owner IN ('codex')
       AND origin = 'catalog'
   `).run();
+
+  // While the pre-rename toolbox tables exist, migrateLegacyToolboxTables can
+  // re-import the very rows just deleted on a later boot — stamping then
+  // would let them resurrect for good.
+  if (!tableExists(database, 'toolbox_tools')) {
+    setMetaValue(database, INHERITED_HARNESS_TOOLS_KEY, '1');
+  }
 }
 
 function runLegacyMigrations(database) {

package/runtime/scripts/amalgm-mcp/state/registry.js

@@ -89,15 +89,10 @@ function listDefaultResources() {
   return names;
 }
 
-function listRegisteredResources() {
-  return Array.from(resources.keys());
-}
-
 module.exports = {
   getResourceSpec,
   hasResource,
   listDefaultResources,
-  listRegisteredResources,
   readRegisteredResource,
   registerResource,
   registerResourcePrefix,

package/runtime/scripts/amalgm-mcp/state/rest.js

@@ -23,10 +23,22 @@ function parsePositiveInt(value, fallback) {
 // overwrites itself, so the client is told to reset instead.
 const STREAM_REPLAY_MAX_EVENTS = 1000;
 
+// One frame per event no matter how many subscribers receive it: built on
+// first write and cached on the event object itself, which covers both live
+// publishes and backlog replay rows. Non-enumerable (defineProperty default)
+// so the cache can never leak into JSON.stringify(event) — the frame embeds
+// exactly that JSON.
+function sseFrameFor(event) {
+  let frame = event.__sseFrame;
+  if (frame === undefined) {
+    frame = `id: ${event.seq}\nevent: state\ndata: ${JSON.stringify(event)}\n\n`;
+    Object.defineProperty(event, '__sseFrame', { value: frame });
+  }
+  return frame;
+}
+
 function sendSseEvent(res, event) {
-  res.write(`id: ${event.seq}\n`);
-  res.write('event: state\n');
-  res.write(`data: ${JSON.stringify(event)}\n\n`);
+  res.write(sseFrameFor(event));
 }
 
 function sendSseControl(res, type, payload) {
@@ -35,10 +47,16 @@ function sendSseControl(res, type, payload) {
 }
 
 async function handleSnapshot(query, sendJson) {
-  // Restores persisted app registrations so their resources are readable
-  // even before the owning app has spoken this boot.
-  appResources.ensureAppResourcesLoaded();
-  sendJson(200, buildSnapshot(query.resources));
+  try {
+    // Restores persisted app registrations so their resources are readable
+    // even before the owning app has spoken this boot.
+    appResources.ensureAppResourcesLoaded();
+    sendJson(200, buildSnapshot(query.resources));
+  } catch (error) {
+    // buildSnapshot rethrows a failed resource read on purpose; without this
+    // the client waits forever on a response that never comes.
+    sendHandlerError(error, sendJson);
+  }
 }
 
 async function handleResourceRegister(body, sendJson) {
@@ -188,9 +206,14 @@ function handleStream(req, res, query) {
   // A named event (not an SSE comment) so EventSource clients can observe it
   // and detect a silently dead stream, e.g. tunnel frame drops. It doubles as
   // the server-side liveness probe: writing to a dead socket tears down.
-  heartbeat = setInterval(() => {
-    writeSafe(() => sendSseControl(res, 'ping', { t: Date.now() }));
-  }, 25_000);
+  // Guarded: a stream that died during backlog replay has already torn down,
+  // and arming after teardown would leak an interval nothing ever clears.
+  if (!tornDown) {
+    heartbeat = setInterval(() => {
+      writeSafe(() => sendSseControl(res, 'ping', { t: Date.now() }));
+    }, 25_000);
+    if (typeof heartbeat.unref === 'function') heartbeat.unref();
+  }
 
   req.on('close', teardown);
   req.on('error', teardown);

package/runtime/scripts/amalgm-mcp/state/snapshot.js

@@ -41,18 +41,12 @@ function buildSnapshot(resourcesInput) {
     lastUnstable = { seq: beforeSeq, stable: false, resources: data };
   }
 
-  if (lastUnstable) {
-    console.warn(`[LocalLive] Snapshot did not stabilize after 3 attempts (writes kept landing mid-read); serving best-effort data at seq ${lastUnstable.seq}.`);
-    return lastUnstable;
-  }
-  return { seq: currentSeq(), stable: true, resources: {} };
+  // Reaching here means every attempt saw a write land mid-read, and each
+  // one recorded its result — lastUnstable is always set.
+  console.warn(`[LocalLive] Snapshot did not stabilize after 3 attempts (writes kept landing mid-read); serving best-effort data at seq ${lastUnstable.seq}.`);
+  return lastUnstable;
 }
 
 module.exports = {
-  // Kept for existing callers; now derived from the registry.
-  get DEFAULT_RESOURCES() {
-    return registry.listDefaultResources();
-  },
   buildSnapshot,
-  normalizeResources,
 };

package/runtime/scripts/amalgm-mcp/tests/state-app-resources.test.js

@@ -14,6 +14,7 @@ const registry = require('../state/registry');
 const appResources = require('../state/app-resources');
 const { buildSnapshot } = require('../state/snapshot');
 const { listEventsAfter, currentSeq, subscribeStateEvents } = require('../state/events');
+const stateRest = require('../state/rest');
 
 test.after(() => {
   closeLocalDb();
@@ -134,3 +135,23 @@ test('unregister removes rows, frees the name, and tells listeners', () => {
   // Cascade cleared the mirror rows.
   assert.deepEqual(appResources.listAppResourceRows('app:demo:notes'), []);
 });
+
+test('a resource whose read() always throws answers 500 instead of hanging', async () => {
+  registry.registerResource('app:demo:boom', {
+    read: () => {
+      throw new Error('boom: backing store is gone');
+    },
+  });
+  try {
+    let status = null;
+    let body = null;
+    await stateRest.handleSnapshot({ resources: 'app:demo:boom' }, (s, b) => {
+      status = s;
+      body = b;
+    });
+    assert.equal(status, 500, 'the snapshot must answer, not hang');
+    assert.match(String(body?.error), /boom/);
+  } finally {
+    registry.unregisterResource('app:demo:boom');
+  }
+});

package/runtime/scripts/amalgm-mcp/tests/state-docs.test.js

@@ -194,9 +194,10 @@ test('guards: binary, missing, oversized, and malformed inputs are rejected', ()
 
 // ---------------------------------------------------------------------------
 // Durability: a crash (SIGKILL — no flush, no graceful close) must never lose
-// an acknowledged edit. State is persisted on every update; the recorded
-// last-disk hash lets reopen tell "disk is merely behind" (write the doc back
-// out) from "disk changed externally" (fold it in).
+// an acknowledged edit. The contract is per acknowledgment, not per update:
+// state is persisted before applyDocUpdates returns (before the POST response
+// acknowledges); the recorded last-disk hash lets reopen tell "disk is merely
+// behind" (write the doc back out) from "disk changed externally" (fold it in).
 // ---------------------------------------------------------------------------
 
 function crashAndReloadDocs() {
@@ -231,6 +232,31 @@ test('a kill inside the disk-write debounce loses nothing: reopen restores the e
   reloaded.closeAllDocs();
 });
 
+test('a multi-update batch is fully persisted by the time it returns', () => {
+  const target = writeDoc('batch.md', 'base\n');
+  const opened = docs.openDoc(target);
+  const client = clientFromState(opened.state);
+  const updates = [
+    encodeLocalEdit(client, (ytext) => ytext.insert(0, 'one ')),
+    encodeLocalEdit(client, (ytext) => ytext.insert(0, 'two ')),
+    encodeLocalEdit(client, (ytext) => ytext.insert(0, 'three ')),
+  ];
+
+  const result = docs.applyDocUpdates(target, updates);
+  assert.equal(result.applied, 3);
+
+  // Crash with no flush: the batch persist alone must carry all three
+  // updates into doc_states.
+  const reloaded = crashAndReloadDocs();
+  const reopened = reloaded.openDoc(target);
+  assert.equal(
+    clientFromState(reopened.state).getText('content').toString(),
+    'three two one base\n',
+    'reopen after a kill must restore the whole batch',
+  );
+  reloaded.closeAllDocs();
+});
+
 test('an external edit made while closed still folds in on reopen', () => {
   const target = writeDoc('closed-edit.md', 'line one\n');
   const opened = docs.openDoc(target);

package/runtime/scripts/amalgm-mcp/tests/state-events-retention.test.js

@@ -12,7 +12,6 @@ process.env.AMALGM_EVENT_LOG_PRUNE_INTERVAL_MS = '0';
 
 const { closeLocalDb, openLocalDb } = require('../state/db');
 const {
-  eventLogBounds,
   insertStateEvent,
   listEventsAfter,
   pruneEventLog,
@@ -58,7 +57,7 @@ test('event log retention removes old batches and reports replay gaps', () => {
   });
 
   assert.equal(result.deleted, 2);
-  assert.deepEqual(eventLogBounds(db), { minSeq: 3, maxSeq: 3 });
+  assert.deepEqual(listEventsAfter(0).map((event) => event.seq), [3], 'only the fresh event survives');
   assert.equal(replayGapAfter(1, db)?.code, 'state_event_gap');
   assert.equal(replayGapAfter(2, db), null);
   assert.deepEqual(

package/runtime/scripts/amalgm-mcp/tests/state-registry.test.js

@@ -27,8 +27,6 @@ const LEGACY_DEFAULT_RESOURCES = ['automations', 'triggers', 'workflows', 'autom
 
 test('default snapshot set matches the legacy hardcoded list exactly', () => {
   assert.deepEqual(registry.listDefaultResources(), LEGACY_DEFAULT_RESOURCES);
-  assert.deepEqual(snapshot.DEFAULT_RESOURCES, LEGACY_DEFAULT_RESOURCES);
-  assert.deepEqual(snapshot.normalizeResources(undefined), LEGACY_DEFAULT_RESOURCES);
 });
 
 test('non-default built-ins are registered but excluded from the default set', () => {

package/runtime/scripts/amalgm-mcp/tests/state-stream-replay.test.js

@@ -19,18 +19,23 @@ test.after(() => {
   fs.rmSync(tempRoot, { recursive: true, force: true });
 });
 
-function fakeStream() {
+// The response must be an EventEmitter: handleStream subscribes to its
+// close/error events for teardown.
+function fakeStream({ failAfterWrites = Infinity } = {}) {
   const req = new EventEmitter();
   const chunks = [];
   let ended = false;
-  const res = {
-    writeHead: () => {},
-    write: (chunk) => {
-      chunks.push(String(chunk));
-    },
-    end: () => {
-      ended = true;
-    },
+  const res = new EventEmitter();
+  res.writeHead = () => res;
+  res.write = (chunk) => {
+    if (chunks.length >= failAfterWrites) {
+      throw new Error('write after socket gone');
+    }
+    chunks.push(String(chunk));
+    return true;
+  };
+  res.end = () => {
+    ended = true;
   };
   const close = () => req.emit('close');
   return {
@@ -59,12 +64,13 @@ test('stream replays a small backlog as state events', () => {
   const stream = fakeStream();
 
   handleStream(stream.req, stream.res, { after: '0' });
-  stream.close();
 
   const output = stream.output();
   assert.equal((output.match(/event: state/g) || []).length, 3);
   assert.ok(!output.includes('event: reset'), 'small backlog must not reset');
   assert.equal(stream.ended(), false, 'stream stays open for live events');
+  stream.close();
+  assert.equal(stream.ended(), true, 'close tears the stream down');
 });
 
 test('stream resets instead of replaying an oversized backlog', () => {
@@ -91,3 +97,28 @@ test('stream replays only events after the requested seq', () => {
   assert.equal((output.match(/event: state/g) || []).length, 2);
   assert.ok(!output.includes('event: reset'));
 });
+
+test('a stream that dies during backlog replay arms no heartbeat', () => {
+  // The SSE preamble succeeds, then the first backlog write throws →
+  // teardown fires mid-replay, before the heartbeat would be armed. An
+  // interval armed after that has no clearer and pins the event loop —
+  // this suite exiting without --test-force-exit is the systemic proof.
+  const stream = fakeStream({ failAfterWrites: 1 });
+  const armed = [];
+  const realSetInterval = global.setInterval;
+  global.setInterval = (...args) => {
+    const timer = realSetInterval(...args);
+    armed.push(timer);
+    return timer;
+  };
+  try {
+    // after 1101 → a 2-event backlog: replay (not reset) is the path under test.
+    handleStream(stream.req, stream.res, { after: '1101' });
+  } finally {
+    global.setInterval = realSetInterval;
+    for (const timer of armed) clearInterval(timer);
+  }
+
+  assert.equal(armed.length, 0, 'teardown during replay must not arm the heartbeat');
+  assert.equal(stream.ended(), true, 'teardown ends the response');
+});

package/runtime/scripts/amalgm-mcp/tests/state-stream.test.js

@@ -80,6 +80,39 @@ test('a write failure tears the stream down: no orphaned subscriber', () => {
   assert.equal(res.writableEnded, true, 'teardown ends the response');
 });
 
+test('one publish fans out one identical frame per subscriber, and the frame never leaks into JSON', () => {
+  const streams = [0, 1, 2].map(() => {
+    const req = fakeRequest();
+    const res = fakeResponse();
+    rest.handleStream(req, res, {});
+    return { req, res };
+  });
+  // Backlog replay from earlier tests lands on connect; only the new publish
+  // counts.
+  const marks = streams.map(({ res }) => res.writes.length);
+
+  const event = appendStateEvent({
+    resource: 'automations',
+    op: 'replace',
+    value: [{ id: 'fan-out' }],
+    source: 'test',
+  });
+
+  const frames = streams.map(({ res }, i) => res.writes.slice(marks[i]));
+  for (const writes of frames) {
+    assert.equal(writes.length, 1, 'a published event is exactly one write per subscriber');
+  }
+  assert.equal(frames[0][0], frames[1][0]);
+  assert.equal(frames[1][0], frames[2][0]);
+
+  const json = JSON.stringify(event);
+  assert.equal(frames[0][0], `id: ${event.seq}\nevent: state\ndata: ${json}\n\n`);
+  assert.ok(!json.includes('__sseFrame'), 'the cached frame must never serialize');
+  assert.ok(!Object.keys(event).includes('__sseFrame'));
+
+  for (const { req } of streams) req.emit('close');
+});
+
 test('close and error firing together tear down exactly once', () => {
   const req = fakeRequest();
   const res = fakeResponse();