amalgm 0.1.152 → 0.1.153

package/lib/tunnel-events.js

@@ -1,5 +1,6 @@
 'use strict';
 
+const crypto = require('crypto');
 const fs = require('fs');
 const http = require('http');
 const os = require('os');
@@ -22,6 +23,31 @@ const DEFAULT_TARGET_PORT = DEFAULT_MCP_PORT;
 const CONNECT_TIMEOUT_MS = 20_000;
 const WATCHDOG_INTERVAL_MS = 15_000;
 const HEARTBEAT_TIMEOUT_MS = 75_000;
+// Per-process identity: a zombie runtime holding a stale gateway slot is
+// indistinguishable from the live one without this. AMALGM_RUNTIME_INSTANCE_ID
+// is directory-derived and inherited via env, so a zombie can share it —
+// this id is fresh per process, always.
+const INSTANCE_ID = crypto.randomBytes(8).toString('hex');
+const BOOT_TS = Date.now();
+const PROTOCOL_VERSION = 2;
+// Gateway close code: this process lost slot arbitration to a newer runtime
+// instance (same computer_id + runtime_label). Reconnecting fast would just
+// steal the slot back and reset the winner's streams — the historical
+// "stuck streams" ping-pong. Retry slowly instead of stopping: the newer
+// instance may die, and regaining the slot then is correct.
+const SUPERSEDED_CLOSE_CODE = 4009;
+const DEFAULT_SUPERSEDED_RETRY_MS = 5 * 60 * 1000;
+// Backoff only resets after a connection has proven stable for this long;
+// resetting on open turns any slot fight into a 1s reconnect war.
+const DEFAULT_STABLE_RESET_MS = 30_000;
+// Send-path tuning: frames above BULK_FRAME_BYTES ride the bulk queue so a
+// big file read cannot head-of-line-block keystroke-sized events; bodies
+// above RES_CHUNK_BYTES additionally split into chunk frames on v2 peers.
+const BULK_FRAME_BYTES = 32 * 1024;
+const RES_CHUNK_BYTES = 256 * 1024;
+const DEFAULT_SEND_HWM_BYTES = 1024 * 1024;
+const DEFAULT_BULK_QUEUE_MAX_BYTES = 64 * 1024 * 1024;
+const DRAIN_POLL_MS = 50;
 const DEFAULT_RUNTIME_PORTS = new Map([
   ['gateway', DEFAULT_GATEWAY_PORT],
   ['portMonitor', DEFAULT_PORT_MONITOR_PORT],
@@ -84,6 +110,15 @@ const WS_HANDSHAKE_HEADERS = new Set([
   'sec-websocket-protocol',
 ]);
 
+function envBytes(name, fallback) {
+  const value = Number(process.env[name]);
+  return Number.isFinite(value) && value > 0 ? value : fallback;
+}
+
+function msAgo(ts, now) {
+  return ts ? `${now - ts}ms ago` : 'never';
+}
+
 function tunnelUrl(rawUrl) {
   const url = new URL(rawUrl);
   url.searchParams.delete('token');
@@ -326,6 +361,23 @@ function createEventTunnel({ record, foreground = false }) {
   let connectStartedAt = 0;
   let lastGatewayFrameAt = 0;
   let malformedFrames = 0;
+  // v2+ gateways acknowledge hello with their protocol version; the deployed
+  // v1 gateway never sends one, so chunking stays off against it.
+  let peerProtocolVersion = 1;
+  // Heartbeat forensics: cheap timestamps so a timeout can say whether the
+  // network went silent, we stopped sending, or backpressure stalled us.
+  let connectedAt = 0;
+  let lastPingSentAt = 0;
+  let lastPongAt = 0;
+  let lastFrameReceivedAt = 0;
+  let lastFrameSentAt = 0;
+  const sendHwmBytes = envBytes('AMALGM_TUNNEL_HWM_BYTES', DEFAULT_SEND_HWM_BYTES);
+  const bulkQueueMaxBytes = envBytes('AMALGM_TUNNEL_BULK_MAX_BYTES', DEFAULT_BULK_QUEUE_MAX_BYTES);
+  const supersededRetryMs = envBytes('AMALGM_TUNNEL_SUPERSEDED_RETRY_MS', DEFAULT_SUPERSEDED_RETRY_MS);
+  const stableResetMs = envBytes('AMALGM_TUNNEL_STABLE_RESET_MS', DEFAULT_STABLE_RESET_MS);
+  const bulkQueue = [];
+  let bulkQueuedBytes = 0;
+  let drainTimer = null;
   const upstreamSockets = new Map();
   const upstreamStreams = new Map();
 
@@ -342,12 +394,146 @@ function createEventTunnel({ record, foreground = false }) {
     console.warn(`[event-tunnel] ${message}`);
   }
 
-  function send(frame) {
+  function sendPayload(payload) {
     if (!ws || ws.readyState !== WebSocket.OPEN) return false;
-    ws.send(JSON.stringify(frame));
+    ws.send(payload);
+    lastFrameSentAt = Date.now();
     return true;
   }
 
+  // Control path: hello, heartbeats, stream events, small responses. Always
+  // sends immediately so it can never starve behind a queued response body.
+  function send(frame) {
+    return sendPayload(JSON.stringify(frame));
+  }
+
+  // A bulk queue entry: `bytes` is the exact frame size (known up front, so
+  // the queue budget applies BEFORE anything is materialized) and `build`
+  // produces the JSON payload only when the frame is about to hit the wire.
+  // A chunked response therefore peaks at body + one in-flight frame, never
+  // body + every frame (~2.5x the body) like the eager version did.
+  function payloadItem(payload) {
+    return { bytes: payload.length, build: () => payload };
+  }
+
+  // Bulk path: big response bodies queue behind ws.bufferedAmount so they
+  // drain between control frames instead of head-of-line-blocking them.
+  // The queue is bounded — a dropped response must fail loudly for its
+  // req_id, never hang silently.
+  function enqueueBulk(items, reqId) {
+    // Same contract as send(): a response finishing after the connection
+    // died is dropped, never parked for the next connection's req ids.
+    if (!ws || ws.readyState !== WebSocket.OPEN) return;
+    const total = items.reduce((sum, item) => sum + item.bytes, 0);
+    if (bulkQueuedBytes + total > bulkQueueMaxBytes) {
+      warn(`bulk queue overflow (queued=${bulkQueuedBytes} incoming=${total} max=${bulkQueueMaxBytes}); dropping response for req ${reqId}`);
+      send({
+        type: 'res',
+        req_id: reqId,
+        status: 503,
+        headers: { 'content-type': 'text/plain; charset=utf-8' },
+        body_b64: Buffer.from('Tunnel backpressure: response dropped').toString('base64'),
+      });
+      return;
+    }
+    for (const item of items) bulkQueue.push(item);
+    bulkQueuedBytes += total;
+    drainBulk();
+  }
+
+  function drainBulk() {
+    while (bulkQueue.length > 0) {
+      if (!ws || ws.readyState !== WebSocket.OPEN) return;
+      if (ws.bufferedAmount > sendHwmBytes) {
+        // ws has no drain event — poll bufferedAmount, but only while blocked.
+        if (!drainTimer) {
+          drainTimer = setTimeout(() => {
+            drainTimer = null;
+            drainBulk();
+          }, DRAIN_POLL_MS);
+          if (typeof drainTimer.unref === 'function') drainTimer.unref();
+        }
+        return;
+      }
+      const item = bulkQueue.shift();
+      bulkQueuedBytes -= item.bytes;
+      sendPayload(item.build());
+    }
+  }
+
+  function resetBulkQueue(reason) {
+    if (drainTimer) {
+      clearTimeout(drainTimer);
+      drainTimer = null;
+    }
+    if (bulkQueue.length > 0) {
+      warn(`dropping ${bulkQueue.length} queued bulk frames (${bulkQueuedBytes} bytes): ${reason}`);
+    }
+    bulkQueue.length = 0;
+    bulkQueuedBytes = 0;
+  }
+
+  // Base64 output length for n raw bytes. Base64's alphabet needs no JSON
+  // escaping, so frame sizes computed with this are exact.
+  function base64Length(bytes) {
+    return Math.ceil(bytes / 3) * 4;
+  }
+
+  // Small bodies ship as today's single res frame, byte-identical. Big ones
+  // ride the bulk queue, and on v2 peers split into res_start/res_chunk/
+  // res_end so control frames interleave between chunks. Chunk frames are
+  // sized exactly but built lazily at send time (see payloadItem): the
+  // budget check runs before any frame exists, and an over-budget response
+  // is dropped without ever materializing its frames.
+  function sendResponse(reqId, status, headers, body) {
+    if (peerProtocolVersion < 2 || body.length <= RES_CHUNK_BYTES) {
+      // The v1-peer big-body case has no choice but to build the full frame:
+      // the v1 wire format is a single res frame.
+      const payload = JSON.stringify({
+        type: 'res',
+        req_id: reqId,
+        status,
+        headers,
+        body_b64: body.toString('base64'),
+      });
+      if (payload.length > BULK_FRAME_BYTES) enqueueBulk([payloadItem(payload)], reqId);
+      else sendPayload(payload);
+      return;
+    }
+
+    const items = [payloadItem(JSON.stringify({
+      type: 'res_start',
+      req_id: reqId,
+      status,
+      headers,
+      body_bytes: body.length,
+    }))];
+    let index = 0;
+    for (let offset = 0; offset < body.length; offset += RES_CHUNK_BYTES) {
+      const chunkIndex = index;
+      const chunkStart = offset;
+      const chunkLength = Math.min(RES_CHUNK_BYTES, body.length - offset);
+      const envelopeBytes = JSON.stringify({
+        type: 'res_chunk',
+        req_id: reqId,
+        index: chunkIndex,
+        body_b64: '',
+      }).length;
+      items.push({
+        bytes: envelopeBytes + base64Length(chunkLength),
+        build: () => JSON.stringify({
+          type: 'res_chunk',
+          req_id: reqId,
+          index: chunkIndex,
+          body_b64: body.subarray(chunkStart, chunkStart + RES_CHUNK_BYTES).toString('base64'),
+        }),
+      });
+      index += 1;
+    }
+    items.push(payloadItem(JSON.stringify({ type: 'res_end', req_id: reqId, chunks: index })));
+    enqueueBulk(items, reqId);
+  }
+
   function terminateSocket(reason) {
     if (!ws) return;
     warn(`${reason}; reconnecting`);
@@ -362,6 +548,23 @@ function createEventTunnel({ record, foreground = false }) {
     }
   }
 
+  function helloFrame() {
+    const appRoutes = readAppRoutes();
+    return {
+      type: 'hello',
+      app_version: require('../package.json').version,
+      runtime_version: require('../package.json').version,
+      runtime_label: record.runtime_label || AMALGM_RUNTIME_LABEL,
+      branch: record.branch || AMALGM_BRANCH,
+      instance_id: INSTANCE_ID,
+      boot_ts: BOOT_TS,
+      protocol_version: PROTOCOL_VERSION,
+      runtime_gateway_port: runtimeGatewayPort(),
+      app_routes: appRoutes,
+      artifact_routes: legacyArtifactRoutes(appRoutes),
+    };
+  }
+
   function startWatchdog() {
     if (watchdogTimer) return;
 
@@ -378,25 +581,25 @@ function createEventTunnel({ record, foreground = false }) {
 
       if (ws.readyState !== WebSocket.OPEN) return;
 
-      const appRoutes = readAppRoutes();
-      send({
-        type: 'hello',
-        app_version: require('../package.json').version,
-        runtime_label: record.runtime_label || AMALGM_RUNTIME_LABEL,
-        branch: record.branch || AMALGM_BRANCH,
-        runtime_gateway_port: runtimeGatewayPort(),
-        app_routes: appRoutes,
-        artifact_routes: legacyArtifactRoutes(appRoutes),
-      });
+      send(helloFrame());
 
       try {
         ws.ping();
+        lastPingSentAt = now;
       } catch {
         terminateSocket('ping failed');
         return;
       }
 
       if (lastGatewayFrameAt && now - lastGatewayFrameAt > HEARTBEAT_TIMEOUT_MS) {
+        // One structured line to tell apart "network went silent" (stale
+        // lastFrameIn/lastPong), "we stopped sending" (stale lastFrameOut),
+        // and a backpressure stall (buffered pinned high).
+        warn(
+          `heartbeat timeout: lastPong=${msAgo(lastPongAt, now)}, lastFrameIn=${msAgo(lastFrameReceivedAt, now)}, `
+          + `lastFrameOut=${msAgo(lastFrameSentAt, now)}, lastPingSent=${msAgo(lastPingSentAt, now)}, `
+          + `buffered=${ws.bufferedAmount}, bulkQueued=${bulkQueuedBytes}`,
+        );
         terminateSocket('heartbeat timeout');
       }
     }, WATCHDOG_INTERVAL_MS);
@@ -440,13 +643,12 @@ function createEventTunnel({ record, foreground = false }) {
   function handleRequest(frame) {
     const targetPort = resolveTargetPort(frame);
     if (!targetPort) {
-      send({
-        type: 'res',
-        req_id: frame.req_id,
-        status: 403,
-        headers: { 'content-type': 'application/json' },
-        body_b64: Buffer.from(JSON.stringify({ error: 'Target port is not allowed' })).toString('base64'),
-      });
+      sendResponse(
+        frame.req_id,
+        403,
+        { 'content-type': 'application/json' },
+        Buffer.from(JSON.stringify({ error: 'Target port is not allowed' })),
+      );
       return;
     }
     const body = frame.body_b64 ? Buffer.from(frame.body_b64, 'base64') : Buffer.alloc(0);
@@ -462,25 +664,18 @@ function createEventTunnel({ record, foreground = false }) {
         const chunks = [];
         res.on('data', (chunk) => chunks.push(Buffer.from(chunk)));
         res.on('end', () => {
-          send({
-            type: 'res',
-            req_id: frame.req_id,
-            status: res.statusCode || 502,
-            headers: responseHeaders(res.headers),
-            body_b64: Buffer.concat(chunks).toString('base64'),
-          });
+          sendResponse(frame.req_id, res.statusCode || 502, responseHeaders(res.headers), Buffer.concat(chunks));
         });
       },
     );
 
     req.on('error', (error) => {
-      send({
-        type: 'res',
-        req_id: frame.req_id,
-        status: 502,
-        headers: { 'content-type': 'text/plain; charset=utf-8' },
-        body_b64: Buffer.from(error.message || 'Local request failed').toString('base64'),
-      });
+      sendResponse(
+        frame.req_id,
+        502,
+        { 'content-type': 'text/plain; charset=utf-8' },
+        Buffer.from(error.message || 'Local request failed'),
+      );
     });
 
     req.end(body);
@@ -596,6 +791,10 @@ function createEventTunnel({ record, foreground = false }) {
 
   function handleFrame(frame) {
     if (!frame || typeof frame.type !== 'string') return;
+    if (frame.type === 'hello_ack') {
+      peerProtocolVersion = Number(frame.protocol_version) || 1;
+      return;
+    }
     if (frame.type === 'ping') {
       send({ type: 'pong' });
       return;
@@ -639,19 +838,20 @@ function createEventTunnel({ record, foreground = false }) {
     startWatchdog();
 
     ws.on('open', () => {
-      backoffMs = 1000;
+      // Deliberately NOT resetting backoffMs here: it resets in the close
+      // handler only once the connection has survived stableResetMs. An
+      // instant reset on open let two live runtimes ping-pong one gateway
+      // slot at ~1s cadence forever.
       connectStartedAt = 0;
-      lastGatewayFrameAt = Date.now();
-      log(`connected ${record.computer_id}`);
+      connectedAt = Date.now();
+      lastGatewayFrameAt = connectedAt;
+      lastPingSentAt = 0;
+      lastPongAt = 0;
+      lastFrameReceivedAt = 0;
+      lastFrameSentAt = 0;
+      log(`connected ${record.computer_id} instance=${INSTANCE_ID}`);
       void postPresence(record, true);
-      const appRoutes = readAppRoutes();
-      send({
-        type: 'hello',
-        app_version: require('../package.json').version,
-        runtime_gateway_port: runtimeGatewayPort(),
-        app_routes: appRoutes,
-        artifact_routes: legacyArtifactRoutes(appRoutes),
-      });
+      send(helloFrame());
       if (!routeTimer) {
         routeTimer = setInterval(advertiseApps, 30000);
         if (typeof routeTimer.unref === 'function') routeTimer.unref();
@@ -659,9 +859,11 @@ function createEventTunnel({ record, foreground = false }) {
     });
     ws.on('pong', () => {
       lastGatewayFrameAt = Date.now();
+      lastPongAt = lastGatewayFrameAt;
     });
     ws.on('message', (data) => {
       lastGatewayFrameAt = Date.now();
+      lastFrameReceivedAt = lastGatewayFrameAt;
       try {
         handleFrame(JSON.parse(data.toString()));
       } catch (error) {
@@ -675,14 +877,30 @@ function createEventTunnel({ record, foreground = false }) {
       }
     });
     ws.on('close', (code, reason) => {
+      const lifetimeMs = connectedAt ? Date.now() - connectedAt : 0;
       connectStartedAt = 0;
+      connectedAt = 0;
       lastGatewayFrameAt = 0;
-      // Streams and sockets belong to THIS gateway connection: their req/conn
-      // ids mean nothing to the next one, and its cancel frames will never
-      // reference them. Dropping them here is the only teardown they get —
-      // orphaned streams otherwise hold local connections open forever.
+      peerProtocolVersion = 1;
+      // Streams, sockets, and queued response frames belong to THIS gateway
+      // connection: their req/conn ids mean nothing to the next one, and its
+      // cancel frames will never reference them. Dropping them here is the
+      // only teardown they get — orphaned streams otherwise hold local
+      // connections open forever.
+      resetBulkQueue('event tunnel connection closed');
       dropActiveProxies('event tunnel connection closed');
-      warn(`closed code=${code} reason=${reason ? reason.toString() : ''}`);
+      warn(`closed code=${code} reason=${reason ? reason.toString() : ''} lifetime=${lifetimeMs}ms`);
+      // Backoff resets only after a connection proved stable; a short-lived
+      // connection keeps (and grows) the previous backoff so contention
+      // degrades to slow retries instead of a 1s war.
+      if (lifetimeMs >= stableResetMs) backoffMs = 1000;
+      if (code === SUPERSEDED_CLOSE_CODE) {
+        warn(
+          `superseded: gateway refused this instance (a newer runtime instance holds the slot). `
+          + `own instance=${INSTANCE_ID} boot_ts=${BOOT_TS}; retrying in ${supersededRetryMs}ms`,
+        );
+        backoffMs = supersededRetryMs;
+      }
       scheduleReconnect();
     });
     ws.on('error', (error) => {
@@ -716,6 +934,7 @@ function createEventTunnel({ record, foreground = false }) {
       if (reconnectTimer) clearTimeout(reconnectTimer);
       if (routeTimer) clearInterval(routeTimer);
       stopWatchdog();
+      resetBulkQueue('event tunnel stopped');
       dropActiveProxies('event tunnel stopped');
       void postPresence(record, false);
       if (ws) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "amalgm",
-  "version": "0.1.152",
+  "version": "0.1.153",
   "description": "Amalgm local computer runtime: login, MCP, chat, events, previews, and tunnels.",
   "license": "UNLICENSED",
   "private": false,

package/runtime/scripts/amalgm-mcp/server/routes/state.js

@@ -48,6 +48,10 @@ async function handleStateRoutes(ctx) {
       await stateRest.handleDocUpdate(await ctx.readJsonBody(), ctx.sendJson);
       return true;
     }
+    if (ctx.pathname === '/state/docs/write') {
+      await stateRest.handleDocWrite(await ctx.readJsonBody(), ctx.sendJson);
+      return true;
+    }
     return false;
   }
 

package/runtime/scripts/amalgm-mcp/state/app-resources.js

@@ -13,7 +13,7 @@
  * them into the in-memory registry on first use.
  */
 
-const { cachedStatement, openLocalDb } = require('./db');
+const { cachedStatement, ensureAppResourcesSchema, openLocalDb } = require('./db');
 const registry = require('./registry');
 const { insertStateEvent, publishStateEvent } = require('./events');
 
@@ -23,25 +23,6 @@ const VALID_OPS = new Set(['insert', 'update', 'delete', 'replace']);
 
 let loaded = false;
 
-function ensureSchema(database) {
-  database.exec(`
-    CREATE TABLE IF NOT EXISTS app_resources (
-      name TEXT PRIMARY KEY,
-      created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
-      updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
-    );
-
-    CREATE TABLE IF NOT EXISTS app_resource_rows (
-      resource TEXT NOT NULL,
-      id TEXT NOT NULL,
-      value_json TEXT NOT NULL,
-      updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
-      PRIMARY KEY (resource, id),
-      FOREIGN KEY (resource) REFERENCES app_resources(name) ON DELETE CASCADE
-    );
-  `);
-}
-
 function isValidAppResourceName(name) {
   return typeof name === 'string' && APP_RESOURCE_NAME_RE.test(name);
 }
@@ -71,7 +52,7 @@ function wireIntoRegistry(name) {
 function ensureAppResourcesLoaded() {
   if (loaded) return;
   const database = openLocalDb();
-  ensureSchema(database);
+  ensureAppResourcesSchema(database);
   const names = database.prepare('SELECT name FROM app_resources ORDER BY name ASC').all();
   for (const { name } of names) wireIntoRegistry(name);
   loaded = true;

package/runtime/scripts/amalgm-mcp/state/db.js

@@ -469,6 +469,54 @@ function migrate(database = openLocalDb()) {
   require('./migrations').runLegacyMigrations(database);
 }
 
+// ---------------------------------------------------------------------------
+// Lazily created tables. The DDL lives here so this file owns every schema
+// statement, but these tables are only created when their feature is first
+// touched — each ensure below is called from that first-use site (docs.js,
+// app-resources.js), preserving the original lazy-creation semantics.
+
+// Live documents (state/docs.js): persisted Y.Doc state per open file.
+function ensureDocStatesSchema(database = openLocalDb()) {
+  database.exec(`
+    CREATE TABLE IF NOT EXISTS doc_states (
+      path TEXT PRIMARY KEY,
+      state BLOB NOT NULL,
+      updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
+    );
+  `);
+  // last_disk_hash records the disk content as we last read or wrote it.
+  // On reopen it distinguishes "disk changed externally" (fold it in) from
+  // "disk is merely behind our persisted state" (write the doc back out) —
+  // without it, a restart would revert edits that hadn't hit the debounced
+  // disk write yet.
+  try {
+    database.exec('ALTER TABLE doc_states ADD COLUMN last_disk_hash TEXT');
+  } catch {
+    // Column already exists.
+  }
+}
+
+// App-registered resources (state/app-resources.js): the engine-side mirror
+// of rows that out-of-process apps push over the local-live rails.
+function ensureAppResourcesSchema(database = openLocalDb()) {
+  database.exec(`
+    CREATE TABLE IF NOT EXISTS app_resources (
+      name TEXT PRIMARY KEY,
+      created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
+      updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
+    );
+
+    CREATE TABLE IF NOT EXISTS app_resource_rows (
+      resource TEXT NOT NULL,
+      id TEXT NOT NULL,
+      value_json TEXT NOT NULL,
+      updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
+      PRIMARY KEY (resource, id),
+      FOREIGN KEY (resource) REFERENCES app_resources(name) ON DELETE CASCADE
+    );
+  `);
+}
+
 function closeLocalDb() {
   if (!db) return;
   db.close();
@@ -478,5 +526,7 @@ function closeLocalDb() {
 module.exports = {
   cachedStatement,
   closeLocalDb,
+  ensureAppResourcesSchema,
+  ensureDocStatesSchema,
   openLocalDb,
 };