@north-light/crouter 0.3.226 → 0.3.227

package/dist/daemon/api/__tests__/bridge-heartbeat.test.js

@@ -11,20 +11,32 @@ class FakeSocket extends EventEmitter {
 function sleep(ms) {
     return new Promise((resolve) => setTimeout(resolve, ms));
 }
-test('remote attach heartbeat pings an idle viewer and keeps a ponging observer alive', async () => {
+test('a quiet attach puts nothing on the wire', async () => {
+    // Regression: an unconditional 25s ping made a hosted guest's connection never
+    // idle, so the sandbox never suspended and whole fleets billed for 15h awake.
     const ws = new FakeSocket();
-    const stop = startRemoteAttachHeartbeat(ws, 20);
+    const heartbeat = startRemoteAttachHeartbeat(ws, 10, 15);
+    await sleep(60);
+    assert.equal(ws.pings, 1, 'only the first tick inside the quiet window pings');
+    assert.equal(ws.terminated, 0, 'a quiet attach is left open, never terminated by the heartbeat');
+    heartbeat.stop();
+});
+test('relayed traffic arms the heartbeat and a pong keeps a relaying observer alive', async () => {
+    const ws = new FakeSocket();
+    const heartbeat = startRemoteAttachHeartbeat(ws, 20, 10_000);
+    heartbeat.noteActivity();
     await sleep(25);
-    assert.equal(ws.pings, 1, 'an idle attach receives a WebSocket ping before proxy idle timeout');
+    assert.equal(ws.pings, 1, 'a relaying attach is pinged across a gap between frames');
     ws.emit('pong');
     await sleep(10);
     assert.equal(ws.terminated, 0, 'a pong renews the heartbeat without terminating the observer');
-    stop();
+    heartbeat.stop();
 });
-test('remote attach heartbeat drops only an unresponsive observer socket', async () => {
+test('an unresponsive peer is dropped only while the relay is active', async () => {
     const ws = new FakeSocket();
-    const stop = startRemoteAttachHeartbeat(ws, 5);
+    const heartbeat = startRemoteAttachHeartbeat(ws, 5, 10_000);
+    heartbeat.noteActivity();
     await sleep(16);
     assert.equal(ws.terminated, 1, 'two missed heartbeat intervals terminate the detached viewer');
-    stop();
+    heartbeat.stop();
 });

package/dist/daemon/api/bridge.d.ts

@@ -1,11 +1,42 @@
 import type { IncomingMessage } from 'node:http';
 import type { Duplex } from 'node:stream';
 import { type WebSocket } from 'ws';
-/** Keep an otherwise quiet observer connection alive through common 60s proxy
- * idle limits. A viewer only observes a broker, so a missed heartbeat may close
- * this socket but must never affect the broker or its in-flight work. */
+/** Keep a RELAYING connection alive across a gap between frames, through common
+ * 60s proxy idle limits. A viewer only observes a broker, so a missed heartbeat
+ * may close this socket but must never affect the broker or its in-flight work. */
 export declare const REMOTE_ATTACH_HEARTBEAT_MS = 25000;
-export declare function startRemoteAttachHeartbeat(ws: WebSocket, intervalMs?: number): () => void;
+/** How long after the last relayed frame the heartbeat keeps pinging. Past this,
+ * the attach is carrying nothing — no viewer output, no streaming node — and the
+ * heartbeat goes silent so the network path underneath can reap the connection.
+ *
+ * This matters far beyond a socket: on a hosted guest (Blaxel) a connection that
+ * is pinged every 25s is never idle, so the host never suspends the sandbox. An
+ * unconditional heartbeat held whole fleets awake for 15h at a time. A drop on a
+ * quiet attach is a normal close the remote peer re-attaches from, and real work
+ * is anchored by its own keep-alive, not by this ping.
+ *
+ * The clock starts when the attach OPENS, not at its first relayed frame: a
+ * never-relaying attach still gets pings until this window elapses. The welcome
+ * frame lands in milliseconds, so this only pads the tail of the very first
+ * window.
+ *
+ * ROLLOUT ORDER — this half is only half the answer, and shipping it FIRST is
+ * strictly worse than shipping nothing. The host that owns the attach must also
+ * let go of a quiet one; Northlight Core does that at its own 60s quiesce
+ * (`CROUTER_FOLLOW_QUIESCE_MS`). Against a host that holds the attach forever,
+ * going quiet here only moves the drop to the network path, the host reports a
+ * detach, its client reconnects, and the box wakes again — the same pin, now
+ * with a full re-snapshot every couple of minutes. Ship the host's quiesce
+ * first, then roll the guest image. */
+export declare const REMOTE_ATTACH_QUIET_MS = 60000;
+export interface RemoteAttachHeartbeat {
+    /** Stop the heartbeat permanently (teardown). Idempotent. */
+    stop: () => void;
+    /** Record a relayed frame in either direction — this is what keeps the
+     *  heartbeat armed, and what re-arms it after a quiet stretch. */
+    noteActivity: () => void;
+}
+export declare function startRemoteAttachHeartbeat(ws: WebSocket, intervalMs?: number, quietMs?: number): RemoteAttachHeartbeat;
 /** Handle the HTTP upgrade for `GET /v1/nodes/{id}/attach`. A-9 matches the
  *  route, extracts `{id}`, and calls this with the raw upgrade triplet; this
  *  owns the WS handshake (its own noServer `wss` above) AND the spec §5.3

package/dist/daemon/api/bridge.js

@@ -33,13 +33,38 @@ import { probeViewSocket, waitForBrokerViewSocket } from '../../core/runtime/vie
  *  client cap) so a remote peer cannot buffer 256MiB before the broker's 24MiB
  *  decoder rejects the message. */
 const wss = new WebSocketServer({ noServer: true, maxPayload: BROKER_READ_CAPS.maxLineBytes });
-/** Keep an otherwise quiet observer connection alive through common 60s proxy
- * idle limits. A viewer only observes a broker, so a missed heartbeat may close
- * this socket but must never affect the broker or its in-flight work. */
+/** Keep a RELAYING connection alive across a gap between frames, through common
+ * 60s proxy idle limits. A viewer only observes a broker, so a missed heartbeat
+ * may close this socket but must never affect the broker or its in-flight work. */
 export const REMOTE_ATTACH_HEARTBEAT_MS = 25_000;
-export function startRemoteAttachHeartbeat(ws, intervalMs = REMOTE_ATTACH_HEARTBEAT_MS) {
+/** How long after the last relayed frame the heartbeat keeps pinging. Past this,
+ * the attach is carrying nothing — no viewer output, no streaming node — and the
+ * heartbeat goes silent so the network path underneath can reap the connection.
+ *
+ * This matters far beyond a socket: on a hosted guest (Blaxel) a connection that
+ * is pinged every 25s is never idle, so the host never suspends the sandbox. An
+ * unconditional heartbeat held whole fleets awake for 15h at a time. A drop on a
+ * quiet attach is a normal close the remote peer re-attaches from, and real work
+ * is anchored by its own keep-alive, not by this ping.
+ *
+ * The clock starts when the attach OPENS, not at its first relayed frame: a
+ * never-relaying attach still gets pings until this window elapses. The welcome
+ * frame lands in milliseconds, so this only pads the tail of the very first
+ * window.
+ *
+ * ROLLOUT ORDER — this half is only half the answer, and shipping it FIRST is
+ * strictly worse than shipping nothing. The host that owns the attach must also
+ * let go of a quiet one; Northlight Core does that at its own 60s quiesce
+ * (`CROUTER_FOLLOW_QUIESCE_MS`). Against a host that holds the attach forever,
+ * going quiet here only moves the drop to the network path, the host reports a
+ * detach, its client reconnects, and the box wakes again — the same pin, now
+ * with a full re-snapshot every couple of minutes. Ship the host's quiesce
+ * first, then roll the guest image. */
+export const REMOTE_ATTACH_QUIET_MS = 60_000;
+export function startRemoteAttachHeartbeat(ws, intervalMs = REMOTE_ATTACH_HEARTBEAT_MS, quietMs = REMOTE_ATTACH_QUIET_MS) {
     let alive = true;
     let stopped = false;
+    let lastActivityAt = Date.now();
     const acknowledge = () => { alive = true; };
     const stop = () => {
         if (stopped)
@@ -50,6 +75,13 @@ export function startRemoteAttachHeartbeat(ws, intervalMs = REMOTE_ATTACH_HEARTB
     };
     ws.on('pong', acknowledge);
     const timer = setInterval(() => {
+        if (Date.now() - lastActivityAt > quietMs) {
+            // Nothing is flowing: send no ping and run no liveness check, so an idle
+            // attach puts zero packets on the wire. A dead peer here is closed by the
+            // network path or by teardown, not by us.
+            alive = true;
+            return;
+        }
         if (!alive) {
             stop();
             ws.terminate();
@@ -65,7 +97,15 @@ export function startRemoteAttachHeartbeat(ws, intervalMs = REMOTE_ATTACH_HEARTB
         }
     }, intervalMs);
     timer.unref();
-    return stop;
+    return {
+        stop,
+        // Frames are not liveness evidence: only a pong proves the peer is there.
+        // Marking a relayed frame alive would disable dead-peer detection for
+        // exactly as long as a turn streams — the one stretch it is for.
+        noteActivity: () => {
+            lastActivityAt = Date.now();
+        },
+    };
 }
 /** A WS close reason is capped at 123 UTF-8 bytes by the protocol. */
 function clampReason(reason) {
@@ -140,7 +180,7 @@ function bridgeConnection(ws, nodeId) {
     const decoder = new FrameDecoder(CLIENT_READ_CAPS);
     let closed = false;
     let viewerFaultRecorded = false;
-    let stopHeartbeat;
+    let heartbeat;
     socket.on('connect', () => {
         clearFault(nodeId, { link: 'relay↔broker' });
     });
@@ -148,7 +188,7 @@ function bridgeConnection(ws, nodeId) {
         if (closed)
             return;
         closed = true;
-        stopHeartbeat?.();
+        heartbeat?.stop();
         if (!socket.destroyed) {
             try {
                 socket.destroy();
@@ -164,9 +204,10 @@ function bridgeConnection(ws, nodeId) {
             /* ignore */
         }
     };
-    stopHeartbeat = startRemoteAttachHeartbeat(ws);
+    heartbeat = startRemoteAttachHeartbeat(ws);
     // socket → WS: bounded decode, one WS text message per complete frame.
     socket.on('data', (chunk) => {
+        heartbeat?.noteActivity();
         let frames;
         try {
             frames = decoder.push(chunk);
@@ -245,6 +286,7 @@ function bridgeConnection(ws, nodeId) {
     ws.on('message', (data) => {
         if (closed || socket.destroyed)
             return;
+        heartbeat?.noteActivity();
         const buf = Array.isArray(data)
             ? Buffer.concat(data)
             : Buffer.isBuffer(data)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@north-light/crouter",
-  "version": "0.3.226",
+  "version": "0.3.227",
   "description": "crtr — agent runtime with memory, plugins, and marketplaces",
   "type": "module",
   "main": "dist/index.js",

package/runtime.lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@north-light/crouter",
-  "version": "0.3.226",
+  "version": "0.3.227",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@north-light/crouter",
-      "version": "0.3.226",
+      "version": "0.3.227",
       "hasInstallScript": true,
       "license": "MIT",
       "dependencies": {