@ricsam/r5d-worker 0.0.91 → 0.0.92

package/README.md

@@ -43,7 +43,7 @@ The supervisor also reconnects automatically after server rollouts and transient
 
 The runtime opens its control connection before recovering interrupted project snapshots. During recovery and initial project configuration it remains connected but non-dispatchable, reports its current phase plus branch/byte progress to the server, and prints the same progress locally. Recovery runs in a credential-sanitized helper process so the control loop can continue answering heartbeats. A durable `restoring` transaction state permits same-filesystem snapshots to be restored with one atomic directory move; cross-filesystem layouts retain the copy-and-fsync fallback.
 
-The server registry is the sole heartbeat timeout authority. The worker answers server pings but does not run an independent connection watchdog, because synchronous bounded workspace operations can temporarily occupy its event loop. The server keeps those operations leased for their bounded lifetime and closes genuinely stale connections after the lease ends.
+Both sides use monotonic, two-stage heartbeat recovery. The server sends pings, grants bounded leases to synchronous workspace and command-preparation work, and requires a fresh missed probe window before force-evicting a stale socket. The worker answers pings and uses a deliberately slower backup watchdog only for a half-open connection; reconnect delays are jittered across workers. A delayed watchdog sweep renews the probe instead of disconnecting, so synchronous local work or a stalled server event loop cannot turn one overdue timer callback into a mass disconnect.
 
 Credential-generation restart fencing protects against stale or accidentally detached descendants. It is not a hostile same-UID sandbox: code running as the worker's OS user can read or tamper with that user's config and auth files, install a persistent user service, or return after reboot. Run untrusted agents in a disposable VM that is reset at the boundary, or use a separately privileged credential broker/service account. When rotating the worker's own local `r5dctl` credential, stop the worker and reboot/reset the host before writing the replacement credential so an old same-UID descendant cannot read the new config.
 

package/dist/cjs/main.cjs

@@ -50,6 +50,7 @@ __export(main_exports, {
   syncSessionArtifacts: () => syncSessionArtifacts,
   workerChildProcessEnvironment: () => workerChildProcessEnvironment,
   workerGitSecurityTestHarness: () => workerGitSecurityTestHarness,
+  workerLifecycleTestHarness: () => workerLifecycleTestHarness,
   workerPtyBridgeTestHarness: () => workerPtyBridgeTestHarness,
   workerPtyTestHarness: () => workerPtyTestHarness,
   writeWorkerTextFile: () => writeWorkerTextFile
@@ -84,6 +85,21 @@ var import_project_snapshot_recovery_runner = require("./project-snapshot-recove
 var import_project_workspace_state = require("./project-workspace-state.cjs");
 var import_workspace_git_sync = require("./workspace-git-sync.cjs");
 const import_meta = {};
+const WORKER_LIFECYCLE_DETAIL_MAX_LENGTH = 500;
+const WORKER_SERVER_HEARTBEAT_INTERVAL_MS = 15e3;
+const WORKER_SERVER_HEARTBEAT_TIMEOUT_MS = 2 * 60 * 1e3;
+function boundedWorkerLifecycleProgress(progress) {
+  return progress.detail === void 0 ? progress : { ...progress, detail: progress.detail.slice(0, WORKER_LIFECYCLE_DETAIL_MAX_LENGTH) };
+}
+function workerServerHeartbeatAction(input) {
+  if (input.elapsedMs < input.timeoutMs) return "wait";
+  if (input.watchdogSweepDelayed || input.probeElapsedMs === void 0) return "start_probe";
+  return input.probeElapsedMs >= input.timeoutMs ? "terminate" : "wait";
+}
+const workerLifecycleTestHarness = {
+  boundedWorkerLifecycleProgress,
+  workerServerHeartbeatAction
+};
 class ProjectWorkspaceConfigurationDeferredError extends Error {
 }
 class WorkerServerUnavailableError extends Error {
@@ -2597,6 +2613,13 @@ async function executeCommand(input) {
   const startedAt = Date.now();
   let timeout;
   let spawnedProcess;
+  let preparationSettled = false;
+  const settlePreparation = () => {
+    if (preparationSettled) return;
+    preparationSettled = true;
+    input.setPreparationHeartbeatLease(false);
+  };
+  input.setPreparationHeartbeatLease(true);
   try {
     const targetProcessEnv = await prepareShellEnvForTarget({
       target: input.resolvedTarget.target,
@@ -2634,6 +2657,7 @@ async function executeCommand(input) {
       startedAt: (/* @__PURE__ */ new Date()).toISOString(),
       ...input.message.workspaceEffect === "none" ? { workspaceEffect: "none" } : {}
     });
+    settlePreparation();
     if (input.message.timeoutMs) {
       timeout = setTimeout(() => {
         void (0, import_process_tree.terminateProcessTree)(subprocess).catch((error) => {
@@ -2669,6 +2693,7 @@ async function executeCommand(input) {
       error: error instanceof Error ? error.message : String(error)
     };
   } finally {
+    settlePreparation();
     if (timeout) {
       clearTimeout(timeout);
     }
@@ -2684,6 +2709,13 @@ async function executeStreamingCommand(input) {
   let timedOut = false;
   let timeout;
   let spawnedProcess;
+  let preparationSettled = false;
+  const settlePreparation = () => {
+    if (preparationSettled) return;
+    preparationSettled = true;
+    input.setPreparationHeartbeatLease(false);
+  };
+  input.setPreparationHeartbeatLease(true);
   try {
     if (cancelledProcessRuns.delete(input.message.runId)) {
       sendWorkerMessage(input.ws, {
@@ -2736,6 +2768,7 @@ async function executeStreamingCommand(input) {
       ...interactive ? { interactive: true, stdin: subprocess.stdin } : {},
       ...input.message.workspaceEffect === "none" ? { workspaceEffect: "none" } : {}
     });
+    settlePreparation();
     started = true;
     sendWorkerMessage(input.ws, {
       type: "exec_started",
@@ -2812,6 +2845,7 @@ async function executeStreamingCommand(input) {
       });
     }
   } finally {
+    settlePreparation();
     if (timeout) {
       clearTimeout(timeout);
     }
@@ -3153,7 +3187,8 @@ function createNodePtyBridge(options) {
   };
 }
 const workerPtyBridgeTestHarness = {
-  create: createNodePtyBridge
+  create: createNodePtyBridge,
+  closeAllBestEffort: closePtysBestEffort
 };
 function resolveHostShell(command, platform = process.platform) {
   if (platform === "win32") {
@@ -3302,11 +3337,23 @@ function closePty(message) {
   }
   ptyProcess.kill();
 }
-function closeAllPtys() {
-  for (const ptyProcess of activePtys.values()) {
-    ptyProcess.kill();
+function closePtysBestEffort(ptys, reportFailure) {
+  for (const [ptyId, ptyProcess] of ptys) {
+    try {
+      ptyProcess.kill();
+    } catch (error) {
+      reportFailure(ptyId, error);
+    }
   }
 }
+function closeAllPtys() {
+  closePtysBestEffort(activePtys, (ptyId, error) => {
+    process.stderr.write(
+      `[r5d-worker] failed to close PTY ${ptyId} after control disconnect: ${error instanceof Error ? error.message : String(error)}
+`
+    );
+  });
+}
 async function terminateCredentialBearingChildren(children) {
   const results = await Promise.allSettled(children.map(async ({ terminate }) => await terminate()));
   const failures = results.flatMap(
@@ -3366,16 +3413,6 @@ function websocketUrl(baseUrl, label) {
 }
 async function startWorker(options) {
   advanceWorkerAdmissionGeneration();
-  process.on("uncaughtException", (error) => {
-    process.stderr.write(`[r5d-worker] uncaught exception: ${error instanceof Error ? error.stack ?? error.message : String(error)}
-`);
-  });
-  process.on("unhandledRejection", (reason) => {
-    process.stderr.write(
-      `[r5d-worker] unhandled rejection: ${reason instanceof Error ? reason.stack ?? reason.message : String(reason)}
-`
-    );
-  });
   const config = readConfig(options.configPath);
   const { baseUrl, token } = resolveCredentials(options, config);
   initializeCredentialStoreDirectoryOnce();
@@ -5019,9 +5056,22 @@ async function startWorker(options) {
       Authorization: `Bearer ${token}`
     }
   });
+  let lastServerControlMessageAtMs = performance.now();
+  let serverHeartbeatProbeStartedAtMs;
+  let lastServerHeartbeatSweepAtMs = performance.now();
+  let serverHeartbeatWatchdog;
+  const noteServerControlActivity = () => {
+    lastServerControlMessageAtMs = performance.now();
+    serverHeartbeatProbeStartedAtMs = void 0;
+  };
+  const stopServerHeartbeatWatchdog = () => {
+    if (!serverHeartbeatWatchdog) return;
+    clearInterval(serverHeartbeatWatchdog);
+    serverHeartbeatWatchdog = void 0;
+  };
   const sendLifecycleProgress = (progress) => {
     if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) return;
-    sendWorkerMessage(ws, { type: "worker_progress", progress });
+    sendWorkerMessage(ws, { type: "worker_progress", progress: boundedWorkerLifecycleProgress(progress) });
   };
   const describeSnapshotProgress = (progress) => {
     const project = import_node_path.default.relative(projectsRoot, progress.projectRoot) || import_node_path.default.basename(progress.projectRoot);
@@ -5102,6 +5152,29 @@ async function startWorker(options) {
   process.on("SIGINT", handleGracefulShutdown);
   ws.addEventListener("open", () => {
     currentWorkerSocket = ws;
+    noteServerControlActivity();
+    lastServerHeartbeatSweepAtMs = performance.now();
+    serverHeartbeatWatchdog = setInterval(() => {
+      if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) return;
+      const nowMs = performance.now();
+      const heartbeatAction = workerServerHeartbeatAction({
+        elapsedMs: Math.max(0, nowMs - lastServerControlMessageAtMs),
+        ...serverHeartbeatProbeStartedAtMs === void 0 ? {} : { probeElapsedMs: Math.max(0, nowMs - serverHeartbeatProbeStartedAtMs) },
+        timeoutMs: WORKER_SERVER_HEARTBEAT_TIMEOUT_MS,
+        watchdogSweepDelayed: nowMs - lastServerHeartbeatSweepAtMs >= WORKER_SERVER_HEARTBEAT_INTERVAL_MS * 2
+      });
+      lastServerHeartbeatSweepAtMs = nowMs;
+      if (heartbeatAction === "start_probe") {
+        serverHeartbeatProbeStartedAtMs = nowMs;
+        return;
+      }
+      if (heartbeatAction === "terminate") {
+        stopServerHeartbeatWatchdog();
+        process.stderr.write("[r5d-worker] server control heartbeat timed out; reconnecting\n");
+        ws.terminate();
+      }
+    }, WORKER_SERVER_HEARTBEAT_INTERVAL_MS);
+    serverHeartbeatWatchdog.unref();
     const startupOperationId = crypto.randomUUID();
     startupRecoveryPromise = Promise.resolve().then(async () => await performStartupRecovery(startupOperationId)).then(() => {
       startupRecoveryComplete = true;
@@ -5158,6 +5231,7 @@ async function startWorker(options) {
     process.stdout.write("[r5d-worker] connected\n");
   });
   ws.addEventListener("message", (event) => {
+    noteServerControlActivity();
     void (async () => {
       const message = JSON.parse(String(event.data));
       if (!startupRecoveryComplete && message.type !== "ping") await startupRecoveryPromise;
@@ -5765,6 +5839,13 @@ async function startWorker(options) {
               token,
               artifactRoot,
               planRoot,
+              setPreparationHeartbeatLease: (active) => {
+                sendWorkerMessageFromCurrentSource(ws, {
+                  type: "heartbeat_lease",
+                  operationId: `exec:${message.runId}`,
+                  active
+                });
+              },
               assertAdmission: () => {
                 if (!workspaceCommandTransportIsAllowed(deferredWorkspaceConfiguration, activeWorkspaceIncidentId, "exec")) {
                   throw new Error(
@@ -5828,6 +5909,13 @@ async function startWorker(options) {
               token,
               artifactRoot,
               planRoot,
+              setPreparationHeartbeatLease: (active) => {
+                sendWorkerMessageFromCurrentSource(ws, {
+                  type: "heartbeat_lease",
+                  operationId: `exec:${message.runId}`,
+                  active
+                });
+              },
               assertAdmission: assertMessageAdmission
             });
           } finally {
@@ -5960,6 +6048,7 @@ async function startWorker(options) {
     const processEvents = process;
     processEvents.off("SIGTERM", handleGracefulShutdown);
     processEvents.off("SIGINT", handleGracefulShutdown);
+    stopServerHeartbeatWatchdog();
     if (workspaceAutomaticTimer) {
       clearTimeout(workspaceAutomaticTimer);
       workspaceAutomaticTimer = void 0;
@@ -5997,31 +6086,40 @@ async function startWorker(options) {
           process.exit(1);
           return;
         }
-        void startWorker(options).catch((error) => {
-          process.stderr.write(`[r5d-worker] reconnect failed: ${error instanceof Error ? error.message : String(error)}
+        const reconnectDelayMs = (0, import_supervisor.workerReconnectDelayMs)();
+        process.stderr.write(`[r5d-worker] workspace recovery complete; reconnecting in ${reconnectDelayMs}ms
 `);
-          process.exit(error instanceof WorkerServerUnavailableError ? import_supervisor.WORKER_RECONNECT_EXIT_CODE : 1);
-        });
+        setTimeout(() => {
+          void startWorker(options).catch((error) => {
+            process.stderr.write(`[r5d-worker] reconnect failed: ${error instanceof Error ? error.message : String(error)}
+`);
+            process.exit(error instanceof WorkerServerUnavailableError ? import_supervisor.WORKER_RECONNECT_EXIT_CODE : 1);
+          });
+        }, reconnectDelayMs);
       });
       return;
     }
     if (activeProcesses.size > 0 || credentialBearingProcessGroups.size > 0 || activePtys.size > 0 || pendingProcessTerminals.size > 0) {
-      process.stderr.write(
-        `[r5d-worker] ${activeProcesses.size} process(es), ${credentialBearingProcessGroups.size} owned process group(s), and ${activePtys.size} PTY(s) active with ${pendingProcessTerminals.size} terminal report(s) pending; reconnecting in ${import_supervisor.WORKER_RECONNECT_DELAY_MS}ms
+      const scheduleReconnect = (reconnect2) => {
+        const reconnectDelayMs = (0, import_supervisor.workerReconnectDelayMs)();
+        process.stderr.write(
+          `[r5d-worker] ${activeProcesses.size} process(es), ${credentialBearingProcessGroups.size} owned process group(s), and ${activePtys.size} PTY(s) active with ${pendingProcessTerminals.size} terminal report(s) pending; reconnecting in ${reconnectDelayMs}ms
 `
-      );
+        );
+        setTimeout(reconnect2, reconnectDelayMs);
+      };
       const reconnect = () => {
         void startWorker(options).catch((error) => {
           process.stderr.write(`[r5d-worker] reconnect failed: ${error instanceof Error ? error.message : String(error)}
 `);
           if (activeProcesses.size > 0 || credentialBearingProcessGroups.size > 0 || activePtys.size > 0 || pendingProcessTerminals.size > 0) {
-            setTimeout(reconnect, import_supervisor.WORKER_RECONNECT_DELAY_MS);
+            scheduleReconnect(reconnect);
             return;
           }
           process.exit(error instanceof WorkerServerUnavailableError ? import_supervisor.WORKER_RECONNECT_EXIT_CODE : 1);
         });
       };
-      setTimeout(reconnect, import_supervisor.WORKER_RECONNECT_DELAY_MS);
+      scheduleReconnect(reconnect);
       return;
     }
     process.exit(import_supervisor.WORKER_RECONNECT_EXIT_CODE);
@@ -6031,7 +6129,20 @@ async function startWorker(options) {
     process.stderr.write(`[r5d-worker] websocket error${message}
 `);
   });
-  await new Promise(() => {
+  await new Promise((resolve) => {
+    ws.addEventListener("close", () => resolve(), { once: true });
+  });
+}
+function installWorkerRuntimeFailureLogging() {
+  process.on("uncaughtException", (error) => {
+    process.stderr.write(`[r5d-worker] uncaught exception: ${error instanceof Error ? error.stack ?? error.message : String(error)}
+`);
+  });
+  process.on("unhandledRejection", (reason) => {
+    process.stderr.write(
+      `[r5d-worker] unhandled rejection: ${reason instanceof Error ? reason.stack ?? reason.message : String(reason)}
+`
+    );
   });
 }
 async function main() {
@@ -6051,6 +6162,7 @@ async function main() {
   }
   if (process.env[import_supervisor.WORKER_RUNTIME_ENV] === "1") {
     delete process.env[import_supervisor.WORKER_RUNTIME_ENV];
+    installWorkerRuntimeFailureLogging();
     try {
       await startWorker(parsed.options);
     } catch (error) {
@@ -6104,6 +6216,7 @@ if (isCliEntrypoint()) {
   syncSessionArtifacts,
   workerChildProcessEnvironment,
   workerGitSecurityTestHarness,
+  workerLifecycleTestHarness,
   workerPtyBridgeTestHarness,
   workerPtyTestHarness,
   writeWorkerTextFile

package/dist/cjs/package.json

@@ -1,5 +1,5 @@
 {
   "name": "@ricsam/r5d-worker",
-  "version": "0.0.91",
+  "version": "0.0.92",
   "type": "commonjs"
 }

package/dist/cjs/supervisor.cjs

@@ -21,10 +21,12 @@ __export(supervisor_exports, {
   WORKER_CREDENTIAL_RESTART_EXIT_CODE: () => WORKER_CREDENTIAL_RESTART_EXIT_CODE,
   WORKER_RECONNECT_DELAY_MS: () => WORKER_RECONNECT_DELAY_MS,
   WORKER_RECONNECT_EXIT_CODE: () => WORKER_RECONNECT_EXIT_CODE,
+  WORKER_RECONNECT_JITTER_MS: () => WORKER_RECONNECT_JITTER_MS,
   WORKER_RELOAD_EXIT_CODE: () => WORKER_RELOAD_EXIT_CODE,
   WORKER_RUNTIME_ENV: () => WORKER_RUNTIME_ENV,
   isRetryableWorkerServerStatus: () => isRetryableWorkerServerStatus,
-  superviseWorkerRuntime: () => superviseWorkerRuntime
+  superviseWorkerRuntime: () => superviseWorkerRuntime,
+  workerReconnectDelayMs: () => workerReconnectDelayMs
 });
 module.exports = __toCommonJS(supervisor_exports);
 var import_node_os = require("node:os");
@@ -34,6 +36,7 @@ const WORKER_RELOAD_EXIT_CODE = 75;
 const WORKER_RECONNECT_EXIT_CODE = 76;
 const WORKER_CREDENTIAL_RESTART_EXIT_CODE = 77;
 const WORKER_RECONNECT_DELAY_MS = 2e3;
+const WORKER_RECONNECT_JITTER_MS = 3e3;
 const SUPERVISOR_SIGNALS = ["SIGINT", "SIGTERM", "SIGHUP"];
 function defaultRuntimeSpawner(argv, options) {
   return Bun.spawn(argv, options);
@@ -41,6 +44,10 @@ function defaultRuntimeSpawner(argv, options) {
 function defaultRuntimeDelay(delayMs) {
   return new Promise((resolve) => setTimeout(resolve, delayMs));
 }
+function workerReconnectDelayMs(random = Math.random) {
+  const sample = Math.max(0, Math.min(1, random()));
+  return WORKER_RECONNECT_DELAY_MS + Math.floor(sample * WORKER_RECONNECT_JITTER_MS);
+}
 async function defaultRuntimeReaper(runtime) {
   await (0, import_process_tree.terminateProcessTree)(runtime);
 }
@@ -53,7 +60,7 @@ function signalExitCode(signal) {
 function osSignalNumber(signal) {
   return import_node_os.constants.signals[signal];
 }
-async function superviseWorkerRuntime(argv, env = process.env, spawnRuntime = defaultRuntimeSpawner, waitBeforeReconnect = defaultRuntimeDelay, reapCredentialRuntime = defaultRuntimeReaper) {
+async function superviseWorkerRuntime(argv, env = process.env, spawnRuntime = defaultRuntimeSpawner, waitBeforeReconnect = defaultRuntimeDelay, reapCredentialRuntime = defaultRuntimeReaper, reconnectDelay = workerReconnectDelayMs) {
   if (!process.argv[1]) {
     throw new Error("Cannot locate the r5d-worker entrypoint");
   }
@@ -105,9 +112,10 @@ async function superviseWorkerRuntime(argv, env = process.env, spawnRuntime = de
         continue;
       }
       if (exitCode === WORKER_RECONNECT_EXIT_CODE) {
-        process.stderr.write(`[r5d-worker] reconnecting in ${WORKER_RECONNECT_DELAY_MS}ms
+        const reconnectDelayMs = reconnectDelay();
+        process.stderr.write(`[r5d-worker] reconnecting in ${reconnectDelayMs}ms
 `);
-        await waitBeforeReconnect(WORKER_RECONNECT_DELAY_MS);
+        await waitBeforeReconnect(reconnectDelayMs);
         if (receivedSignal) return signalExitCode(receivedSignal);
         continue;
       }
@@ -123,8 +131,10 @@ async function superviseWorkerRuntime(argv, env = process.env, spawnRuntime = de
   WORKER_CREDENTIAL_RESTART_EXIT_CODE,
   WORKER_RECONNECT_DELAY_MS,
   WORKER_RECONNECT_EXIT_CODE,
+  WORKER_RECONNECT_JITTER_MS,
   WORKER_RELOAD_EXIT_CODE,
   WORKER_RUNTIME_ENV,
   isRetryableWorkerServerStatus,
-  superviseWorkerRuntime
+  superviseWorkerRuntime,
+  workerReconnectDelayMs
 });

package/dist/mjs/main.mjs

@@ -49,10 +49,10 @@ import {
   isRetryableWorkerServerStatus,
   superviseWorkerRuntime,
   WORKER_CREDENTIAL_RESTART_EXIT_CODE,
-  WORKER_RECONNECT_DELAY_MS,
   WORKER_RECONNECT_EXIT_CODE,
   WORKER_RELOAD_EXIT_CODE,
-  WORKER_RUNTIME_ENV
+  WORKER_RUNTIME_ENV,
+  workerReconnectDelayMs
 } from "./supervisor.mjs";
 import { assertDisjointManagedRoots, managedProjectRoot, validateManagedBranchName } from "./managed-paths.mjs";
 import {
@@ -91,6 +91,21 @@ import {
   workspaceGitMountsForRemediation,
   WorkspaceRemediationAncestryError
 } from "./workspace-git-sync.mjs";
+const WORKER_LIFECYCLE_DETAIL_MAX_LENGTH = 500;
+const WORKER_SERVER_HEARTBEAT_INTERVAL_MS = 15e3;
+const WORKER_SERVER_HEARTBEAT_TIMEOUT_MS = 2 * 60 * 1e3;
+function boundedWorkerLifecycleProgress(progress) {
+  return progress.detail === void 0 ? progress : { ...progress, detail: progress.detail.slice(0, WORKER_LIFECYCLE_DETAIL_MAX_LENGTH) };
+}
+function workerServerHeartbeatAction(input) {
+  if (input.elapsedMs < input.timeoutMs) return "wait";
+  if (input.watchdogSweepDelayed || input.probeElapsedMs === void 0) return "start_probe";
+  return input.probeElapsedMs >= input.timeoutMs ? "terminate" : "wait";
+}
+const workerLifecycleTestHarness = {
+  boundedWorkerLifecycleProgress,
+  workerServerHeartbeatAction
+};
 class ProjectWorkspaceConfigurationDeferredError extends Error {
 }
 class WorkerServerUnavailableError extends Error {
@@ -2604,6 +2619,13 @@ async function executeCommand(input) {
   const startedAt = Date.now();
   let timeout;
   let spawnedProcess;
+  let preparationSettled = false;
+  const settlePreparation = () => {
+    if (preparationSettled) return;
+    preparationSettled = true;
+    input.setPreparationHeartbeatLease(false);
+  };
+  input.setPreparationHeartbeatLease(true);
   try {
     const targetProcessEnv = await prepareShellEnvForTarget({
       target: input.resolvedTarget.target,
@@ -2641,6 +2663,7 @@ async function executeCommand(input) {
       startedAt: (/* @__PURE__ */ new Date()).toISOString(),
       ...input.message.workspaceEffect === "none" ? { workspaceEffect: "none" } : {}
     });
+    settlePreparation();
     if (input.message.timeoutMs) {
       timeout = setTimeout(() => {
         void terminateProcessTree(subprocess).catch((error) => {
@@ -2676,6 +2699,7 @@ async function executeCommand(input) {
       error: error instanceof Error ? error.message : String(error)
     };
   } finally {
+    settlePreparation();
     if (timeout) {
       clearTimeout(timeout);
     }
@@ -2691,6 +2715,13 @@ async function executeStreamingCommand(input) {
   let timedOut = false;
   let timeout;
   let spawnedProcess;
+  let preparationSettled = false;
+  const settlePreparation = () => {
+    if (preparationSettled) return;
+    preparationSettled = true;
+    input.setPreparationHeartbeatLease(false);
+  };
+  input.setPreparationHeartbeatLease(true);
   try {
     if (cancelledProcessRuns.delete(input.message.runId)) {
       sendWorkerMessage(input.ws, {
@@ -2743,6 +2774,7 @@ async function executeStreamingCommand(input) {
       ...interactive ? { interactive: true, stdin: subprocess.stdin } : {},
       ...input.message.workspaceEffect === "none" ? { workspaceEffect: "none" } : {}
     });
+    settlePreparation();
     started = true;
     sendWorkerMessage(input.ws, {
       type: "exec_started",
@@ -2819,6 +2851,7 @@ async function executeStreamingCommand(input) {
       });
     }
   } finally {
+    settlePreparation();
     if (timeout) {
       clearTimeout(timeout);
     }
@@ -3160,7 +3193,8 @@ function createNodePtyBridge(options) {
   };
 }
 const workerPtyBridgeTestHarness = {
-  create: createNodePtyBridge
+  create: createNodePtyBridge,
+  closeAllBestEffort: closePtysBestEffort
 };
 function resolveHostShell(command, platform = process.platform) {
   if (platform === "win32") {
@@ -3309,11 +3343,23 @@ function closePty(message) {
   }
   ptyProcess.kill();
 }
-function closeAllPtys() {
-  for (const ptyProcess of activePtys.values()) {
-    ptyProcess.kill();
+function closePtysBestEffort(ptys, reportFailure) {
+  for (const [ptyId, ptyProcess] of ptys) {
+    try {
+      ptyProcess.kill();
+    } catch (error) {
+      reportFailure(ptyId, error);
+    }
   }
 }
+function closeAllPtys() {
+  closePtysBestEffort(activePtys, (ptyId, error) => {
+    process.stderr.write(
+      `[r5d-worker] failed to close PTY ${ptyId} after control disconnect: ${error instanceof Error ? error.message : String(error)}
+`
+    );
+  });
+}
 async function terminateCredentialBearingChildren(children) {
   const results = await Promise.allSettled(children.map(async ({ terminate }) => await terminate()));
   const failures = results.flatMap(
@@ -3373,16 +3419,6 @@ function websocketUrl(baseUrl, label) {
 }
 async function startWorker(options) {
   advanceWorkerAdmissionGeneration();
-  process.on("uncaughtException", (error) => {
-    process.stderr.write(`[r5d-worker] uncaught exception: ${error instanceof Error ? error.stack ?? error.message : String(error)}
-`);
-  });
-  process.on("unhandledRejection", (reason) => {
-    process.stderr.write(
-      `[r5d-worker] unhandled rejection: ${reason instanceof Error ? reason.stack ?? reason.message : String(reason)}
-`
-    );
-  });
   const config = readConfig(options.configPath);
   const { baseUrl, token } = resolveCredentials(options, config);
   initializeCredentialStoreDirectoryOnce();
@@ -5026,9 +5062,22 @@ async function startWorker(options) {
       Authorization: `Bearer ${token}`
     }
   });
+  let lastServerControlMessageAtMs = performance.now();
+  let serverHeartbeatProbeStartedAtMs;
+  let lastServerHeartbeatSweepAtMs = performance.now();
+  let serverHeartbeatWatchdog;
+  const noteServerControlActivity = () => {
+    lastServerControlMessageAtMs = performance.now();
+    serverHeartbeatProbeStartedAtMs = void 0;
+  };
+  const stopServerHeartbeatWatchdog = () => {
+    if (!serverHeartbeatWatchdog) return;
+    clearInterval(serverHeartbeatWatchdog);
+    serverHeartbeatWatchdog = void 0;
+  };
   const sendLifecycleProgress = (progress) => {
     if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) return;
-    sendWorkerMessage(ws, { type: "worker_progress", progress });
+    sendWorkerMessage(ws, { type: "worker_progress", progress: boundedWorkerLifecycleProgress(progress) });
   };
   const describeSnapshotProgress = (progress) => {
     const project = path.relative(projectsRoot, progress.projectRoot) || path.basename(progress.projectRoot);
@@ -5109,6 +5158,29 @@ async function startWorker(options) {
   process.on("SIGINT", handleGracefulShutdown);
   ws.addEventListener("open", () => {
     currentWorkerSocket = ws;
+    noteServerControlActivity();
+    lastServerHeartbeatSweepAtMs = performance.now();
+    serverHeartbeatWatchdog = setInterval(() => {
+      if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) return;
+      const nowMs = performance.now();
+      const heartbeatAction = workerServerHeartbeatAction({
+        elapsedMs: Math.max(0, nowMs - lastServerControlMessageAtMs),
+        ...serverHeartbeatProbeStartedAtMs === void 0 ? {} : { probeElapsedMs: Math.max(0, nowMs - serverHeartbeatProbeStartedAtMs) },
+        timeoutMs: WORKER_SERVER_HEARTBEAT_TIMEOUT_MS,
+        watchdogSweepDelayed: nowMs - lastServerHeartbeatSweepAtMs >= WORKER_SERVER_HEARTBEAT_INTERVAL_MS * 2
+      });
+      lastServerHeartbeatSweepAtMs = nowMs;
+      if (heartbeatAction === "start_probe") {
+        serverHeartbeatProbeStartedAtMs = nowMs;
+        return;
+      }
+      if (heartbeatAction === "terminate") {
+        stopServerHeartbeatWatchdog();
+        process.stderr.write("[r5d-worker] server control heartbeat timed out; reconnecting\n");
+        ws.terminate();
+      }
+    }, WORKER_SERVER_HEARTBEAT_INTERVAL_MS);
+    serverHeartbeatWatchdog.unref();
     const startupOperationId = crypto.randomUUID();
     startupRecoveryPromise = Promise.resolve().then(async () => await performStartupRecovery(startupOperationId)).then(() => {
       startupRecoveryComplete = true;
@@ -5165,6 +5237,7 @@ async function startWorker(options) {
     process.stdout.write("[r5d-worker] connected\n");
   });
   ws.addEventListener("message", (event) => {
+    noteServerControlActivity();
     void (async () => {
       const message = JSON.parse(String(event.data));
       if (!startupRecoveryComplete && message.type !== "ping") await startupRecoveryPromise;
@@ -5772,6 +5845,13 @@ async function startWorker(options) {
               token,
               artifactRoot,
               planRoot,
+              setPreparationHeartbeatLease: (active) => {
+                sendWorkerMessageFromCurrentSource(ws, {
+                  type: "heartbeat_lease",
+                  operationId: `exec:${message.runId}`,
+                  active
+                });
+              },
               assertAdmission: () => {
                 if (!workspaceCommandTransportIsAllowed(deferredWorkspaceConfiguration, activeWorkspaceIncidentId, "exec")) {
                   throw new Error(
@@ -5835,6 +5915,13 @@ async function startWorker(options) {
               token,
               artifactRoot,
               planRoot,
+              setPreparationHeartbeatLease: (active) => {
+                sendWorkerMessageFromCurrentSource(ws, {
+                  type: "heartbeat_lease",
+                  operationId: `exec:${message.runId}`,
+                  active
+                });
+              },
               assertAdmission: assertMessageAdmission
             });
           } finally {
@@ -5967,6 +6054,7 @@ async function startWorker(options) {
     const processEvents = process;
     processEvents.off("SIGTERM", handleGracefulShutdown);
     processEvents.off("SIGINT", handleGracefulShutdown);
+    stopServerHeartbeatWatchdog();
     if (workspaceAutomaticTimer) {
       clearTimeout(workspaceAutomaticTimer);
       workspaceAutomaticTimer = void 0;
@@ -6004,31 +6092,40 @@ async function startWorker(options) {
           process.exit(1);
           return;
         }
-        void startWorker(options).catch((error) => {
-          process.stderr.write(`[r5d-worker] reconnect failed: ${error instanceof Error ? error.message : String(error)}
+        const reconnectDelayMs = workerReconnectDelayMs();
+        process.stderr.write(`[r5d-worker] workspace recovery complete; reconnecting in ${reconnectDelayMs}ms
 `);
-          process.exit(error instanceof WorkerServerUnavailableError ? WORKER_RECONNECT_EXIT_CODE : 1);
-        });
+        setTimeout(() => {
+          void startWorker(options).catch((error) => {
+            process.stderr.write(`[r5d-worker] reconnect failed: ${error instanceof Error ? error.message : String(error)}
+`);
+            process.exit(error instanceof WorkerServerUnavailableError ? WORKER_RECONNECT_EXIT_CODE : 1);
+          });
+        }, reconnectDelayMs);
       });
       return;
     }
     if (activeProcesses.size > 0 || credentialBearingProcessGroups.size > 0 || activePtys.size > 0 || pendingProcessTerminals.size > 0) {
-      process.stderr.write(
-        `[r5d-worker] ${activeProcesses.size} process(es), ${credentialBearingProcessGroups.size} owned process group(s), and ${activePtys.size} PTY(s) active with ${pendingProcessTerminals.size} terminal report(s) pending; reconnecting in ${WORKER_RECONNECT_DELAY_MS}ms
+      const scheduleReconnect = (reconnect2) => {
+        const reconnectDelayMs = workerReconnectDelayMs();
+        process.stderr.write(
+          `[r5d-worker] ${activeProcesses.size} process(es), ${credentialBearingProcessGroups.size} owned process group(s), and ${activePtys.size} PTY(s) active with ${pendingProcessTerminals.size} terminal report(s) pending; reconnecting in ${reconnectDelayMs}ms
 `
-      );
+        );
+        setTimeout(reconnect2, reconnectDelayMs);
+      };
       const reconnect = () => {
         void startWorker(options).catch((error) => {
           process.stderr.write(`[r5d-worker] reconnect failed: ${error instanceof Error ? error.message : String(error)}
 `);
           if (activeProcesses.size > 0 || credentialBearingProcessGroups.size > 0 || activePtys.size > 0 || pendingProcessTerminals.size > 0) {
-            setTimeout(reconnect, WORKER_RECONNECT_DELAY_MS);
+            scheduleReconnect(reconnect);
             return;
           }
           process.exit(error instanceof WorkerServerUnavailableError ? WORKER_RECONNECT_EXIT_CODE : 1);
         });
       };
-      setTimeout(reconnect, WORKER_RECONNECT_DELAY_MS);
+      scheduleReconnect(reconnect);
       return;
     }
     process.exit(WORKER_RECONNECT_EXIT_CODE);
@@ -6038,7 +6135,20 @@ async function startWorker(options) {
     process.stderr.write(`[r5d-worker] websocket error${message}
 `);
   });
-  await new Promise(() => {
+  await new Promise((resolve) => {
+    ws.addEventListener("close", () => resolve(), { once: true });
+  });
+}
+function installWorkerRuntimeFailureLogging() {
+  process.on("uncaughtException", (error) => {
+    process.stderr.write(`[r5d-worker] uncaught exception: ${error instanceof Error ? error.stack ?? error.message : String(error)}
+`);
+  });
+  process.on("unhandledRejection", (reason) => {
+    process.stderr.write(
+      `[r5d-worker] unhandled rejection: ${reason instanceof Error ? reason.stack ?? reason.message : String(reason)}
+`
+    );
   });
 }
 async function main() {
@@ -6058,6 +6168,7 @@ async function main() {
   }
   if (process.env[WORKER_RUNTIME_ENV] === "1") {
     delete process.env[WORKER_RUNTIME_ENV];
+    installWorkerRuntimeFailureLogging();
     try {
       await startWorker(parsed.options);
     } catch (error) {
@@ -6110,6 +6221,7 @@ export {
   syncSessionArtifacts,
   workerChildProcessEnvironment,
   workerGitSecurityTestHarness,
+  workerLifecycleTestHarness,
   workerPtyBridgeTestHarness,
   workerPtyTestHarness,
   writeWorkerTextFile

package/dist/mjs/package.json

@@ -1,5 +1,5 @@
 {
   "name": "@ricsam/r5d-worker",
-  "version": "0.0.91",
+  "version": "0.0.92",
   "type": "module"
 }

package/dist/mjs/supervisor.mjs

@@ -5,6 +5,7 @@ const WORKER_RELOAD_EXIT_CODE = 75;
 const WORKER_RECONNECT_EXIT_CODE = 76;
 const WORKER_CREDENTIAL_RESTART_EXIT_CODE = 77;
 const WORKER_RECONNECT_DELAY_MS = 2e3;
+const WORKER_RECONNECT_JITTER_MS = 3e3;
 const SUPERVISOR_SIGNALS = ["SIGINT", "SIGTERM", "SIGHUP"];
 function defaultRuntimeSpawner(argv, options) {
   return Bun.spawn(argv, options);
@@ -12,6 +13,10 @@ function defaultRuntimeSpawner(argv, options) {
 function defaultRuntimeDelay(delayMs) {
   return new Promise((resolve) => setTimeout(resolve, delayMs));
 }
+function workerReconnectDelayMs(random = Math.random) {
+  const sample = Math.max(0, Math.min(1, random()));
+  return WORKER_RECONNECT_DELAY_MS + Math.floor(sample * WORKER_RECONNECT_JITTER_MS);
+}
 async function defaultRuntimeReaper(runtime) {
   await terminateProcessTree(runtime);
 }
@@ -24,7 +29,7 @@ function signalExitCode(signal) {
 function osSignalNumber(signal) {
   return osConstants.signals[signal];
 }
-async function superviseWorkerRuntime(argv, env = process.env, spawnRuntime = defaultRuntimeSpawner, waitBeforeReconnect = defaultRuntimeDelay, reapCredentialRuntime = defaultRuntimeReaper) {
+async function superviseWorkerRuntime(argv, env = process.env, spawnRuntime = defaultRuntimeSpawner, waitBeforeReconnect = defaultRuntimeDelay, reapCredentialRuntime = defaultRuntimeReaper, reconnectDelay = workerReconnectDelayMs) {
   if (!process.argv[1]) {
     throw new Error("Cannot locate the r5d-worker entrypoint");
   }
@@ -76,9 +81,10 @@ async function superviseWorkerRuntime(argv, env = process.env, spawnRuntime = de
         continue;
       }
       if (exitCode === WORKER_RECONNECT_EXIT_CODE) {
-        process.stderr.write(`[r5d-worker] reconnecting in ${WORKER_RECONNECT_DELAY_MS}ms
+        const reconnectDelayMs = reconnectDelay();
+        process.stderr.write(`[r5d-worker] reconnecting in ${reconnectDelayMs}ms
 `);
-        await waitBeforeReconnect(WORKER_RECONNECT_DELAY_MS);
+        await waitBeforeReconnect(reconnectDelayMs);
         if (receivedSignal) return signalExitCode(receivedSignal);
         continue;
       }
@@ -93,8 +99,10 @@ export {
   WORKER_CREDENTIAL_RESTART_EXIT_CODE,
   WORKER_RECONNECT_DELAY_MS,
   WORKER_RECONNECT_EXIT_CODE,
+  WORKER_RECONNECT_JITTER_MS,
   WORKER_RELOAD_EXIT_CODE,
   WORKER_RUNTIME_ENV,
   isRetryableWorkerServerStatus,
-  superviseWorkerRuntime
+  superviseWorkerRuntime,
+  workerReconnectDelayMs
 };

package/dist/types/main.d.ts

@@ -23,6 +23,26 @@ type WorkerProjectConfig = {
         baseCommitHash: string;
     }>;
 };
+type WorkerLifecycleProgress = {
+    phase: "starting" | "recovering" | "configuring" | "syncing" | "ready" | "failed";
+    operationId: string;
+    detail?: string;
+    completedBytes?: number;
+    totalBytes?: number;
+    completedItems?: number;
+    totalItems?: number;
+};
+declare function boundedWorkerLifecycleProgress(progress: WorkerLifecycleProgress): WorkerLifecycleProgress;
+declare function workerServerHeartbeatAction(input: {
+    elapsedMs: number;
+    probeElapsedMs?: number;
+    timeoutMs: number;
+    watchdogSweepDelayed: boolean;
+}): "wait" | "start_probe" | "terminate";
+export declare const workerLifecycleTestHarness: {
+    boundedWorkerLifecycleProgress: typeof boundedWorkerLifecycleProgress;
+    workerServerHeartbeatAction: typeof workerServerHeartbeatAction;
+};
 type WorkspaceSyncTrigger = {
     type: "connect" | "inbound_head" | "write" | "edit" | "shell_inline" | "process_terminal" | "process_cancel" | "periodic" | "manual" | "remediation" | "remediation_confirm" | "remediation_reset";
     sessionId?: string;
@@ -813,12 +833,16 @@ declare function createNodePtyBridge(options: {
 }): WorkerPty;
 export declare const workerPtyBridgeTestHarness: {
     create: typeof createNodePtyBridge;
+    closeAllBestEffort: typeof closePtysBestEffort;
 };
 export declare function resolveHostShell(command?: string, platform?: NodeJS.Platform): {
     file: string;
     args: string[];
     env?: Record<string, string>;
 };
+declare function closePtysBestEffort(ptys: Iterable<readonly [string, {
+    kill: () => void;
+}]>, reportFailure: (ptyId: string, error: unknown) => void): void;
 declare function terminateCredentialBearingChildren(children: readonly {
     label: string;
     terminate: () => void | Promise<void>;

package/dist/types/supervisor.d.ts

@@ -4,6 +4,7 @@ export declare const WORKER_RELOAD_EXIT_CODE = 75;
 export declare const WORKER_RECONNECT_EXIT_CODE = 76;
 export declare const WORKER_CREDENTIAL_RESTART_EXIT_CODE = 77;
 export declare const WORKER_RECONNECT_DELAY_MS = 2000;
+export declare const WORKER_RECONNECT_JITTER_MS = 3000;
 type RuntimeProcess = KillableSubprocess;
 export type RuntimeSpawner = (argv: string[], options: {
     stdin: "inherit";
@@ -14,6 +15,8 @@ export type RuntimeSpawner = (argv: string[], options: {
 }) => RuntimeProcess;
 export type RuntimeDelay = (delayMs: number) => Promise<void>;
 export type RuntimeReaper = (runtime: RuntimeProcess) => Promise<void>;
+export type RuntimeReconnectDelay = () => number;
+export declare function workerReconnectDelayMs(random?: () => number): number;
 export declare function isRetryableWorkerServerStatus(status: number): boolean;
-export declare function superviseWorkerRuntime(argv: string[], env?: Record<string, string | undefined>, spawnRuntime?: RuntimeSpawner, waitBeforeReconnect?: RuntimeDelay, reapCredentialRuntime?: RuntimeReaper): Promise<number>;
+export declare function superviseWorkerRuntime(argv: string[], env?: Record<string, string | undefined>, spawnRuntime?: RuntimeSpawner, waitBeforeReconnect?: RuntimeDelay, reapCredentialRuntime?: RuntimeReaper, reconnectDelay?: RuntimeReconnectDelay): Promise<number>;
 export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ricsam/r5d-worker",
-  "version": "0.0.91",
+  "version": "0.0.92",
   "type": "module",
   "main": "./dist/cjs/main.cjs",
   "module": "./dist/mjs/main.mjs",