@ricsam/r5d-worker 0.0.90 → 0.0.92

package/README.md

@@ -41,7 +41,9 @@ Worker labels are mandatory and unique per user. Choose labels that describe hos
 
 The supervisor also reconnects automatically after server rollouts and transient network failures. A brief bounded delay prevents a tight retry loop while the server is unavailable. Pressing Ctrl+C still stops the worker, and agent-managed processes retain their existing in-process recovery path while disconnected.
 
-The server registry is the sole heartbeat timeout authority. The worker answers server pings but does not run an independent connection watchdog, because synchronous bounded workspace operations can temporarily occupy its event loop. The server keeps those operations leased for their bounded lifetime and closes genuinely stale connections after the lease ends.
+The runtime opens its control connection before recovering interrupted project snapshots. During recovery and initial project configuration it remains connected but non-dispatchable, reports its current phase plus branch/byte progress to the server, and prints the same progress locally. Recovery runs in a credential-sanitized helper process so the control loop can continue answering heartbeats. A durable `restoring` transaction state permits same-filesystem snapshots to be restored with one atomic directory move; cross-filesystem layouts retain the copy-and-fsync fallback.
+
+Both sides use monotonic, two-stage heartbeat recovery. The server sends pings, grants bounded leases to synchronous workspace and command-preparation work, and requires a fresh missed probe window before force-evicting a stale socket. The worker answers pings and uses a deliberately slower backup watchdog only for a half-open connection; reconnect delays are jittered across workers. A delayed watchdog sweep renews the probe instead of disconnecting, so synchronous local work or a stalled server event loop cannot turn one overdue timer callback into a mass disconnect.
 
 Credential-generation restart fencing protects against stale or accidentally detached descendants. It is not a hostile same-UID sandbox: code running as the worker's OS user can read or tamper with that user's config and auth files, install a persistent user service, or return after reboot. Run untrusted agents in a disposable VM that is reset at the boundary, or use a separately privileged credential broker/service account. When rotating the worker's own local `r5dctl` credential, stop the worker and reboot/reset the host before writing the replacement credential so an old same-UID descendant cannot read the new config.
 

package/dist/cjs/main.cjs

@@ -50,6 +50,7 @@ __export(main_exports, {
   syncSessionArtifacts: () => syncSessionArtifacts,
   workerChildProcessEnvironment: () => workerChildProcessEnvironment,
   workerGitSecurityTestHarness: () => workerGitSecurityTestHarness,
+  workerLifecycleTestHarness: () => workerLifecycleTestHarness,
   workerPtyBridgeTestHarness: () => workerPtyBridgeTestHarness,
   workerPtyTestHarness: () => workerPtyTestHarness,
   writeWorkerTextFile: () => writeWorkerTextFile
@@ -80,9 +81,25 @@ var import_repository_transition_policy = require("./repository-transition-polic
 var import_supervisor = require("./supervisor.cjs");
 var import_managed_paths = require("./managed-paths.cjs");
 var import_project_worktrees = require("./project-worktrees.cjs");
+var import_project_snapshot_recovery_runner = require("./project-snapshot-recovery-runner.cjs");
 var import_project_workspace_state = require("./project-workspace-state.cjs");
 var import_workspace_git_sync = require("./workspace-git-sync.cjs");
 const import_meta = {};
+const WORKER_LIFECYCLE_DETAIL_MAX_LENGTH = 500;
+const WORKER_SERVER_HEARTBEAT_INTERVAL_MS = 15e3;
+const WORKER_SERVER_HEARTBEAT_TIMEOUT_MS = 2 * 60 * 1e3;
+function boundedWorkerLifecycleProgress(progress) {
+  return progress.detail === void 0 ? progress : { ...progress, detail: progress.detail.slice(0, WORKER_LIFECYCLE_DETAIL_MAX_LENGTH) };
+}
+function workerServerHeartbeatAction(input) {
+  if (input.elapsedMs < input.timeoutMs) return "wait";
+  if (input.watchdogSweepDelayed || input.probeElapsedMs === void 0) return "start_probe";
+  return input.probeElapsedMs >= input.timeoutMs ? "terminate" : "wait";
+}
+const workerLifecycleTestHarness = {
+  boundedWorkerLifecycleProgress,
+  workerServerHeartbeatAction
+};
 class ProjectWorkspaceConfigurationDeferredError extends Error {
 }
 class WorkerServerUnavailableError extends Error {
@@ -2596,6 +2613,13 @@ async function executeCommand(input) {
   const startedAt = Date.now();
   let timeout;
   let spawnedProcess;
+  let preparationSettled = false;
+  const settlePreparation = () => {
+    if (preparationSettled) return;
+    preparationSettled = true;
+    input.setPreparationHeartbeatLease(false);
+  };
+  input.setPreparationHeartbeatLease(true);
   try {
     const targetProcessEnv = await prepareShellEnvForTarget({
       target: input.resolvedTarget.target,
@@ -2633,6 +2657,7 @@ async function executeCommand(input) {
       startedAt: (/* @__PURE__ */ new Date()).toISOString(),
       ...input.message.workspaceEffect === "none" ? { workspaceEffect: "none" } : {}
     });
+    settlePreparation();
     if (input.message.timeoutMs) {
       timeout = setTimeout(() => {
         void (0, import_process_tree.terminateProcessTree)(subprocess).catch((error) => {
@@ -2668,6 +2693,7 @@ async function executeCommand(input) {
       error: error instanceof Error ? error.message : String(error)
     };
   } finally {
+    settlePreparation();
     if (timeout) {
       clearTimeout(timeout);
     }
@@ -2683,6 +2709,13 @@ async function executeStreamingCommand(input) {
   let timedOut = false;
   let timeout;
   let spawnedProcess;
+  let preparationSettled = false;
+  const settlePreparation = () => {
+    if (preparationSettled) return;
+    preparationSettled = true;
+    input.setPreparationHeartbeatLease(false);
+  };
+  input.setPreparationHeartbeatLease(true);
   try {
     if (cancelledProcessRuns.delete(input.message.runId)) {
       sendWorkerMessage(input.ws, {
@@ -2735,6 +2768,7 @@ async function executeStreamingCommand(input) {
       ...interactive ? { interactive: true, stdin: subprocess.stdin } : {},
       ...input.message.workspaceEffect === "none" ? { workspaceEffect: "none" } : {}
     });
+    settlePreparation();
     started = true;
     sendWorkerMessage(input.ws, {
       type: "exec_started",
@@ -2811,6 +2845,7 @@ async function executeStreamingCommand(input) {
       });
     }
   } finally {
+    settlePreparation();
     if (timeout) {
       clearTimeout(timeout);
     }
@@ -3152,7 +3187,8 @@ function createNodePtyBridge(options) {
   };
 }
 const workerPtyBridgeTestHarness = {
-  create: createNodePtyBridge
+  create: createNodePtyBridge,
+  closeAllBestEffort: closePtysBestEffort
 };
 function resolveHostShell(command, platform = process.platform) {
   if (platform === "win32") {
@@ -3301,11 +3337,23 @@ function closePty(message) {
   }
   ptyProcess.kill();
 }
-function closeAllPtys() {
-  for (const ptyProcess of activePtys.values()) {
-    ptyProcess.kill();
+function closePtysBestEffort(ptys, reportFailure) {
+  for (const [ptyId, ptyProcess] of ptys) {
+    try {
+      ptyProcess.kill();
+    } catch (error) {
+      reportFailure(ptyId, error);
+    }
   }
 }
+function closeAllPtys() {
+  closePtysBestEffort(activePtys, (ptyId, error) => {
+    process.stderr.write(
+      `[r5d-worker] failed to close PTY ${ptyId} after control disconnect: ${error instanceof Error ? error.message : String(error)}
+`
+    );
+  });
+}
 async function terminateCredentialBearingChildren(children) {
   const results = await Promise.allSettled(children.map(async ({ terminate }) => await terminate()));
   const failures = results.flatMap(
@@ -3365,16 +3413,6 @@ function websocketUrl(baseUrl, label) {
 }
 async function startWorker(options) {
   advanceWorkerAdmissionGeneration();
-  process.on("uncaughtException", (error) => {
-    process.stderr.write(`[r5d-worker] uncaught exception: ${error instanceof Error ? error.stack ?? error.message : String(error)}
-`);
-  });
-  process.on("unhandledRejection", (reason) => {
-    process.stderr.write(
-      `[r5d-worker] unhandled rejection: ${reason instanceof Error ? reason.stack ?? reason.message : String(reason)}
-`
-    );
-  });
   const config = readConfig(options.configPath);
   const { baseUrl, token } = resolveCredentials(options, config);
   initializeCredentialStoreDirectoryOnce();
@@ -3407,28 +3445,6 @@ async function startWorker(options) {
   runGit(["--version"]);
   await verifyR5dctlAuth(baseUrl, token, label);
   const initializedWorkspaceState = await workspaceSyncSingleFlight.runExclusive(() => {
-    if (!startupProjectSnapshotRecoveryCompleted) {
-      const snapshotRecovery = (0, import_project_worktrees.recoverStaleProjectWorktreeSnapshots)({
-        projectsRoot,
-        temporaryRoot: projectWorktreeSnapshotsRoot
-      });
-      if (snapshotRecovery.restored.length > 0) {
-        process.stdout.write(`[r5d-worker] restored ${snapshotRecovery.restored.length} interrupted project snapshot(s)
-`);
-      }
-      if (snapshotRecovery.removed.length > 0) {
-        process.stdout.write(`[r5d-worker] removed ${snapshotRecovery.removed.length} completed/incomplete project snapshot(s)
-`);
-      }
-      for (const failure of snapshotRecovery.failed) {
-        process.stderr.write(`[r5d-worker] failed to recover stale project snapshot ${failure.path}: ${failure.error}
-`);
-      }
-      if (snapshotRecovery.failed.length > 0) {
-        throw new Error("Project snapshot recovery is incomplete; refusing to mutate project checkouts");
-      }
-      startupProjectSnapshotRecoveryCompleted = true;
-    }
     import_node_fs.default.mkdirSync(projectsRoot, { recursive: true });
     import_node_fs.default.mkdirSync(syncRoot, { recursive: true });
     import_node_fs.default.mkdirSync(artifactRoot, { recursive: true });
@@ -3467,6 +3483,8 @@ async function startWorker(options) {
   let cliUpdateInProgress = false;
   let reloadAfterClose = false;
   let shutdownAfterClose = false;
+  let startupRecoveryComplete = startupProjectSnapshotRecoveryCompleted;
+  let startupRecoveryPromise = Promise.resolve();
   const automaticWorkspaceSyncIsCurrentlyAdmitted = () => workspaceAutomaticSyncIsAdmitted({
     workspaceConfigured,
     activeIncidentId: activeWorkspaceIncidentId,
@@ -3883,8 +3901,14 @@ async function startWorker(options) {
       })
     )
   ].sort();
-  const ensureConfiguredProjects = async () => {
-    for (const project of projectConfigById.values()) {
+  const ensureConfiguredProjects = async (onProgress) => {
+    const projects = [...projectConfigById.values()];
+    for (const [projectIndex, project] of projects.entries()) {
+      onProgress?.({
+        detail: `Configuring ${project.projectPath}`,
+        completedItems: projectIndex,
+        totalItems: projects.length
+      });
       validateProjectId(project.projectId);
       (0, import_repository_transition_policy.assertRepositoryTransitionState)(project);
       for (const branch of project.branches) validateBranchName(branch.branchName);
@@ -3932,7 +3956,14 @@ async function startWorker(options) {
           mirrorUrl: connection.mirrorUrl,
           mirrorCredentialUsername: connection.mirrorCredentialUsername,
           credentialHelper: connection.credentialHelper,
-          gitIdentity: workspaceGitIdentity
+          gitIdentity: workspaceGitIdentity,
+          onSnapshotProgress: (progress) => onProgress?.({
+            detail: progress.phase === "scanning" ? `Measuring ${project.projectPath} workspace snapshot` : progress.phase === "syncing" ? `Making ${project.projectPath} workspace snapshot durable` : `${progress.operation === "restore" ? "Restoring" : "Snapshotting"} ${project.projectPath}${progress.branchName ? `/${progress.branchName}` : ""}`,
+            completedBytes: progress.completedBytes,
+            completedItems: progress.completedBranches,
+            totalItems: progress.totalBranches,
+            ...progress.totalBytes === void 0 ? {} : { totalBytes: progress.totalBytes }
+          })
         });
         readyProjectIds.add(project.projectId);
         reconciledProjectConfigFingerprints.set(project.projectId, configFingerprint);
@@ -3967,6 +3998,11 @@ async function startWorker(options) {
 `
         );
       }
+      onProgress?.({
+        detail: `Configured ${project.projectPath}`,
+        completedItems: projectIndex + 1,
+        totalItems: projects.length
+      });
     }
   };
   const collectAheadOfOriginBranches = async () => {
@@ -4359,6 +4395,11 @@ async function startWorker(options) {
       if (input.sendResult !== false) sendWorkspaceSyncResult(input.requestId, result2);
       return result2;
     }
+    sendLifecycleProgress({
+      phase: "syncing",
+      operationId: attemptId,
+      detail: input.trigger.detail ?? `Synchronizing workspace (${input.trigger.type})`
+    });
     if (!input.requestId && input.sendResult !== false) {
       if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
         throw new Error("Cannot start autonomous workspace synchronization without an open worker control socket");
@@ -4461,6 +4502,9 @@ async function startWorker(options) {
       }
     };
     if (input.sendResult !== false) sendWorkspaceSyncResult(input.requestId, result);
+    sendLifecycleProgress(
+      result.outcome === "failed" ? { phase: "failed", operationId: attemptId, detail: result.error ?? "Workspace synchronization failed" } : { phase: "ready", operationId: attemptId, detail: "Workspace synchronized" }
+    );
     return result;
   };
   const scheduleAutomaticWorkspaceSync = (trigger, delayMs = WORKSPACE_GIT_QUIET_MS) => {
@@ -4535,6 +4579,14 @@ async function startWorker(options) {
     });
   };
   const configureWorkerWorkspace = async (message, receiptGeneration) => {
+    const configurationOperationId = crypto.randomUUID();
+    sendLifecycleProgress({
+      phase: "configuring",
+      operationId: configurationOperationId,
+      detail: `Preparing ${message.projects.length} project(s)`,
+      completedItems: 0,
+      totalItems: message.projects.length
+    });
     const incidentDeferral = workspaceConfigurationIncidentDeferral({
       requestedIncidentId: message.deferWorkspaceSyncForIncidentId,
       activeIncidentId: activeWorkspaceIncidentId
@@ -4899,7 +4951,13 @@ async function startWorker(options) {
           // checkout initialization from erasing synchronized dirt.
           { ignoreBusy: true }
         );
-        await ensureConfiguredProjects();
+        await ensureConfiguredProjects((progress) => {
+          sendLifecycleProgress({ phase: "configuring", operationId: configurationOperationId, ...progress });
+          process.stdout.write(
+            `[r5d-worker] ${progress.detail ?? "Configuring workspace"}${progress.completedBytes === void 0 ? "" : `: ${progress.completedBytes}/${progress.totalBytes ?? "?"} bytes`}${progress.completedItems === void 0 ? "" : `, ${progress.completedItems}/${progress.totalItems ?? "?"} item(s)`}
+`
+          );
+        });
         const configurationSyncAdmissionGeneration = workerAdmissionGeneration;
         const assertConfigurationSyncStillAdmitted = () => {
           if (receiptGeneration !== workspaceConfigurationReceiptGeneration || configurationSyncAdmissionGeneration !== workerAdmissionGeneration || !workspaceConfigurationIncidentSnapshotIsCurrent(incidentDeferral.incidentId, activeWorkspaceIncidentId) || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
@@ -4998,6 +5056,73 @@ async function startWorker(options) {
       Authorization: `Bearer ${token}`
     }
   });
+  let lastServerControlMessageAtMs = performance.now();
+  let serverHeartbeatProbeStartedAtMs;
+  let lastServerHeartbeatSweepAtMs = performance.now();
+  let serverHeartbeatWatchdog;
+  const noteServerControlActivity = () => {
+    lastServerControlMessageAtMs = performance.now();
+    serverHeartbeatProbeStartedAtMs = void 0;
+  };
+  const stopServerHeartbeatWatchdog = () => {
+    if (!serverHeartbeatWatchdog) return;
+    clearInterval(serverHeartbeatWatchdog);
+    serverHeartbeatWatchdog = void 0;
+  };
+  const sendLifecycleProgress = (progress) => {
+    if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) return;
+    sendWorkerMessage(ws, { type: "worker_progress", progress: boundedWorkerLifecycleProgress(progress) });
+  };
+  const describeSnapshotProgress = (progress) => {
+    const project = import_node_path.default.relative(projectsRoot, progress.projectRoot) || import_node_path.default.basename(progress.projectRoot);
+    const branch = progress.branchName ? `/${progress.branchName}` : "";
+    if (progress.phase === "scanning") return `Scanning ${project} recovery snapshot`;
+    if (progress.phase === "syncing") return `Making restored ${project} files durable`;
+    if (progress.phase === "cleaning") return `Cleaning recovered ${project} snapshot`;
+    return `Restoring ${project}${branch}`;
+  };
+  const performStartupRecovery = async (operationId) => {
+    if (startupProjectSnapshotRecoveryCompleted) return;
+    sendLifecycleProgress({ phase: "recovering", operationId, detail: "Checking interrupted workspace transactions" });
+    const snapshotRecovery = await workspaceSyncSingleFlight.runExclusive(
+      async () => await (0, import_project_snapshot_recovery_runner.recoverProjectSnapshotsInChild)({
+        projectsRoot,
+        snapshotRoot: projectWorktreeSnapshotsRoot,
+        onProgress: (progress) => {
+          const lifecycleProgress = {
+            phase: "recovering",
+            operationId,
+            detail: describeSnapshotProgress(progress),
+            completedBytes: progress.completedBytes,
+            completedItems: progress.completedBranches,
+            totalItems: progress.totalBranches,
+            ...progress.totalBytes === void 0 ? {} : { totalBytes: progress.totalBytes }
+          };
+          sendLifecycleProgress(lifecycleProgress);
+          process.stdout.write(
+            `[r5d-worker] ${lifecycleProgress.detail}: ${progress.completedBranches}/${progress.totalBranches} branch(es), ${progress.completedBytes}/${progress.totalBytes ?? "?"} bytes
+`
+          );
+        }
+      })
+    );
+    if (snapshotRecovery.restored.length > 0) {
+      process.stdout.write(`[r5d-worker] restored ${snapshotRecovery.restored.length} interrupted project snapshot(s)
+`);
+    }
+    if (snapshotRecovery.removed.length > 0) {
+      process.stdout.write(`[r5d-worker] removed ${snapshotRecovery.removed.length} completed/incomplete project snapshot(s)
+`);
+    }
+    for (const failure of snapshotRecovery.failed) {
+      process.stderr.write(`[r5d-worker] failed to recover stale project snapshot ${failure.path}: ${failure.error}
+`);
+    }
+    if (snapshotRecovery.failed.length > 0) {
+      throw new Error("Project snapshot recovery is incomplete; refusing to mutate project checkouts");
+    }
+    startupProjectSnapshotRecoveryCompleted = true;
+  };
   const handleGracefulShutdown = () => {
     if (shutdownAfterClose) return;
     shutdownAfterClose = true;
@@ -5027,8 +5152,50 @@ async function startWorker(options) {
   process.on("SIGINT", handleGracefulShutdown);
   ws.addEventListener("open", () => {
     currentWorkerSocket = ws;
+    noteServerControlActivity();
+    lastServerHeartbeatSweepAtMs = performance.now();
+    serverHeartbeatWatchdog = setInterval(() => {
+      if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) return;
+      const nowMs = performance.now();
+      const heartbeatAction = workerServerHeartbeatAction({
+        elapsedMs: Math.max(0, nowMs - lastServerControlMessageAtMs),
+        ...serverHeartbeatProbeStartedAtMs === void 0 ? {} : { probeElapsedMs: Math.max(0, nowMs - serverHeartbeatProbeStartedAtMs) },
+        timeoutMs: WORKER_SERVER_HEARTBEAT_TIMEOUT_MS,
+        watchdogSweepDelayed: nowMs - lastServerHeartbeatSweepAtMs >= WORKER_SERVER_HEARTBEAT_INTERVAL_MS * 2
+      });
+      lastServerHeartbeatSweepAtMs = nowMs;
+      if (heartbeatAction === "start_probe") {
+        serverHeartbeatProbeStartedAtMs = nowMs;
+        return;
+      }
+      if (heartbeatAction === "terminate") {
+        stopServerHeartbeatWatchdog();
+        process.stderr.write("[r5d-worker] server control heartbeat timed out; reconnecting\n");
+        ws.terminate();
+      }
+    }, WORKER_SERVER_HEARTBEAT_INTERVAL_MS);
+    serverHeartbeatWatchdog.unref();
+    const startupOperationId = crypto.randomUUID();
+    startupRecoveryPromise = Promise.resolve().then(async () => await performStartupRecovery(startupOperationId)).then(() => {
+      startupRecoveryComplete = true;
+      if (currentWorkerSocket === ws && ws.readyState === WebSocket.OPEN) {
+        sendWorkerMessage(ws, { type: "startup_ready" });
+        sendLifecycleProgress({
+          phase: "configuring",
+          operationId: crypto.randomUUID(),
+          detail: "Waiting for workspace configuration"
+        });
+      }
+    }).catch((error) => {
+      const message = error instanceof Error ? error.message : String(error);
+      sendLifecycleProgress({ phase: "failed", operationId: startupOperationId, detail: message });
+      process.stderr.write(`[r5d-worker] startup recovery failed: ${message}
+`);
+      ws.close(1011, "Project snapshot recovery failed");
+    });
     const hello = {
       type: "hello",
+      startupReady: startupRecoveryComplete,
       hostInfo: {
         hostname: (0, import_node_os.hostname)(),
         platform: process.platform,
@@ -5064,8 +5231,11 @@ async function startWorker(options) {
     process.stdout.write("[r5d-worker] connected\n");
   });
   ws.addEventListener("message", (event) => {
+    noteServerControlActivity();
     void (async () => {
       const message = JSON.parse(String(event.data));
+      if (!startupRecoveryComplete && message.type !== "ping") await startupRecoveryPromise;
+      if (!startupRecoveryComplete && message.type !== "ping") return;
       const messageAdmissionGeneration = workerAdmissionGeneration;
       const assertMessageAdmission = () => assertWorkerChildAdmission({
         capturedGeneration: messageAdmissionGeneration,
@@ -5093,6 +5263,19 @@ async function startWorker(options) {
           aheadOfOriginBranches: configured.aheadOfOriginBranches,
           ...configured.deferredWorkspaceSyncForIncidentId ? { deferredWorkspaceSyncForIncidentId: configured.deferredWorkspaceSyncForIncidentId } : {}
         });
+        sendLifecycleProgress(
+          configured.result.outcome === "failed" ? {
+            phase: "failed",
+            operationId: configured.result.attemptId,
+            detail: configured.result.error ?? "Workspace configuration failed"
+          } : {
+            phase: "ready",
+            operationId: configured.result.attemptId,
+            detail: `${message.projects.length} project(s) configured`,
+            completedItems: message.projects.length,
+            totalItems: message.projects.length
+          }
+        );
         process.stdout.write(
           `[r5d-worker] workspace configured: ${message.projects.length} project(s), ${configured.pending.length} pending checkout(s)
 `
@@ -5656,6 +5839,13 @@ async function startWorker(options) {
               token,
               artifactRoot,
               planRoot,
+              setPreparationHeartbeatLease: (active) => {
+                sendWorkerMessageFromCurrentSource(ws, {
+                  type: "heartbeat_lease",
+                  operationId: `exec:${message.runId}`,
+                  active
+                });
+              },
               assertAdmission: () => {
                 if (!workspaceCommandTransportIsAllowed(deferredWorkspaceConfiguration, activeWorkspaceIncidentId, "exec")) {
                   throw new Error(
@@ -5719,6 +5909,13 @@ async function startWorker(options) {
               token,
               artifactRoot,
               planRoot,
+              setPreparationHeartbeatLease: (active) => {
+                sendWorkerMessageFromCurrentSource(ws, {
+                  type: "heartbeat_lease",
+                  operationId: `exec:${message.runId}`,
+                  active
+                });
+              },
               assertAdmission: assertMessageAdmission
             });
           } finally {
@@ -5851,6 +6048,7 @@ async function startWorker(options) {
     const processEvents = process;
     processEvents.off("SIGTERM", handleGracefulShutdown);
     processEvents.off("SIGINT", handleGracefulShutdown);
+    stopServerHeartbeatWatchdog();
     if (workspaceAutomaticTimer) {
       clearTimeout(workspaceAutomaticTimer);
       workspaceAutomaticTimer = void 0;
@@ -5881,23 +6079,47 @@ async function startWorker(options) {
     if (shutdownAfterClose) {
       process.exit(0);
     }
+    if (!startupRecoveryComplete) {
+      process.stderr.write("[r5d-worker] control connection closed during workspace recovery; finishing recovery before reconnecting\n");
+      void startupRecoveryPromise.then(() => {
+        if (!startupRecoveryComplete) {
+          process.exit(1);
+          return;
+        }
+        const reconnectDelayMs = (0, import_supervisor.workerReconnectDelayMs)();
+        process.stderr.write(`[r5d-worker] workspace recovery complete; reconnecting in ${reconnectDelayMs}ms
+`);
+        setTimeout(() => {
+          void startWorker(options).catch((error) => {
+            process.stderr.write(`[r5d-worker] reconnect failed: ${error instanceof Error ? error.message : String(error)}
+`);
+            process.exit(error instanceof WorkerServerUnavailableError ? import_supervisor.WORKER_RECONNECT_EXIT_CODE : 1);
+          });
+        }, reconnectDelayMs);
+      });
+      return;
+    }
     if (activeProcesses.size > 0 || credentialBearingProcessGroups.size > 0 || activePtys.size > 0 || pendingProcessTerminals.size > 0) {
-      process.stderr.write(
-        `[r5d-worker] ${activeProcesses.size} process(es), ${credentialBearingProcessGroups.size} owned process group(s), and ${activePtys.size} PTY(s) active with ${pendingProcessTerminals.size} terminal report(s) pending; reconnecting in ${import_supervisor.WORKER_RECONNECT_DELAY_MS}ms
+      const scheduleReconnect = (reconnect2) => {
+        const reconnectDelayMs = (0, import_supervisor.workerReconnectDelayMs)();
+        process.stderr.write(
+          `[r5d-worker] ${activeProcesses.size} process(es), ${credentialBearingProcessGroups.size} owned process group(s), and ${activePtys.size} PTY(s) active with ${pendingProcessTerminals.size} terminal report(s) pending; reconnecting in ${reconnectDelayMs}ms
 `
-      );
+        );
+        setTimeout(reconnect2, reconnectDelayMs);
+      };
       const reconnect = () => {
         void startWorker(options).catch((error) => {
           process.stderr.write(`[r5d-worker] reconnect failed: ${error instanceof Error ? error.message : String(error)}
 `);
           if (activeProcesses.size > 0 || credentialBearingProcessGroups.size > 0 || activePtys.size > 0 || pendingProcessTerminals.size > 0) {
-            setTimeout(reconnect, import_supervisor.WORKER_RECONNECT_DELAY_MS);
+            scheduleReconnect(reconnect);
             return;
           }
           process.exit(error instanceof WorkerServerUnavailableError ? import_supervisor.WORKER_RECONNECT_EXIT_CODE : 1);
         });
       };
-      setTimeout(reconnect, import_supervisor.WORKER_RECONNECT_DELAY_MS);
+      scheduleReconnect(reconnect);
       return;
     }
     process.exit(import_supervisor.WORKER_RECONNECT_EXIT_CODE);
@@ -5907,10 +6129,28 @@ async function startWorker(options) {
     process.stderr.write(`[r5d-worker] websocket error${message}
 `);
   });
-  await new Promise(() => {
+  await new Promise((resolve) => {
+    ws.addEventListener("close", () => resolve(), { once: true });
+  });
+}
+function installWorkerRuntimeFailureLogging() {
+  process.on("uncaughtException", (error) => {
+    process.stderr.write(`[r5d-worker] uncaught exception: ${error instanceof Error ? error.stack ?? error.message : String(error)}
+`);
+  });
+  process.on("unhandledRejection", (reason) => {
+    process.stderr.write(
+      `[r5d-worker] unhandled rejection: ${reason instanceof Error ? reason.stack ?? reason.message : String(reason)}
+`
+    );
   });
 }
 async function main() {
+  if (process.argv[2] === import_project_snapshot_recovery_runner.PROJECT_SNAPSHOT_RECOVERY_HELPER_COMMAND) {
+    const exitCode2 = (0, import_project_snapshot_recovery_runner.runProjectSnapshotRecoveryHelper)(process.argv.slice(3));
+    if (exitCode2 !== 0) process.exit(exitCode2);
+    return;
+  }
   const parsed = parseArgs(process.argv.slice(2));
   if (parsed.command === "help") {
     printHelp();
@@ -5922,6 +6162,7 @@ async function main() {
   }
   if (process.env[import_supervisor.WORKER_RUNTIME_ENV] === "1") {
     delete process.env[import_supervisor.WORKER_RUNTIME_ENV];
+    installWorkerRuntimeFailureLogging();
     try {
       await startWorker(parsed.options);
     } catch (error) {
@@ -5975,6 +6216,7 @@ if (isCliEntrypoint()) {
   syncSessionArtifacts,
   workerChildProcessEnvironment,
   workerGitSecurityTestHarness,
+  workerLifecycleTestHarness,
   workerPtyBridgeTestHarness,
   workerPtyTestHarness,
   writeWorkerTextFile

package/dist/cjs/package.json

@@ -1,5 +1,5 @@
 {
   "name": "@ricsam/r5d-worker",
-  "version": "0.0.90",
+  "version": "0.0.92",
   "type": "commonjs"
 }