@ricsam/r5d-worker 0.0.90 → 0.0.92

package/dist/mjs/main.mjs

@@ -49,16 +49,15 @@ import {
   isRetryableWorkerServerStatus,
   superviseWorkerRuntime,
   WORKER_CREDENTIAL_RESTART_EXIT_CODE,
-  WORKER_RECONNECT_DELAY_MS,
   WORKER_RECONNECT_EXIT_CODE,
   WORKER_RELOAD_EXIT_CODE,
-  WORKER_RUNTIME_ENV
+  WORKER_RUNTIME_ENV,
+  workerReconnectDelayMs
 } from "./supervisor.mjs";
 import { assertDisjointManagedRoots, managedProjectRoot, validateManagedBranchName } from "./managed-paths.mjs";
 import {
   applyObservedProjectMirrorHeads,
   assertProjectMirrorHeadsPushed,
-  recoverStaleProjectWorktreeSnapshots,
   createOrRetryLinkedProjectBranch,
   deleteLinkedProjectBranch,
   deleteProjectMirrorBranch,
@@ -71,6 +70,11 @@ import {
   pushProjectMirrorHeads,
   removeProjectWorktrees
 } from "./project-worktrees.mjs";
+import {
+  PROJECT_SNAPSHOT_RECOVERY_HELPER_COMMAND,
+  recoverProjectSnapshotsInChild,
+  runProjectSnapshotRecoveryHelper
+} from "./project-snapshot-recovery-runner.mjs";
 import {
   ProjectWorkspacePendingBranchPathChangeError,
   ProjectWorkspaceStateStore,
@@ -87,6 +91,21 @@ import {
   workspaceGitMountsForRemediation,
   WorkspaceRemediationAncestryError
 } from "./workspace-git-sync.mjs";
+const WORKER_LIFECYCLE_DETAIL_MAX_LENGTH = 500;
+const WORKER_SERVER_HEARTBEAT_INTERVAL_MS = 15e3;
+const WORKER_SERVER_HEARTBEAT_TIMEOUT_MS = 2 * 60 * 1e3;
+function boundedWorkerLifecycleProgress(progress) {
+  return progress.detail === void 0 ? progress : { ...progress, detail: progress.detail.slice(0, WORKER_LIFECYCLE_DETAIL_MAX_LENGTH) };
+}
+function workerServerHeartbeatAction(input) {
+  if (input.elapsedMs < input.timeoutMs) return "wait";
+  if (input.watchdogSweepDelayed || input.probeElapsedMs === void 0) return "start_probe";
+  return input.probeElapsedMs >= input.timeoutMs ? "terminate" : "wait";
+}
+const workerLifecycleTestHarness = {
+  boundedWorkerLifecycleProgress,
+  workerServerHeartbeatAction
+};
 class ProjectWorkspaceConfigurationDeferredError extends Error {
 }
 class WorkerServerUnavailableError extends Error {
@@ -2600,6 +2619,13 @@ async function executeCommand(input) {
   const startedAt = Date.now();
   let timeout;
   let spawnedProcess;
+  let preparationSettled = false;
+  const settlePreparation = () => {
+    if (preparationSettled) return;
+    preparationSettled = true;
+    input.setPreparationHeartbeatLease(false);
+  };
+  input.setPreparationHeartbeatLease(true);
   try {
     const targetProcessEnv = await prepareShellEnvForTarget({
       target: input.resolvedTarget.target,
@@ -2637,6 +2663,7 @@ async function executeCommand(input) {
       startedAt: (/* @__PURE__ */ new Date()).toISOString(),
       ...input.message.workspaceEffect === "none" ? { workspaceEffect: "none" } : {}
     });
+    settlePreparation();
     if (input.message.timeoutMs) {
       timeout = setTimeout(() => {
         void terminateProcessTree(subprocess).catch((error) => {
@@ -2672,6 +2699,7 @@ async function executeCommand(input) {
       error: error instanceof Error ? error.message : String(error)
     };
   } finally {
+    settlePreparation();
     if (timeout) {
       clearTimeout(timeout);
     }
@@ -2687,6 +2715,13 @@ async function executeStreamingCommand(input) {
   let timedOut = false;
   let timeout;
   let spawnedProcess;
+  let preparationSettled = false;
+  const settlePreparation = () => {
+    if (preparationSettled) return;
+    preparationSettled = true;
+    input.setPreparationHeartbeatLease(false);
+  };
+  input.setPreparationHeartbeatLease(true);
   try {
     if (cancelledProcessRuns.delete(input.message.runId)) {
       sendWorkerMessage(input.ws, {
@@ -2739,6 +2774,7 @@ async function executeStreamingCommand(input) {
       ...interactive ? { interactive: true, stdin: subprocess.stdin } : {},
       ...input.message.workspaceEffect === "none" ? { workspaceEffect: "none" } : {}
     });
+    settlePreparation();
     started = true;
     sendWorkerMessage(input.ws, {
       type: "exec_started",
@@ -2815,6 +2851,7 @@ async function executeStreamingCommand(input) {
       });
     }
   } finally {
+    settlePreparation();
     if (timeout) {
       clearTimeout(timeout);
     }
@@ -3156,7 +3193,8 @@ function createNodePtyBridge(options) {
   };
 }
 const workerPtyBridgeTestHarness = {
-  create: createNodePtyBridge
+  create: createNodePtyBridge,
+  closeAllBestEffort: closePtysBestEffort
 };
 function resolveHostShell(command, platform = process.platform) {
   if (platform === "win32") {
@@ -3305,11 +3343,23 @@ function closePty(message) {
   }
   ptyProcess.kill();
 }
-function closeAllPtys() {
-  for (const ptyProcess of activePtys.values()) {
-    ptyProcess.kill();
+function closePtysBestEffort(ptys, reportFailure) {
+  for (const [ptyId, ptyProcess] of ptys) {
+    try {
+      ptyProcess.kill();
+    } catch (error) {
+      reportFailure(ptyId, error);
+    }
   }
 }
+function closeAllPtys() {
+  closePtysBestEffort(activePtys, (ptyId, error) => {
+    process.stderr.write(
+      `[r5d-worker] failed to close PTY ${ptyId} after control disconnect: ${error instanceof Error ? error.message : String(error)}
+`
+    );
+  });
+}
 async function terminateCredentialBearingChildren(children) {
   const results = await Promise.allSettled(children.map(async ({ terminate }) => await terminate()));
   const failures = results.flatMap(
@@ -3369,16 +3419,6 @@ function websocketUrl(baseUrl, label) {
 }
 async function startWorker(options) {
   advanceWorkerAdmissionGeneration();
-  process.on("uncaughtException", (error) => {
-    process.stderr.write(`[r5d-worker] uncaught exception: ${error instanceof Error ? error.stack ?? error.message : String(error)}
-`);
-  });
-  process.on("unhandledRejection", (reason) => {
-    process.stderr.write(
-      `[r5d-worker] unhandled rejection: ${reason instanceof Error ? reason.stack ?? reason.message : String(reason)}
-`
-    );
-  });
   const config = readConfig(options.configPath);
   const { baseUrl, token } = resolveCredentials(options, config);
   initializeCredentialStoreDirectoryOnce();
@@ -3411,28 +3451,6 @@ async function startWorker(options) {
   runGit(["--version"]);
   await verifyR5dctlAuth(baseUrl, token, label);
   const initializedWorkspaceState = await workspaceSyncSingleFlight.runExclusive(() => {
-    if (!startupProjectSnapshotRecoveryCompleted) {
-      const snapshotRecovery = recoverStaleProjectWorktreeSnapshots({
-        projectsRoot,
-        temporaryRoot: projectWorktreeSnapshotsRoot
-      });
-      if (snapshotRecovery.restored.length > 0) {
-        process.stdout.write(`[r5d-worker] restored ${snapshotRecovery.restored.length} interrupted project snapshot(s)
-`);
-      }
-      if (snapshotRecovery.removed.length > 0) {
-        process.stdout.write(`[r5d-worker] removed ${snapshotRecovery.removed.length} completed/incomplete project snapshot(s)
-`);
-      }
-      for (const failure of snapshotRecovery.failed) {
-        process.stderr.write(`[r5d-worker] failed to recover stale project snapshot ${failure.path}: ${failure.error}
-`);
-      }
-      if (snapshotRecovery.failed.length > 0) {
-        throw new Error("Project snapshot recovery is incomplete; refusing to mutate project checkouts");
-      }
-      startupProjectSnapshotRecoveryCompleted = true;
-    }
     fs.mkdirSync(projectsRoot, { recursive: true });
     fs.mkdirSync(syncRoot, { recursive: true });
     fs.mkdirSync(artifactRoot, { recursive: true });
@@ -3471,6 +3489,8 @@ async function startWorker(options) {
   let cliUpdateInProgress = false;
   let reloadAfterClose = false;
   let shutdownAfterClose = false;
+  let startupRecoveryComplete = startupProjectSnapshotRecoveryCompleted;
+  let startupRecoveryPromise = Promise.resolve();
   const automaticWorkspaceSyncIsCurrentlyAdmitted = () => workspaceAutomaticSyncIsAdmitted({
     workspaceConfigured,
     activeIncidentId: activeWorkspaceIncidentId,
@@ -3887,8 +3907,14 @@ async function startWorker(options) {
       })
     )
   ].sort();
-  const ensureConfiguredProjects = async () => {
-    for (const project of projectConfigById.values()) {
+  const ensureConfiguredProjects = async (onProgress) => {
+    const projects = [...projectConfigById.values()];
+    for (const [projectIndex, project] of projects.entries()) {
+      onProgress?.({
+        detail: `Configuring ${project.projectPath}`,
+        completedItems: projectIndex,
+        totalItems: projects.length
+      });
       validateProjectId(project.projectId);
       assertRepositoryTransitionState(project);
       for (const branch of project.branches) validateBranchName(branch.branchName);
@@ -3936,7 +3962,14 @@ async function startWorker(options) {
           mirrorUrl: connection.mirrorUrl,
           mirrorCredentialUsername: connection.mirrorCredentialUsername,
           credentialHelper: connection.credentialHelper,
-          gitIdentity: workspaceGitIdentity
+          gitIdentity: workspaceGitIdentity,
+          onSnapshotProgress: (progress) => onProgress?.({
+            detail: progress.phase === "scanning" ? `Measuring ${project.projectPath} workspace snapshot` : progress.phase === "syncing" ? `Making ${project.projectPath} workspace snapshot durable` : `${progress.operation === "restore" ? "Restoring" : "Snapshotting"} ${project.projectPath}${progress.branchName ? `/${progress.branchName}` : ""}`,
+            completedBytes: progress.completedBytes,
+            completedItems: progress.completedBranches,
+            totalItems: progress.totalBranches,
+            ...progress.totalBytes === void 0 ? {} : { totalBytes: progress.totalBytes }
+          })
         });
         readyProjectIds.add(project.projectId);
         reconciledProjectConfigFingerprints.set(project.projectId, configFingerprint);
@@ -3971,6 +4004,11 @@ async function startWorker(options) {
 `
         );
       }
+      onProgress?.({
+        detail: `Configured ${project.projectPath}`,
+        completedItems: projectIndex + 1,
+        totalItems: projects.length
+      });
     }
   };
   const collectAheadOfOriginBranches = async () => {
@@ -4363,6 +4401,11 @@ async function startWorker(options) {
       if (input.sendResult !== false) sendWorkspaceSyncResult(input.requestId, result2);
       return result2;
     }
+    sendLifecycleProgress({
+      phase: "syncing",
+      operationId: attemptId,
+      detail: input.trigger.detail ?? `Synchronizing workspace (${input.trigger.type})`
+    });
     if (!input.requestId && input.sendResult !== false) {
       if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
         throw new Error("Cannot start autonomous workspace synchronization without an open worker control socket");
@@ -4465,6 +4508,9 @@ async function startWorker(options) {
       }
     };
     if (input.sendResult !== false) sendWorkspaceSyncResult(input.requestId, result);
+    sendLifecycleProgress(
+      result.outcome === "failed" ? { phase: "failed", operationId: attemptId, detail: result.error ?? "Workspace synchronization failed" } : { phase: "ready", operationId: attemptId, detail: "Workspace synchronized" }
+    );
     return result;
   };
   const scheduleAutomaticWorkspaceSync = (trigger, delayMs = WORKSPACE_GIT_QUIET_MS) => {
@@ -4539,6 +4585,14 @@ async function startWorker(options) {
     });
   };
   const configureWorkerWorkspace = async (message, receiptGeneration) => {
+    const configurationOperationId = crypto.randomUUID();
+    sendLifecycleProgress({
+      phase: "configuring",
+      operationId: configurationOperationId,
+      detail: `Preparing ${message.projects.length} project(s)`,
+      completedItems: 0,
+      totalItems: message.projects.length
+    });
     const incidentDeferral = workspaceConfigurationIncidentDeferral({
       requestedIncidentId: message.deferWorkspaceSyncForIncidentId,
       activeIncidentId: activeWorkspaceIncidentId
@@ -4903,7 +4957,13 @@ async function startWorker(options) {
           // checkout initialization from erasing synchronized dirt.
           { ignoreBusy: true }
         );
-        await ensureConfiguredProjects();
+        await ensureConfiguredProjects((progress) => {
+          sendLifecycleProgress({ phase: "configuring", operationId: configurationOperationId, ...progress });
+          process.stdout.write(
+            `[r5d-worker] ${progress.detail ?? "Configuring workspace"}${progress.completedBytes === void 0 ? "" : `: ${progress.completedBytes}/${progress.totalBytes ?? "?"} bytes`}${progress.completedItems === void 0 ? "" : `, ${progress.completedItems}/${progress.totalItems ?? "?"} item(s)`}
+`
+          );
+        });
         const configurationSyncAdmissionGeneration = workerAdmissionGeneration;
         const assertConfigurationSyncStillAdmitted = () => {
           if (receiptGeneration !== workspaceConfigurationReceiptGeneration || configurationSyncAdmissionGeneration !== workerAdmissionGeneration || !workspaceConfigurationIncidentSnapshotIsCurrent(incidentDeferral.incidentId, activeWorkspaceIncidentId) || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
@@ -5002,6 +5062,73 @@ async function startWorker(options) {
       Authorization: `Bearer ${token}`
     }
   });
+  let lastServerControlMessageAtMs = performance.now();
+  let serverHeartbeatProbeStartedAtMs;
+  let lastServerHeartbeatSweepAtMs = performance.now();
+  let serverHeartbeatWatchdog;
+  const noteServerControlActivity = () => {
+    lastServerControlMessageAtMs = performance.now();
+    serverHeartbeatProbeStartedAtMs = void 0;
+  };
+  const stopServerHeartbeatWatchdog = () => {
+    if (!serverHeartbeatWatchdog) return;
+    clearInterval(serverHeartbeatWatchdog);
+    serverHeartbeatWatchdog = void 0;
+  };
+  const sendLifecycleProgress = (progress) => {
+    if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) return;
+    sendWorkerMessage(ws, { type: "worker_progress", progress: boundedWorkerLifecycleProgress(progress) });
+  };
+  const describeSnapshotProgress = (progress) => {
+    const project = path.relative(projectsRoot, progress.projectRoot) || path.basename(progress.projectRoot);
+    const branch = progress.branchName ? `/${progress.branchName}` : "";
+    if (progress.phase === "scanning") return `Scanning ${project} recovery snapshot`;
+    if (progress.phase === "syncing") return `Making restored ${project} files durable`;
+    if (progress.phase === "cleaning") return `Cleaning recovered ${project} snapshot`;
+    return `Restoring ${project}${branch}`;
+  };
+  const performStartupRecovery = async (operationId) => {
+    if (startupProjectSnapshotRecoveryCompleted) return;
+    sendLifecycleProgress({ phase: "recovering", operationId, detail: "Checking interrupted workspace transactions" });
+    const snapshotRecovery = await workspaceSyncSingleFlight.runExclusive(
+      async () => await recoverProjectSnapshotsInChild({
+        projectsRoot,
+        snapshotRoot: projectWorktreeSnapshotsRoot,
+        onProgress: (progress) => {
+          const lifecycleProgress = {
+            phase: "recovering",
+            operationId,
+            detail: describeSnapshotProgress(progress),
+            completedBytes: progress.completedBytes,
+            completedItems: progress.completedBranches,
+            totalItems: progress.totalBranches,
+            ...progress.totalBytes === void 0 ? {} : { totalBytes: progress.totalBytes }
+          };
+          sendLifecycleProgress(lifecycleProgress);
+          process.stdout.write(
+            `[r5d-worker] ${lifecycleProgress.detail}: ${progress.completedBranches}/${progress.totalBranches} branch(es), ${progress.completedBytes}/${progress.totalBytes ?? "?"} bytes
+`
+          );
+        }
+      })
+    );
+    if (snapshotRecovery.restored.length > 0) {
+      process.stdout.write(`[r5d-worker] restored ${snapshotRecovery.restored.length} interrupted project snapshot(s)
+`);
+    }
+    if (snapshotRecovery.removed.length > 0) {
+      process.stdout.write(`[r5d-worker] removed ${snapshotRecovery.removed.length} completed/incomplete project snapshot(s)
+`);
+    }
+    for (const failure of snapshotRecovery.failed) {
+      process.stderr.write(`[r5d-worker] failed to recover stale project snapshot ${failure.path}: ${failure.error}
+`);
+    }
+    if (snapshotRecovery.failed.length > 0) {
+      throw new Error("Project snapshot recovery is incomplete; refusing to mutate project checkouts");
+    }
+    startupProjectSnapshotRecoveryCompleted = true;
+  };
   const handleGracefulShutdown = () => {
     if (shutdownAfterClose) return;
     shutdownAfterClose = true;
@@ -5031,8 +5158,50 @@ async function startWorker(options) {
   process.on("SIGINT", handleGracefulShutdown);
   ws.addEventListener("open", () => {
     currentWorkerSocket = ws;
+    noteServerControlActivity();
+    lastServerHeartbeatSweepAtMs = performance.now();
+    serverHeartbeatWatchdog = setInterval(() => {
+      if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) return;
+      const nowMs = performance.now();
+      const heartbeatAction = workerServerHeartbeatAction({
+        elapsedMs: Math.max(0, nowMs - lastServerControlMessageAtMs),
+        ...serverHeartbeatProbeStartedAtMs === void 0 ? {} : { probeElapsedMs: Math.max(0, nowMs - serverHeartbeatProbeStartedAtMs) },
+        timeoutMs: WORKER_SERVER_HEARTBEAT_TIMEOUT_MS,
+        watchdogSweepDelayed: nowMs - lastServerHeartbeatSweepAtMs >= WORKER_SERVER_HEARTBEAT_INTERVAL_MS * 2
+      });
+      lastServerHeartbeatSweepAtMs = nowMs;
+      if (heartbeatAction === "start_probe") {
+        serverHeartbeatProbeStartedAtMs = nowMs;
+        return;
+      }
+      if (heartbeatAction === "terminate") {
+        stopServerHeartbeatWatchdog();
+        process.stderr.write("[r5d-worker] server control heartbeat timed out; reconnecting\n");
+        ws.terminate();
+      }
+    }, WORKER_SERVER_HEARTBEAT_INTERVAL_MS);
+    serverHeartbeatWatchdog.unref();
+    const startupOperationId = crypto.randomUUID();
+    startupRecoveryPromise = Promise.resolve().then(async () => await performStartupRecovery(startupOperationId)).then(() => {
+      startupRecoveryComplete = true;
+      if (currentWorkerSocket === ws && ws.readyState === WebSocket.OPEN) {
+        sendWorkerMessage(ws, { type: "startup_ready" });
+        sendLifecycleProgress({
+          phase: "configuring",
+          operationId: crypto.randomUUID(),
+          detail: "Waiting for workspace configuration"
+        });
+      }
+    }).catch((error) => {
+      const message = error instanceof Error ? error.message : String(error);
+      sendLifecycleProgress({ phase: "failed", operationId: startupOperationId, detail: message });
+      process.stderr.write(`[r5d-worker] startup recovery failed: ${message}
+`);
+      ws.close(1011, "Project snapshot recovery failed");
+    });
     const hello = {
       type: "hello",
+      startupReady: startupRecoveryComplete,
       hostInfo: {
         hostname: hostname(),
         platform: process.platform,
@@ -5068,8 +5237,11 @@ async function startWorker(options) {
     process.stdout.write("[r5d-worker] connected\n");
   });
   ws.addEventListener("message", (event) => {
+    noteServerControlActivity();
     void (async () => {
       const message = JSON.parse(String(event.data));
+      if (!startupRecoveryComplete && message.type !== "ping") await startupRecoveryPromise;
+      if (!startupRecoveryComplete && message.type !== "ping") return;
       const messageAdmissionGeneration = workerAdmissionGeneration;
       const assertMessageAdmission = () => assertWorkerChildAdmission({
         capturedGeneration: messageAdmissionGeneration,
@@ -5097,6 +5269,19 @@ async function startWorker(options) {
           aheadOfOriginBranches: configured.aheadOfOriginBranches,
           ...configured.deferredWorkspaceSyncForIncidentId ? { deferredWorkspaceSyncForIncidentId: configured.deferredWorkspaceSyncForIncidentId } : {}
         });
+        sendLifecycleProgress(
+          configured.result.outcome === "failed" ? {
+            phase: "failed",
+            operationId: configured.result.attemptId,
+            detail: configured.result.error ?? "Workspace configuration failed"
+          } : {
+            phase: "ready",
+            operationId: configured.result.attemptId,
+            detail: `${message.projects.length} project(s) configured`,
+            completedItems: message.projects.length,
+            totalItems: message.projects.length
+          }
+        );
         process.stdout.write(
           `[r5d-worker] workspace configured: ${message.projects.length} project(s), ${configured.pending.length} pending checkout(s)
 `
@@ -5660,6 +5845,13 @@ async function startWorker(options) {
               token,
               artifactRoot,
               planRoot,
+              setPreparationHeartbeatLease: (active) => {
+                sendWorkerMessageFromCurrentSource(ws, {
+                  type: "heartbeat_lease",
+                  operationId: `exec:${message.runId}`,
+                  active
+                });
+              },
               assertAdmission: () => {
                 if (!workspaceCommandTransportIsAllowed(deferredWorkspaceConfiguration, activeWorkspaceIncidentId, "exec")) {
                   throw new Error(
@@ -5723,6 +5915,13 @@ async function startWorker(options) {
               token,
               artifactRoot,
               planRoot,
+              setPreparationHeartbeatLease: (active) => {
+                sendWorkerMessageFromCurrentSource(ws, {
+                  type: "heartbeat_lease",
+                  operationId: `exec:${message.runId}`,
+                  active
+                });
+              },
               assertAdmission: assertMessageAdmission
             });
           } finally {
@@ -5855,6 +6054,7 @@ async function startWorker(options) {
     const processEvents = process;
     processEvents.off("SIGTERM", handleGracefulShutdown);
     processEvents.off("SIGINT", handleGracefulShutdown);
+    stopServerHeartbeatWatchdog();
     if (workspaceAutomaticTimer) {
       clearTimeout(workspaceAutomaticTimer);
       workspaceAutomaticTimer = void 0;
@@ -5885,23 +6085,47 @@ async function startWorker(options) {
     if (shutdownAfterClose) {
       process.exit(0);
     }
+    if (!startupRecoveryComplete) {
+      process.stderr.write("[r5d-worker] control connection closed during workspace recovery; finishing recovery before reconnecting\n");
+      void startupRecoveryPromise.then(() => {
+        if (!startupRecoveryComplete) {
+          process.exit(1);
+          return;
+        }
+        const reconnectDelayMs = workerReconnectDelayMs();
+        process.stderr.write(`[r5d-worker] workspace recovery complete; reconnecting in ${reconnectDelayMs}ms
+`);
+        setTimeout(() => {
+          void startWorker(options).catch((error) => {
+            process.stderr.write(`[r5d-worker] reconnect failed: ${error instanceof Error ? error.message : String(error)}
+`);
+            process.exit(error instanceof WorkerServerUnavailableError ? WORKER_RECONNECT_EXIT_CODE : 1);
+          });
+        }, reconnectDelayMs);
+      });
+      return;
+    }
     if (activeProcesses.size > 0 || credentialBearingProcessGroups.size > 0 || activePtys.size > 0 || pendingProcessTerminals.size > 0) {
-      process.stderr.write(
-        `[r5d-worker] ${activeProcesses.size} process(es), ${credentialBearingProcessGroups.size} owned process group(s), and ${activePtys.size} PTY(s) active with ${pendingProcessTerminals.size} terminal report(s) pending; reconnecting in ${WORKER_RECONNECT_DELAY_MS}ms
+      const scheduleReconnect = (reconnect2) => {
+        const reconnectDelayMs = workerReconnectDelayMs();
+        process.stderr.write(
+          `[r5d-worker] ${activeProcesses.size} process(es), ${credentialBearingProcessGroups.size} owned process group(s), and ${activePtys.size} PTY(s) active with ${pendingProcessTerminals.size} terminal report(s) pending; reconnecting in ${reconnectDelayMs}ms
 `
-      );
+        );
+        setTimeout(reconnect2, reconnectDelayMs);
+      };
       const reconnect = () => {
         void startWorker(options).catch((error) => {
           process.stderr.write(`[r5d-worker] reconnect failed: ${error instanceof Error ? error.message : String(error)}
 `);
           if (activeProcesses.size > 0 || credentialBearingProcessGroups.size > 0 || activePtys.size > 0 || pendingProcessTerminals.size > 0) {
-            setTimeout(reconnect, WORKER_RECONNECT_DELAY_MS);
+            scheduleReconnect(reconnect);
             return;
           }
           process.exit(error instanceof WorkerServerUnavailableError ? WORKER_RECONNECT_EXIT_CODE : 1);
         });
       };
-      setTimeout(reconnect, WORKER_RECONNECT_DELAY_MS);
+      scheduleReconnect(reconnect);
       return;
     }
     process.exit(WORKER_RECONNECT_EXIT_CODE);
@@ -5911,10 +6135,28 @@ async function startWorker(options) {
     process.stderr.write(`[r5d-worker] websocket error${message}
 `);
   });
-  await new Promise(() => {
+  await new Promise((resolve) => {
+    ws.addEventListener("close", () => resolve(), { once: true });
+  });
+}
+function installWorkerRuntimeFailureLogging() {
+  process.on("uncaughtException", (error) => {
+    process.stderr.write(`[r5d-worker] uncaught exception: ${error instanceof Error ? error.stack ?? error.message : String(error)}
+`);
+  });
+  process.on("unhandledRejection", (reason) => {
+    process.stderr.write(
+      `[r5d-worker] unhandled rejection: ${reason instanceof Error ? reason.stack ?? reason.message : String(reason)}
+`
+    );
   });
 }
 async function main() {
+  if (process.argv[2] === PROJECT_SNAPSHOT_RECOVERY_HELPER_COMMAND) {
+    const exitCode2 = runProjectSnapshotRecoveryHelper(process.argv.slice(3));
+    if (exitCode2 !== 0) process.exit(exitCode2);
+    return;
+  }
   const parsed = parseArgs(process.argv.slice(2));
   if (parsed.command === "help") {
     printHelp();
@@ -5926,6 +6168,7 @@ async function main() {
   }
   if (process.env[WORKER_RUNTIME_ENV] === "1") {
     delete process.env[WORKER_RUNTIME_ENV];
+    installWorkerRuntimeFailureLogging();
     try {
       await startWorker(parsed.options);
     } catch (error) {
@@ -5978,6 +6221,7 @@ export {
   syncSessionArtifacts,
   workerChildProcessEnvironment,
   workerGitSecurityTestHarness,
+  workerLifecycleTestHarness,
   workerPtyBridgeTestHarness,
   workerPtyTestHarness,
   writeWorkerTextFile

package/dist/mjs/package.json

@@ -1,5 +1,5 @@
 {
   "name": "@ricsam/r5d-worker",
-  "version": "0.0.90",
+  "version": "0.0.92",
   "type": "module"
 }