@ricsam/r5d-worker 0.0.108 → 0.0.110

package/dist/mjs/main.mjs

@@ -3,6 +3,12 @@ import { getWorkerPlanPath, getWorkerPlansPath, validatePlanId } from "./plan-pa
 import { createPlanMarkdown, updateTaskCheckbox } from "./plan-parser.mjs";
 import { lintPlans, formatPlanLintErrors } from "./plan-lint.mjs";
 import fs from "node:fs";
+import { WorkerRecoveryStore, workerOperationDigest } from "./recovery-store.mjs";
+import { WorkerOutputWindow } from "./recovery-output-window.mjs";
+import {
+  WORKER_RESUMABLE_PROTOCOL,
+  WORKER_RECOVERY_TIMEOUT_MS
+} from "./recovery-protocol.mjs";
 import path from "node:path";
 import { createHash } from "node:crypto";
 import os, { hostname } from "node:os";
@@ -172,6 +178,63 @@ const cancelledProcessRuns = /* @__PURE__ */ new Set();
 const activePtys = /* @__PURE__ */ new Map();
 const workspaceSyncPriorityPtyTargets = /* @__PURE__ */ new Map();
 let currentWorkerSocket = null;
+const workerRuntimeId = crypto.randomUUID();
+const workerOutputWindow = new WorkerOutputWindow();
+let recoveryStore;
+let workerReconnectAttempt = 0;
+let workerRecoveryId;
+let workerExecutionLeaseDeadline = 0;
+let workerExecutionLeaseTimer;
+let workerLeaseExpired = false;
+const infrastructureStoppedRunIds = /* @__PURE__ */ new Set();
+function isWorkerCommunicationFailure(message) {
+  return /Could not resolve host|Couldn't connect|Failed to connect|Connection reset|Connection timed out|remote end hung up|Empty reply|requested URL returned error: (?:408|429|5\d\d)|fetch failed|socket connection|ECONNRESET|ECONNREFUSED|ETIMEDOUT|Unable to connect|network connection was lost|Worker communication recovery expired|operation was aborted due to timeout|operation timed out|Failed to (?:fetch|download).*\((?:408|429|5\d\d)\)/i.test(
+    message
+  );
+}
+async function retryWorkerServerRead(read) {
+  let deadline;
+  let retry = 0;
+  while (true) {
+    try {
+      return await read();
+    } catch (error) {
+      if (!recoveryStore || !isWorkerCommunicationFailure(error instanceof Error ? error.message : String(error))) throw error;
+      deadline ??= Date.now() + WORKER_RECOVERY_TIMEOUT_MS;
+      if (workerLeaseExpired || Date.now() >= deadline) throw new WorkerServerUnavailableError("Worker communication recovery expired");
+      currentWorkerSocket?.close(1012, "Communication recovery");
+      await Bun.sleep(workerReconnectDelayMs(Math.random, retry++));
+    }
+  }
+}
+function renewWorkerExecutionLease(leaseMs) {
+  if (!Number.isFinite(leaseMs) || leaseMs <= 0) return;
+  workerLeaseExpired = false;
+  workerExecutionLeaseDeadline = Date.now() + Math.min(leaseMs, WORKER_RECOVERY_TIMEOUT_MS);
+  if (workerExecutionLeaseTimer) clearTimeout(workerExecutionLeaseTimer);
+  workerExecutionLeaseTimer = setTimeout(
+    () => {
+      if (Date.now() < workerExecutionLeaseDeadline) return;
+      workerLeaseExpired = true;
+      for (const operation of recoveryStore?.operations() ?? []) {
+        if (operation.sessionId && operation.state !== "completed") recoveryStore?.cancelSession(operation.sessionId);
+      }
+      for (const [runId, active] of activeProcesses) {
+        if (!active.sessionId) continue;
+        infrastructureStoppedRunIds.add(runId);
+        recoveryStore?.unknownRun(runId);
+        cancelledProcessRuns.add(runId);
+        closeProcessStdin(active);
+        void terminateProcessTree(active.process).catch(
+          (error) => process.stderr.write(`[r5d-worker] execution lease termination failed: ${error}
+`)
+        );
+      }
+    },
+    Math.min(leaseMs, WORKER_RECOVERY_TIMEOUT_MS)
+  );
+  workerExecutionLeaseTimer.unref();
+}
 let workerAdmissionGeneration = 0;
 const workspaceMutationGate = new WorkspaceMutationGate();
 let workspaceSyncQueue = Promise.resolve();
@@ -630,40 +693,46 @@ function parseRemoteFileManifest(value, collectionKey, label) {
   });
 }
 async function fetchRemoteFileManifest(input) {
-  const response = await fetch(input.url, {
-    headers: {
-      Authorization: `Bearer ${input.token}`
+  return retryWorkerServerRead(async () => {
+    const response = await fetch(input.url, {
+      signal: AbortSignal.timeout(3e4),
+      headers: {
+        Authorization: `Bearer ${input.token}`
+      }
+    });
+    if (!response.ok) {
+      const detail = (await readResponseText(response)).trim();
+      throw new Error(`Failed to fetch ${input.label} manifest (${response.status})${detail ? `: ${detail}` : ""}`);
     }
+    return parseRemoteFileManifest(await response.json(), input.collectionKey, input.label);
   });
-  if (!response.ok) {
-    const detail = (await readResponseText(response)).trim();
-    throw new Error(`Failed to fetch ${input.label} manifest (${response.status})${detail ? `: ${detail}` : ""}`);
-  }
-  return parseRemoteFileManifest(await response.json(), input.collectionKey, input.label);
 }
 async function downloadRemoteSyncedFile(input) {
-  const targetPath = path.join(input.targetDir, input.file.filename);
-  assertInsideRoot(input.targetDir, targetPath, input.label);
-  const response = await fetch(input.url, {
-    headers: {
-      Authorization: `Bearer ${input.token}`
+  return retryWorkerServerRead(async () => {
+    const targetPath = path.join(input.targetDir, input.file.filename);
+    assertInsideRoot(input.targetDir, targetPath, input.label);
+    const response = await fetch(input.url, {
+      signal: AbortSignal.timeout(3e4),
+      headers: {
+        Authorization: `Bearer ${input.token}`
+      }
+    });
+    if (!response.ok) {
+      const detail = (await readResponseText(response)).trim();
+      throw new Error(`Failed to download ${input.label} ${input.file.filename} (${response.status})${detail ? `: ${detail}` : ""}`);
     }
+    const buffer = Buffer.from(await response.arrayBuffer());
+    const receivedHash = createHash("sha256").update(buffer).digest("hex");
+    if (receivedHash !== input.file.sha256) {
+      throw new Error(`Downloaded ${input.label} ${input.file.filename} failed sha256 verification`);
+    }
+    if (buffer.length !== input.file.size) {
+      throw new Error(`Downloaded ${input.label} ${input.file.filename} has unexpected size`);
+    }
+    const tempPath = `${targetPath}.${process.pid}.${Date.now()}.tmp`;
+    fs.writeFileSync(tempPath, buffer);
+    fs.renameSync(tempPath, targetPath);
   });
-  if (!response.ok) {
-    const detail = (await readResponseText(response)).trim();
-    throw new Error(`Failed to download ${input.label} ${input.file.filename} (${response.status})${detail ? `: ${detail}` : ""}`);
-  }
-  const buffer = Buffer.from(await response.arrayBuffer());
-  const receivedHash = createHash("sha256").update(buffer).digest("hex");
-  if (receivedHash !== input.file.sha256) {
-    throw new Error(`Downloaded ${input.label} ${input.file.filename} failed sha256 verification`);
-  }
-  if (buffer.length !== input.file.size) {
-    throw new Error(`Downloaded ${input.label} ${input.file.filename} has unexpected size`);
-  }
-  const tempPath = `${targetPath}.${process.pid}.${Date.now()}.tmp`;
-  fs.writeFileSync(tempPath, buffer);
-  fs.renameSync(tempPath, targetPath);
 }
 async function syncRemoteFileSet(input) {
   fs.mkdirSync(input.targetDir, { recursive: true });
@@ -790,6 +859,7 @@ async function verifyR5dctlAuth(baseUrl, token, workerLabel) {
   let response;
   try {
     response = await fetch(statusUrl, {
+      signal: AbortSignal.timeout(3e4),
       headers: {
         Authorization: `Bearer ${token}`
       }
@@ -2953,23 +3023,46 @@ function closeProcessStdin(active) {
   }
   active.stdin = void 0;
 }
-function sendWorkerMessage(ws, message) {
+function sendReplayWorkerMessage(ws, message) {
   const target = currentWorkerSocket ?? ws;
-  try {
-    target.send(JSON.stringify(message));
-  } catch (error) {
-    process.stderr.write(`[r5d-worker] failed to send ${message.type}: ${error instanceof Error ? error.message : String(error)}
-`);
+  if (target.readyState !== WebSocket.OPEN) return false;
+  const send = (payload) => {
+    try {
+      target.send(JSON.stringify(payload));
+      return true;
+    } catch {
+      return false;
+    }
+  };
+  return message.type === "exec_output" ? workerOutputWindow.send(message, send, target.bufferedAmount) : send(message);
+}
+function sendWorkerMessage(ws, message) {
+  if (recoveryStore && "requestId" in message && typeof message.requestId === "string" && recoveryStore.isUnknown(message.requestId))
+    return;
+  const transportError = "error" in message && typeof message.error === "string" ? message.error : message.type === "workspace_sync_result" && message.result.outcome === "failed" ? message.result.error : void 0;
+  if (recoveryStore && transportError && isWorkerCommunicationFailure(transportError) && "requestId" in message && typeof message.requestId === "string") {
+    recoveryStore.unknown(message.requestId);
+    currentWorkerSocket?.close(1012, "Communication recovery");
+    return;
+  }
+  const recorded = recoveryStore?.record(message) ?? message;
+  if ((message.type === "exec_exit" || message.type === "exec_error") && infrastructureStoppedRunIds.has(message.runId)) {
+    recoveryStore?.unknownRun(message.runId);
+    return;
   }
+  if (message.type === "exec_output" && workerRecoveryId !== void 0) return;
+  sendReplayWorkerMessage(ws, recorded);
+}
+function sendSerializedWorkerMessage(ws, serialized) {
+  sendWorkerMessage(ws, JSON.parse(serialized));
 }
 function sendWorkerMessageFromCurrentSource(ws, message) {
-  if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) return;
-  try {
-    ws.send(JSON.stringify(message));
-  } catch (error) {
-    process.stderr.write(`[r5d-worker] failed to send ${message.type}: ${error instanceof Error ? error.message : String(error)}
-`);
+  if ("requestId" in message && typeof message.requestId === "string" && message.type !== "workspace_configured") {
+    sendWorkerMessage(ws, message);
+    return;
   }
+  if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) return;
+  sendWorkerMessage(ws, message);
 }
 function buildActiveProcessReports() {
   return Array.from(activeProcesses.entries()).filter(([, active]) => active.sessionId.length > 0).map(([runId, active]) => ({
@@ -3489,13 +3582,15 @@ async function startWorker(options, projectRuntime = {
   reconciledProjectConfigFingerprints: /* @__PURE__ */ new Map(),
   gitIdentity: null
 }) {
-  advanceWorkerAdmissionGeneration();
   const config = readConfig(options.configPath);
   const { baseUrl, token } = resolveCredentials(options, config);
   initializeCredentialStoreDirectoryOnce();
   const label = requireValue(options.label, "Missing required --label <label>");
   validateLabel(label);
   const rootDir = path.resolve(options.rootDir ?? process.env.R5D_ROOT ?? defaultRootDir());
+  recoveryStore ??= new WorkerRecoveryStore(
+    path.join(rootDir, ".worker-recovery", createHash("sha256").update(`${baseUrl}\0${label}`).digest("hex"), "protocol.sqlite")
+  );
   const projectsRoot = path.join(rootDir, "projects");
   const syncRoot = path.join(rootDir, "sync");
   const workspaceShadowRoot = path.join(syncRoot, "workspace");
@@ -3516,7 +3611,7 @@ async function startWorker(options, projectRuntime = {
 `);
   runGit(["--version"]);
   await verifyR5dctlAuth(baseUrl, token, label);
-  const initializedWorkspaceState = await workspaceSyncSingleFlight.runExclusive(() => {
+  const initializedWorkspaceState = projectRuntime.initializedWorkspaceState ?? await workspaceSyncSingleFlight.runExclusive(() => {
     fs.mkdirSync(projectsRoot, { recursive: true });
     fs.mkdirSync(syncRoot, { recursive: true });
     fs.mkdirSync(artifactRoot, { recursive: true });
@@ -3527,8 +3622,9 @@ async function startWorker(options, projectRuntime = {
     });
     return { store, state: store.read() };
   });
+  projectRuntime.initializedWorkspaceState = initializedWorkspaceState;
   const projectWorkspaceStateStore = initializedWorkspaceState.store;
-  let projectWorkspaceState = initializedWorkspaceState.state;
+  let projectWorkspaceState = initializedWorkspaceState.store.read();
   const { projectConfigById, readyProjectIds, reconciledProjectConfigFingerprints } = projectRuntime;
   const pendingCheckouts = /* @__PURE__ */ new Map();
   const lastObservedProjectHeads = /* @__PURE__ */ new Map();
@@ -3568,11 +3664,12 @@ async function startWorker(options, projectRuntime = {
   let workspaceConfigurationReceiptGeneration = 0;
   let workspaceAutomaticTimer;
   let workspacePeriodicTimer;
-  let pendingAutomaticTrigger;
+  let pendingAutomaticTrigger = projectRuntime.pendingAutomaticTrigger;
   const pendingCreatedBranchPublicationNotBefore = /* @__PURE__ */ new Map();
   const workspaceProjectionLedger = new WorkspaceProjectionLedger();
   let automaticSyncInFlight = false;
   let terminalReplayTimer;
+  let outputReplayTimer;
   let workspaceSyncRequestsInFlight = 0;
   let lastWorkspaceSyncMirrorObservationMs = 0;
   let lastInventoriedMissingCheckouts;
@@ -4664,7 +4761,7 @@ async function startWorker(options, projectRuntime = {
   });
   const workspaceSyncAdmissionSnapshotsMatch = (left, right) => left.generation === right.generation && left.activeIncidentId === right.activeIncidentId && left.deferredIncidentId === right.deferredIncidentId;
   const assertWorkspaceSyncStillAdmitted = (admitted, trigger) => {
-    if (!workspaceSyncAdmissionSnapshotsMatch(admitted, captureWorkspaceSyncAdmission()) || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN || !deferredWorkspaceSyncTriggerIsAllowed(deferredWorkspaceConfiguration, activeWorkspaceIncidentId, trigger)) {
+    if (!workspaceSyncAdmissionSnapshotsMatch(admitted, captureWorkspaceSyncAdmission()) || !deferredWorkspaceSyncTriggerIsAllowed(deferredWorkspaceConfiguration, activeWorkspaceIncidentId, trigger)) {
       throw new WorkspaceSyncAdmissionChangedError("Workspace synchronization admission changed while asynchronous work was in flight");
     }
   };
@@ -4718,6 +4815,7 @@ async function startWorker(options, projectRuntime = {
         bouncePending: unsafeWorkspaceSyncBouncePending,
         invalidateExecution: () => {
           workspaceConfigured = false;
+          projectRuntime.configuration = void 0;
           advanceWorkerAdmissionGeneration();
         },
         scheduleRecoveryBounce: () => {
@@ -4828,7 +4926,7 @@ async function startWorker(options, projectRuntime = {
       if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
         throw new Error("Cannot start autonomous workspace synchronization without an open worker control socket");
       }
-      ws.send(JSON.stringify({ type: "workspace_sync_started", attemptId }));
+      sendSerializedWorkerMessage(ws, JSON.stringify({ type: "workspace_sync_started", attemptId }));
     }
     let queueMs = 0;
     let prepareMs = 0;
@@ -4880,6 +4978,7 @@ async function startWorker(options, projectRuntime = {
   };
   const scheduleAutomaticWorkspaceSync = (trigger, delayMs = WORKSPACE_GIT_QUIET_MS) => {
     pendingAutomaticTrigger = trigger;
+    projectRuntime.pendingAutomaticTrigger = trigger;
     const pendingBranchDeferral = () => pendingCreatedBranchAutomaticSyncDeferral(
       projectWorkspaceState.locallyPendingCreatedBranches,
       activeWorkspaceMutationTargets(),
@@ -4902,6 +5001,7 @@ async function startWorker(options, projectRuntime = {
           return;
         }
         pendingAutomaticTrigger = void 0;
+        projectRuntime.pendingAutomaticTrigger = void 0;
         automaticSyncInFlight = true;
         void runWorkspaceSync({ trigger: scheduledTrigger }).then((result) => {
           if (result.outcome === "failed") {
@@ -4925,9 +5025,14 @@ async function startWorker(options, projectRuntime = {
     workspaceAutomaticTimer.unref();
   };
   const markWorkspaceDirty = (trigger, force = false, target) => {
+    if (projectRuntime.markWorkspaceDirty && projectRuntime.markWorkspaceDirty !== markWorkspaceDirty) {
+      projectRuntime.markWorkspaceDirty(trigger, force, target);
+      return;
+    }
     if (target) recordVisibleWorkspaceMutation(target);
     scheduleAutomaticWorkspaceSync(trigger, force ? 0 : WORKSPACE_GIT_QUIET_MS);
   };
+  projectRuntime.markWorkspaceDirty = markWorkspaceDirty;
   const targetMayMutateVisibleWorkspace = (target) => target.type === "project" || target.rootProfile === "visible_projects";
   const resolveMessageTarget = (target) => {
     if (!workspaceConfigured) throw new Error("Worker workspace configuration has not completed successfully");
@@ -5445,6 +5550,10 @@ async function startWorker(options, projectRuntime = {
       Authorization: `Bearer ${token}`
     }
   });
+  const connectionAttemptTimer = setTimeout(() => {
+    if (ws.readyState === WebSocket.CONNECTING) ws.terminate();
+  }, 3e4);
+  connectionAttemptTimer.unref();
   let lastServerControlMessageAtMs = performance.now();
   let serverHeartbeatProbeStartedAtMs;
   let lastServerHeartbeatSweepAtMs = performance.now();
@@ -5540,7 +5649,9 @@ async function startWorker(options, projectRuntime = {
   process.on("SIGTERM", handleGracefulShutdown);
   process.on("SIGINT", handleGracefulShutdown);
   ws.addEventListener("open", () => {
+    clearTimeout(connectionAttemptTimer);
     currentWorkerSocket = ws;
+    workerRecoveryId = "connecting";
     noteServerControlActivity();
     lastServerHeartbeatSweepAtMs = performance.now();
     serverHeartbeatWatchdog = setInterval(() => {
@@ -5584,6 +5695,9 @@ async function startWorker(options, projectRuntime = {
     });
     const hello = {
       type: "hello",
+      resumableProtocol: WORKER_RESUMABLE_PROTOCOL,
+      runtimeId: workerRuntimeId,
+      ledgerId: recoveryStore.ledgerId,
       startupReady: startupRecoveryComplete,
       hostInfo: {
         hostname: hostname(),
@@ -5608,15 +5722,17 @@ async function startWorker(options, projectRuntime = {
         root: rootDir
       }
     };
-    ws.send(JSON.stringify(hello));
-    sendActiveProcessReport(ws);
-    for (const terminal of pendingProcessTerminals.values()) {
-      sendWorkerMessage(ws, terminal);
-    }
-    terminalReplayTimer = setInterval(() => {
-      for (const terminal of pendingProcessTerminals.values()) {
-        sendWorkerMessage(ws, terminal);
+    sendSerializedWorkerMessage(ws, JSON.stringify(hello));
+    outputReplayTimer = setInterval(() => {
+      if (workerRecoveryId !== void 0 || currentWorkerSocket !== ws) return;
+      for (const output of recoveryStore.output()) {
+        if (!sendReplayWorkerMessage(ws, output)) break;
       }
+    }, 250);
+    outputReplayTimer.unref();
+    terminalReplayTimer = setInterval(() => {
+      if (workerRecoveryId !== void 0) return;
+      for (const terminal of recoveryStore?.terminals() ?? []) sendReplayWorkerMessage(ws, terminal);
     }, 5e3);
     terminalReplayTimer.unref();
     process.stdout.write("[r5d-worker] connected\n");
@@ -5624,18 +5740,103 @@ async function startWorker(options, projectRuntime = {
   ws.addEventListener("message", (event) => {
     noteServerControlActivity();
     void (async () => {
+      if (currentWorkerSocket !== ws) return;
       const message = JSON.parse(String(event.data));
+      if (message.type === "recovery_request") {
+        workerRecoveryId = message.recoveryId;
+        sendActiveProcessReport(ws);
+        sendWorkerMessage(ws, {
+          type: "recovery_state",
+          recoveryId: message.recoveryId,
+          runtimeId: workerRuntimeId,
+          ledgerId: recoveryStore.ledgerId,
+          operations: recoveryStore.operations()
+        });
+        const cursors = new Map(recoveryStore.outputCursors().map((cursor) => [cursor.runId, cursor]));
+        for (const cursor of message.outputOffsets) cursors.set(cursor.runId, cursor);
+        workerOutputWindow.reset([...cursors.values()]);
+        for (const response of recoveryStore.responses(message.requests)) sendReplayWorkerMessage(ws, response);
+        for (const output of recoveryStore.output(message.outputOffsets)) {
+          while (true) {
+            if (currentWorkerSocket !== ws || workerRecoveryId !== message.recoveryId || ws.readyState !== WebSocket.OPEN || workerLeaseExpired)
+              return;
+            if (sendReplayWorkerMessage(ws, output)) break;
+            await Bun.sleep(10);
+          }
+        }
+        if (currentWorkerSocket !== ws || workerRecoveryId !== message.recoveryId) return;
+        for (const terminal of recoveryStore.terminals()) sendReplayWorkerMessage(ws, terminal);
+        sendWorkerMessage(ws, { type: "recovery_complete", recoveryId: message.recoveryId });
+        return;
+      }
+      if (message.type === "recovery_ready") {
+        if (workerRecoveryId !== message.recoveryId) return;
+        workerRecoveryId = void 0;
+        workerReconnectAttempt = 0;
+        renewWorkerExecutionLease(message.leaseMs);
+        return;
+      }
+      if (message.type === "worker_execution_lease") {
+        renewWorkerExecutionLease(message.leaseMs);
+        return;
+      }
+      if (message.type === "operation_result_ack") {
+        recoveryStore.acknowledgeResult(message.requestId);
+        return;
+      }
+      if (message.type === "exec_output_ack") {
+        recoveryStore.acknowledgeOutput(message.runId, message.stream, message.nextOffset);
+        workerOutputWindow.acknowledge(message.runId, message.stream, message.nextOffset);
+        return;
+      }
+      if (message.type === "cancel_session") {
+        recoveryStore.cancelSession(message.sessionId);
+        await Promise.all(
+          [...activeProcesses].filter(([, active]) => active.sessionId === message.sessionId).map(async ([runId, active]) => {
+            cancelledProcessRuns.add(runId);
+            closeProcessStdin(active);
+            await terminateProcessTree(active.process);
+          })
+        );
+        sendWorkerMessage(ws, { type: "cancel_session_ack", cancellationId: message.cancellationId });
+        return;
+      }
       if (!startupRecoveryComplete && message.type !== "ping") await startupRecoveryPromise;
       if (!startupRecoveryComplete && message.type !== "ping") return;
+      if (currentWorkerSocket !== ws) return;
+      const operationRequestId = "requestId" in message && typeof message.requestId === "string" && message.type !== "workspace_config" ? message.requestId : void 0;
+      if (operationRequestId) {
+        const admission = recoveryStore.admit({ ...message, requestId: operationRequestId });
+        if (admission === "conflict") {
+          ws.close(1008, "Operation identity conflict");
+          return;
+        }
+        sendWorkerMessage(ws, { type: "operation_received", requestId: operationRequestId });
+        if (admission !== "new") {
+          const response = recoveryStore.response(operationRequestId);
+          if (response && admission !== "unknown") sendReplayWorkerMessage(ws, response);
+          return;
+        }
+      }
       const messageAdmissionGeneration = workerAdmissionGeneration;
-      const assertMessageAdmission = () => assertWorkerChildAdmission({
-        capturedGeneration: messageAdmissionGeneration,
-        currentGeneration: workerAdmissionGeneration,
-        sourceSocketIsCurrent: currentWorkerSocket === ws,
-        sourceSocketOpen: ws.readyState === WebSocket.OPEN,
-        workspaceConfigured,
-        runtimeHealthy: !shutdownAfterClose
-      });
+      const admissionCredentials = configuredCredentialGenerationFingerprint;
+      const assertMessageAdmission = () => {
+        if (operationRequestId) {
+          if (workerLeaseExpired || recoveryStore.isRequestCancelled(operationRequestId) || admissionCredentials !== configuredCredentialGenerationFingerprint || shutdownAfterClose || !workspaceConfigured) {
+            recoveryStore.unknown(operationRequestId);
+            throw new StaleWorkerAdmissionError();
+          }
+          return;
+        }
+        assertWorkerChildAdmission({
+          capturedGeneration: messageAdmissionGeneration,
+          currentGeneration: workerAdmissionGeneration,
+          sourceSocketIsCurrent: currentWorkerSocket === ws,
+          sourceSocketOpen: ws.readyState === WebSocket.OPEN,
+          workspaceConfigured,
+          runtimeHealthy: !shutdownAfterClose
+        });
+      };
       if (message.type === "connected") {
         return;
       }
@@ -5651,11 +5852,15 @@ async function startWorker(options, projectRuntime = {
       if (message.type === "workspace_config") {
         const receiptGeneration = ++workspaceConfigurationReceiptGeneration;
         const refreshesDeferredConfiguration = deferredWorkspaceConfiguration !== null && activeWorkspaceIncidentId === null && message.deferWorkspaceSyncForIncidentId === void 0;
-        advanceWorkerAdmissionGeneration();
-        const configured = await configureWorkerWorkspace(message, receiptGeneration);
-        unsafeWorkspaceSyncBouncePending = false;
-        if (configured.result.outcome !== "failed") unsafeWorkspaceSyncRecoveryAttempts = 0;
-        sendWorkerMessageFromCurrentSource(ws, {
+        const digest = workerOperationDigest(message);
+        const cached = projectRuntime.configuration;
+        const canReuseConfiguration = cached?.digest === digest && cached.workspaceSync?.remoteUrl === message.workspaceRemoteUrl && Boolean(cached.workspaceSync.credentialHelper) && cached.workspaceSync.credentialUsername === workerCredentialUsername && cached.workspaceSync.gitIdentity?.name === message.gitIdentity.name && cached.workspaceSync.gitIdentity?.email === message.gitIdentity.email && !unsafeWorkspaceSyncBouncePending && credentialGenerationFingerprint({ ...message, workerBaseUrl: baseUrl, workerAuthHeader: bearerAuthHeader }) === configuredCredentialGenerationFingerprint;
+        if (!canReuseConfiguration) {
+          advanceWorkerAdmissionGeneration();
+          projectRuntime.configuration = void 0;
+        }
+        const configured = canReuseConfiguration ? cached.configured : await configureWorkerWorkspace(message, receiptGeneration);
+        const sendConfigured = () => sendWorkerMessageFromCurrentSource(ws, {
           type: "workspace_configured",
           requestId: message.requestId,
           result: configured.result,
@@ -5663,6 +5868,34 @@ async function startWorker(options, projectRuntime = {
           aheadOfOriginBranches: configured.aheadOfOriginBranches,
           ...configured.deferredWorkspaceSyncForIncidentId ? { deferredWorkspaceSyncForIncidentId: configured.deferredWorkspaceSyncForIncidentId } : {}
         });
+        if (receiptGeneration !== workspaceConfigurationReceiptGeneration || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
+          sendConfigured();
+          return;
+        }
+        if (canReuseConfiguration && configured.result.outcome !== "failed") {
+          workspaceRemoteUrl = cached.workspaceSync.remoteUrl;
+          workspaceCredentialHelper = cached.workspaceSync.credentialHelper;
+          workspaceCredentialUsername = cached.workspaceSync.credentialUsername;
+          workspaceGitIdentity = cached.workspaceSync.gitIdentity;
+          workspaceConfigured = true;
+          activeWorkspaceIncidentId = cached.incidentId;
+          deferredWorkspaceConfiguration = cached.deferred;
+        }
+        projectRuntime.configuration = configured.result.outcome !== "failed" && workspaceRemoteUrl && workspaceCredentialHelper && workspaceCredentialUsername && workspaceGitIdentity ? {
+          digest,
+          workspaceSync: {
+            remoteUrl: workspaceRemoteUrl,
+            credentialHelper: workspaceCredentialHelper,
+            credentialUsername: workspaceCredentialUsername,
+            gitIdentity: workspaceGitIdentity
+          },
+          configured,
+          incidentId: activeWorkspaceIncidentId,
+          deferred: deferredWorkspaceConfiguration
+        } : void 0;
+        unsafeWorkspaceSyncBouncePending = false;
+        if (configured.result.outcome !== "failed") unsafeWorkspaceSyncRecoveryAttempts = 0;
+        sendConfigured();
         sendLifecycleProgress(
           configured.result.outcome === "failed" ? {
             phase: "failed",
@@ -5698,16 +5931,31 @@ async function startWorker(options, projectRuntime = {
         return;
       }
       if (message.type === "sync_workspace") {
-        await runWorkspaceSync({
-          requestId: message.requestId,
-          attemptId: message.attemptId,
-          trigger: message.trigger,
-          confirmedLargeDiff: message.confirmedLargeDiff,
-          confirmationReason: message.confirmationReason,
-          resetToCanonical: message.resetToCanonical,
-          requiredAncestorHeads: message.requiredAncestorHeads
-        });
-        return;
+        let retry = 0;
+        let recoveryDeadline;
+        while (true) {
+          const result = await runWorkspaceSync({
+            requestId: message.requestId,
+            attemptId: message.attemptId,
+            trigger: message.trigger,
+            confirmedLargeDiff: message.confirmedLargeDiff,
+            confirmationReason: message.confirmationReason,
+            resetToCanonical: message.resetToCanonical,
+            requiredAncestorHeads: message.requiredAncestorHeads,
+            sendResult: false
+          });
+          if (result.outcome !== "failed" || !isWorkerCommunicationFailure(result.error ?? "")) {
+            sendWorkspaceSyncResult(message.requestId, result);
+            return;
+          }
+          recoveryDeadline ??= Date.now() + WORKER_RECOVERY_TIMEOUT_MS;
+          if (workerLeaseExpired || Date.now() >= recoveryDeadline) {
+            recoveryStore.unknown(message.requestId);
+            return;
+          }
+          currentWorkerSocket?.close(1012, "Communication recovery");
+          await Bun.sleep(workerReconnectDelayMs(Math.random, retry++));
+        }
       }
       if (message.type === "create_project_branch") {
         if (workspaceOperationsAreFenced(deferredWorkspaceConfiguration, activeWorkspaceIncidentId)) {
@@ -5973,6 +6221,7 @@ async function startWorker(options, projectRuntime = {
       }
       if (message.type === "exec_terminal_ack") {
         pendingProcessTerminals.delete(message.runId);
+        recoveryStore.acknowledgeTerminal(message.runId);
         return;
       }
       if (message.type === "port_forward_connect") {
@@ -6024,7 +6273,7 @@ async function startWorker(options, projectRuntime = {
         return;
       }
       if (message.type === "ping") {
-        ws.send(JSON.stringify({ type: "pong" }));
+        sendSerializedWorkerMessage(ws, JSON.stringify({ type: "pong" }));
         return;
       }
       if (message.type === "cancel") {
@@ -6044,7 +6293,8 @@ async function startWorker(options, projectRuntime = {
           cancelledProcessRuns.add(message.runId);
           cancelMessage = "Cancellation queued before command start";
         }
-        ws.send(
+        sendSerializedWorkerMessage(
+          ws,
           JSON.stringify({
             type: "cancel_result",
             requestId: message.requestId,
@@ -6056,7 +6306,8 @@ async function startWorker(options, projectRuntime = {
         return;
       }
       if (message.type === "exec_stdin") {
-        const sendAck = (payload) => ws.send(
+        const sendAck = (payload) => sendSerializedWorkerMessage(
+          ws,
           JSON.stringify({
             type: "operation_result",
             requestId: message.requestId,
@@ -6179,16 +6430,19 @@ async function startWorker(options, projectRuntime = {
             ptyId: message.ptyId,
             error: `Shell input is deferred for workspace incident ${deferredWorkspaceConfiguration?.incidentId ?? activeWorkspaceIncidentId ?? "unknown"}`
           });
+          sendWorkerMessage(ws, { type: "operation_completed", requestId: message.requestId });
           return;
         }
         if (activePty && targetMayMutateVisibleWorkspace(activePty.target)) {
           markWorkspaceDirty({ type: "shell_inline", detail: `pty ${message.ptyId}` }, false, activePty.target);
         }
         writePty(ws, message);
+        sendWorkerMessage(ws, { type: "operation_completed", requestId: message.requestId });
         return;
       }
       if (message.type === "pty_resize") {
         resizePty(message);
+        sendWorkerMessage(ws, { type: "operation_completed", requestId: message.requestId });
         return;
       }
       if (message.type === "pty_close") {
@@ -6197,6 +6451,7 @@ async function startWorker(options, projectRuntime = {
         if (activePty && targetMayMutateVisibleWorkspace(activePty.target)) {
           markWorkspaceDirty({ type: "process_terminal", detail: `pty ${message.ptyId} closed` }, true, activePty.target);
         }
+        sendWorkerMessage(ws, { type: "operation_completed", requestId: message.requestId });
         return;
       }
       if (message.type === "exec") {
@@ -6278,7 +6533,11 @@ async function startWorker(options, projectRuntime = {
         } finally {
           if (targetReserved) workspaceSyncPriorityProcessTargets.delete(message.runId);
         }
-        ws.send(JSON.stringify(result));
+        if (infrastructureStoppedRunIds.has(message.runId)) {
+          recoveryStore.unknown(message.requestId);
+          return;
+        }
+        sendSerializedWorkerMessage(ws, JSON.stringify(result));
         if (hasWorkspaceEffect && targetMayMutateVisibleWorkspace(message.target)) {
           markWorkspaceDirty(
             { type: "shell_inline", sessionId: message.sessionId, processRunId: message.runId, detail: "foreground command completed" },
@@ -6401,7 +6660,8 @@ async function startWorker(options, projectRuntime = {
               assertAdmission: assertMessageAdmission
             });
           });
-          ws.send(
+          sendSerializedWorkerMessage(
+            ws,
             JSON.stringify({
               type: "operation_result",
               requestId: message.requestId,
@@ -6418,7 +6678,8 @@ async function startWorker(options, projectRuntime = {
             markWorkspaceDirty(dirtyTrigger, false, message.target);
           }
         } catch (error) {
-          ws.send(
+          sendSerializedWorkerMessage(
+            ws,
             JSON.stringify({
               type: "operation_result",
               requestId: message.requestId,
@@ -6437,7 +6698,8 @@ async function startWorker(options, projectRuntime = {
       }
       const unhandledMessage = message;
       if (typeof unhandledMessage.requestId === "string") {
-        ws.send(
+        sendSerializedWorkerMessage(
+          ws,
           JSON.stringify({
             type: "operation_result",
             requestId: unhandledMessage.requestId,
@@ -6451,6 +6713,7 @@ async function startWorker(options, projectRuntime = {
     });
   });
   ws.addEventListener("close", (event) => {
+    clearTimeout(connectionAttemptTimer);
     const processEvents = process;
     processEvents.off("SIGTERM", handleGracefulShutdown);
     processEvents.off("SIGINT", handleGracefulShutdown);
@@ -6467,15 +6730,20 @@ async function startWorker(options, projectRuntime = {
       clearTimeout(deferredWorkspaceConfigurationRefreshTimer);
       deferredWorkspaceConfigurationRefreshTimer = void 0;
     }
+    if (outputReplayTimer) {
+      clearInterval(outputReplayTimer);
+      outputReplayTimer = void 0;
+    }
     if (terminalReplayTimer) {
       clearInterval(terminalReplayTimer);
       terminalReplayTimer = void 0;
     }
-    if (currentWorkerSocket === ws) {
-      advanceWorkerAdmissionGeneration();
-      currentWorkerSocket = null;
+    if (currentWorkerSocket === ws && projectRuntime.configuration) {
+      projectRuntime.configuration.incidentId = activeWorkspaceIncidentId;
+      projectRuntime.configuration.deferred = deferredWorkspaceConfiguration;
     }
-    closeAllPtys();
+    if (currentWorkerSocket === ws) currentWorkerSocket = null;
+    workerRecoveryId = "reconnecting";
     const reason = event.reason ? `: ${event.reason}` : "";
     process.stderr.write(`[r5d-worker] disconnected (${event.code}${reason})
 `);
@@ -6485,50 +6753,19 @@ async function startWorker(options, projectRuntime = {
     if (shutdownAfterClose) {
       process.exit(0);
     }
-    if (!startupRecoveryComplete) {
-      process.stderr.write("[r5d-worker] control connection closed during workspace recovery; finishing recovery before reconnecting\n");
-      void startupRecoveryPromise.then(() => {
-        if (!startupRecoveryComplete) {
-          process.exit(1);
-          return;
-        }
-        const reconnectDelayMs = workerReconnectDelayMs();
-        process.stderr.write(`[r5d-worker] workspace recovery complete; reconnecting in ${reconnectDelayMs}ms
+    const reconnect = () => {
+      const delay = workerReconnectDelayMs(Math.random, workerReconnectAttempt++);
+      process.stderr.write(`[r5d-worker] reconnecting in ${delay}ms
 `);
-        setTimeout(() => {
-          void startWorker(options, projectRuntime).catch((error) => {
-            process.stderr.write(`[r5d-worker] reconnect failed: ${error instanceof Error ? error.message : String(error)}
-`);
-            process.exit(error instanceof WorkerServerUnavailableError ? WORKER_RECONNECT_EXIT_CODE : 1);
-          });
-        }, reconnectDelayMs);
-      });
-      return;
-    }
-    if (activeProcesses.size > 0 || credentialBearingProcessGroups.size > 0 || activePtys.size > 0 || pendingProcessTerminals.size > 0) {
-      const scheduleReconnect = (reconnect2) => {
-        const reconnectDelayMs = workerReconnectDelayMs();
-        process.stderr.write(
-          `[r5d-worker] ${activeProcesses.size} process(es), ${credentialBearingProcessGroups.size} owned process group(s), and ${activePtys.size} PTY(s) active with ${pendingProcessTerminals.size} terminal report(s) pending; reconnecting in ${reconnectDelayMs}ms
-`
-        );
-        setTimeout(reconnect2, reconnectDelayMs);
-      };
-      const reconnect = () => {
-        void startWorker(options, projectRuntime).catch((error) => {
+      setTimeout(() => {
+        void startupRecoveryPromise.then(() => startWorker(options, projectRuntime)).catch((error) => {
           process.stderr.write(`[r5d-worker] reconnect failed: ${error instanceof Error ? error.message : String(error)}
 `);
-          if (activeProcesses.size > 0 || credentialBearingProcessGroups.size > 0 || activePtys.size > 0 || pendingProcessTerminals.size > 0) {
-            scheduleReconnect(reconnect);
-            return;
-          }
-          process.exit(error instanceof WorkerServerUnavailableError ? WORKER_RECONNECT_EXIT_CODE : 1);
+          reconnect();
         });
-      };
-      scheduleReconnect(reconnect);
-      return;
-    }
-    process.exit(WORKER_RECONNECT_EXIT_CODE);
+      }, delay);
+    };
+    reconnect();
   });
   ws.addEventListener("error", (event) => {
     const message = "message" in event && typeof event.message === "string" && event.message.trim() ? `: ${event.message.trim()}` : "";