@ricsam/r5d-worker 0.0.107 → 0.0.109

package/dist/mjs/main.mjs

@@ -1,5 +1,14 @@
 #!/usr/bin/env bun
+import { getWorkerPlanPath, getWorkerPlansPath, validatePlanId } from "./plan-paths.mjs";
+import { createPlanMarkdown, updateTaskCheckbox } from "./plan-parser.mjs";
+import { lintPlans, formatPlanLintErrors } from "./plan-lint.mjs";
 import fs from "node:fs";
+import { WorkerRecoveryStore, workerOperationDigest } from "./recovery-store.mjs";
+import { WorkerOutputWindow } from "./recovery-output-window.mjs";
+import {
+  WORKER_RESUMABLE_PROTOCOL,
+  WORKER_RECOVERY_TIMEOUT_MS
+} from "./recovery-protocol.mjs";
 import path from "node:path";
 import { createHash } from "node:crypto";
 import os, { hostname } from "node:os";
@@ -169,6 +178,63 @@ const cancelledProcessRuns = /* @__PURE__ */ new Set();
 const activePtys = /* @__PURE__ */ new Map();
 const workspaceSyncPriorityPtyTargets = /* @__PURE__ */ new Map();
 let currentWorkerSocket = null;
+const workerRuntimeId = crypto.randomUUID();
+const workerOutputWindow = new WorkerOutputWindow();
+let recoveryStore;
+let workerReconnectAttempt = 0;
+let workerRecoveryId;
+let workerExecutionLeaseDeadline = 0;
+let workerExecutionLeaseTimer;
+let workerLeaseExpired = false;
+const infrastructureStoppedRunIds = /* @__PURE__ */ new Set();
+function isWorkerCommunicationFailure(message) {
+  return /Could not resolve host|Couldn't connect|Failed to connect|Connection reset|Connection timed out|remote end hung up|Empty reply|requested URL returned error: (?:408|429|5\d\d)|fetch failed|socket connection|ECONNRESET|ECONNREFUSED|ETIMEDOUT|Unable to connect|network connection was lost|Worker communication recovery expired|operation was aborted due to timeout|operation timed out|Failed to (?:fetch|download).*\((?:408|429|5\d\d)\)/i.test(
+    message
+  );
+}
+async function retryWorkerServerRead(read) {
+  let deadline;
+  let retry = 0;
+  while (true) {
+    try {
+      return await read();
+    } catch (error) {
+      if (!recoveryStore || !isWorkerCommunicationFailure(error instanceof Error ? error.message : String(error))) throw error;
+      deadline ??= Date.now() + WORKER_RECOVERY_TIMEOUT_MS;
+      if (workerLeaseExpired || Date.now() >= deadline) throw new WorkerServerUnavailableError("Worker communication recovery expired");
+      currentWorkerSocket?.close(1012, "Communication recovery");
+      await Bun.sleep(workerReconnectDelayMs(Math.random, retry++));
+    }
+  }
+}
+function renewWorkerExecutionLease(leaseMs) {
+  if (!Number.isFinite(leaseMs) || leaseMs <= 0) return;
+  workerLeaseExpired = false;
+  workerExecutionLeaseDeadline = Date.now() + Math.min(leaseMs, WORKER_RECOVERY_TIMEOUT_MS);
+  if (workerExecutionLeaseTimer) clearTimeout(workerExecutionLeaseTimer);
+  workerExecutionLeaseTimer = setTimeout(
+    () => {
+      if (Date.now() < workerExecutionLeaseDeadline) return;
+      workerLeaseExpired = true;
+      for (const operation of recoveryStore?.operations() ?? []) {
+        if (operation.sessionId && operation.state !== "completed") recoveryStore?.cancelSession(operation.sessionId);
+      }
+      for (const [runId, active] of activeProcesses) {
+        if (!active.sessionId) continue;
+        infrastructureStoppedRunIds.add(runId);
+        recoveryStore?.unknownRun(runId);
+        cancelledProcessRuns.add(runId);
+        closeProcessStdin(active);
+        void terminateProcessTree(active.process).catch(
+          (error) => process.stderr.write(`[r5d-worker] execution lease termination failed: ${error}
+`)
+        );
+      }
+    },
+    Math.min(leaseMs, WORKER_RECOVERY_TIMEOUT_MS)
+  );
+  workerExecutionLeaseTimer.unref();
+}
 let workerAdmissionGeneration = 0;
 const workspaceMutationGate = new WorkspaceMutationGate();
 let workspaceSyncQueue = Promise.resolve();
@@ -627,40 +693,46 @@ function parseRemoteFileManifest(value, collectionKey, label) {
   });
 }
 async function fetchRemoteFileManifest(input) {
-  const response = await fetch(input.url, {
-    headers: {
-      Authorization: `Bearer ${input.token}`
+  return retryWorkerServerRead(async () => {
+    const response = await fetch(input.url, {
+      signal: AbortSignal.timeout(3e4),
+      headers: {
+        Authorization: `Bearer ${input.token}`
+      }
+    });
+    if (!response.ok) {
+      const detail = (await readResponseText(response)).trim();
+      throw new Error(`Failed to fetch ${input.label} manifest (${response.status})${detail ? `: ${detail}` : ""}`);
     }
+    return parseRemoteFileManifest(await response.json(), input.collectionKey, input.label);
   });
-  if (!response.ok) {
-    const detail = (await readResponseText(response)).trim();
-    throw new Error(`Failed to fetch ${input.label} manifest (${response.status})${detail ? `: ${detail}` : ""}`);
-  }
-  return parseRemoteFileManifest(await response.json(), input.collectionKey, input.label);
 }
 async function downloadRemoteSyncedFile(input) {
-  const targetPath = path.join(input.targetDir, input.file.filename);
-  assertInsideRoot(input.targetDir, targetPath, input.label);
-  const response = await fetch(input.url, {
-    headers: {
-      Authorization: `Bearer ${input.token}`
+  return retryWorkerServerRead(async () => {
+    const targetPath = path.join(input.targetDir, input.file.filename);
+    assertInsideRoot(input.targetDir, targetPath, input.label);
+    const response = await fetch(input.url, {
+      signal: AbortSignal.timeout(3e4),
+      headers: {
+        Authorization: `Bearer ${input.token}`
+      }
+    });
+    if (!response.ok) {
+      const detail = (await readResponseText(response)).trim();
+      throw new Error(`Failed to download ${input.label} ${input.file.filename} (${response.status})${detail ? `: ${detail}` : ""}`);
+    }
+    const buffer = Buffer.from(await response.arrayBuffer());
+    const receivedHash = createHash("sha256").update(buffer).digest("hex");
+    if (receivedHash !== input.file.sha256) {
+      throw new Error(`Downloaded ${input.label} ${input.file.filename} failed sha256 verification`);
+    }
+    if (buffer.length !== input.file.size) {
+      throw new Error(`Downloaded ${input.label} ${input.file.filename} has unexpected size`);
     }
+    const tempPath = `${targetPath}.${process.pid}.${Date.now()}.tmp`;
+    fs.writeFileSync(tempPath, buffer);
+    fs.renameSync(tempPath, targetPath);
   });
-  if (!response.ok) {
-    const detail = (await readResponseText(response)).trim();
-    throw new Error(`Failed to download ${input.label} ${input.file.filename} (${response.status})${detail ? `: ${detail}` : ""}`);
-  }
-  const buffer = Buffer.from(await response.arrayBuffer());
-  const receivedHash = createHash("sha256").update(buffer).digest("hex");
-  if (receivedHash !== input.file.sha256) {
-    throw new Error(`Downloaded ${input.label} ${input.file.filename} failed sha256 verification`);
-  }
-  if (buffer.length !== input.file.size) {
-    throw new Error(`Downloaded ${input.label} ${input.file.filename} has unexpected size`);
-  }
-  const tempPath = `${targetPath}.${process.pid}.${Date.now()}.tmp`;
-  fs.writeFileSync(tempPath, buffer);
-  fs.renameSync(tempPath, targetPath);
 }
 async function syncRemoteFileSet(input) {
   fs.mkdirSync(input.targetDir, { recursive: true });
@@ -787,6 +859,7 @@ async function verifyR5dctlAuth(baseUrl, token, workerLabel) {
   let response;
   try {
     response = await fetch(statusUrl, {
+      signal: AbortSignal.timeout(3e4),
       headers: {
         Authorization: `Bearer ${token}`
       }
@@ -845,11 +918,6 @@ function validateProjectId(projectId) {
 function validateBranchName(branchName) {
   validateManagedBranchName(branchName);
 }
-function validatePlanId(planId) {
-  if (!/^[a-z0-9][a-z0-9-]*$/.test(planId)) {
-    throw new Error(`Invalid plan id from server: ${planId}`);
-  }
-}
 const NON_RECURSIVE_GIT_CONFIG_ARGS = [
   "-c",
   "submodule.recurse=false",
@@ -1756,6 +1824,7 @@ const workerGitSecurityTestHarness = {
   workspaceSyncFailureHydrationIsSafe,
   assertWorkerChildAdmission,
   executeWriteFileOperation,
+  executeOperation,
   executeEditFileOperation,
   terminateCredentialBearingChildren,
   terminateCredentialBearingChildrenWithRetention,
@@ -2194,6 +2263,55 @@ function editWorkerTextFile(branchPath, filePath, edits, builtInPaths) {
     edits: edits.length
   };
 }
+async function executePlanOperation(input) {
+  const { message, resolvedTarget } = input;
+  const filePath = message.type === "lint_plans" ? getWorkerPlansPath(resolvedTarget.target) : getWorkerPlanPath(resolvedTarget.target, message.planId);
+  const builtInPaths = await prepareBuiltInToolPathsForTarget({
+    target: resolvedTarget.target,
+    filePath,
+    baseUrl: input.baseUrl,
+    token: input.token,
+    rootDir: input.rootDir,
+    sessionId: message.sessionId,
+    access: message.type === "update_task_status" || message.type === "read_plan" && message.createIfMissing ? "write" : "read"
+  });
+  const resolved = resolveWorkerFilePath(resolvedTarget.rootPath, filePath, builtInPaths);
+  if (message.type === "lint_plans") {
+    const result = await lintPlans(resolved.absolutePath, message.glob);
+    return {
+      type: "lint_plans",
+      content: result.lintedPlanCount === 0 ? message.glob ? `No plan files matched glob "${message.glob}".` : "No plan files found." : formatPlanLintErrors(result.errors) || "No plan lint issues found."
+    };
+  }
+  return withFileMutationQueue(mutationQueueKey(resolvedTarget.target, resolved), async () => {
+    input.assertAdmission();
+    if (message.type === "read_plan") {
+      let created = false;
+      if (message.createIfMissing) {
+        try {
+          fs.writeFileSync(resolved.absolutePath, createPlanMarkdown(message.planId), { encoding: "utf8", flag: "wx" });
+          created = true;
+        } catch (error) {
+          if (error.code !== "EEXIST") throw error;
+        }
+      }
+      return {
+        type: "read_plan",
+        created,
+        read: readWorkerTextFile(resolvedTarget.rootPath, filePath, void 0, void 0, builtInPaths)
+      };
+    }
+    const content = fs.readFileSync(resolved.absolutePath, "utf8");
+    writeWorkerTextFile(resolvedTarget.rootPath, filePath, updateTaskCheckbox(content, message.taskId, message.checked), builtInPaths);
+    return {
+      type: "update_task_status",
+      planId: message.planId,
+      taskId: message.taskId,
+      checked: message.checked,
+      message: `Task ${message.taskId} marked ${message.checked ? "complete" : "incomplete"}.`
+    };
+  });
+}
 async function executeReadFileOperation(input) {
   const builtInPaths = await prepareBuiltInToolPathsForTarget({
     target: input.resolvedTarget.target,
@@ -2572,6 +2690,10 @@ async function executeViewFileBytesOperation(input) {
 }
 async function executeOperation(input) {
   switch (input.message.type) {
+    case "read_plan":
+    case "lint_plans":
+    case "update_task_status":
+      return executePlanOperation({ ...input, message: input.message });
     case "read":
       return executeReadFileOperation({ ...input, message: input.message });
     case "write":
@@ -2901,23 +3023,46 @@ function closeProcessStdin(active) {
   }
   active.stdin = void 0;
 }
-function sendWorkerMessage(ws, message) {
+function sendReplayWorkerMessage(ws, message) {
   const target = currentWorkerSocket ?? ws;
-  try {
-    target.send(JSON.stringify(message));
-  } catch (error) {
-    process.stderr.write(`[r5d-worker] failed to send ${message.type}: ${error instanceof Error ? error.message : String(error)}
-`);
+  if (target.readyState !== WebSocket.OPEN) return false;
+  const send = (payload) => {
+    try {
+      target.send(JSON.stringify(payload));
+      return true;
+    } catch {
+      return false;
+    }
+  };
+  return message.type === "exec_output" ? workerOutputWindow.send(message, send, target.bufferedAmount) : send(message);
+}
+function sendWorkerMessage(ws, message) {
+  if (recoveryStore && "requestId" in message && typeof message.requestId === "string" && recoveryStore.isUnknown(message.requestId))
+    return;
+  const transportError = "error" in message && typeof message.error === "string" ? message.error : message.type === "workspace_sync_result" && message.result.outcome === "failed" ? message.result.error : void 0;
+  if (recoveryStore && transportError && isWorkerCommunicationFailure(transportError) && "requestId" in message && typeof message.requestId === "string") {
+    recoveryStore.unknown(message.requestId);
+    currentWorkerSocket?.close(1012, "Communication recovery");
+    return;
   }
+  const recorded = recoveryStore?.record(message) ?? message;
+  if ((message.type === "exec_exit" || message.type === "exec_error") && infrastructureStoppedRunIds.has(message.runId)) {
+    recoveryStore?.unknownRun(message.runId);
+    return;
+  }
+  if (message.type === "exec_output" && workerRecoveryId !== void 0) return;
+  sendReplayWorkerMessage(ws, recorded);
+}
+function sendSerializedWorkerMessage(ws, serialized) {
+  sendWorkerMessage(ws, JSON.parse(serialized));
 }
 function sendWorkerMessageFromCurrentSource(ws, message) {
-  if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) return;
-  try {
-    ws.send(JSON.stringify(message));
-  } catch (error) {
-    process.stderr.write(`[r5d-worker] failed to send ${message.type}: ${error instanceof Error ? error.message : String(error)}
-`);
+  if ("requestId" in message && typeof message.requestId === "string" && message.type !== "workspace_configured") {
+    sendWorkerMessage(ws, message);
+    return;
   }
+  if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) return;
+  sendWorkerMessage(ws, message);
 }
 function buildActiveProcessReports() {
   return Array.from(activeProcesses.entries()).filter(([, active]) => active.sessionId.length > 0).map(([runId, active]) => ({
@@ -3431,14 +3576,21 @@ function websocketUrl(baseUrl, label) {
   url.searchParams.set("version", getWorkerVersion());
   return url.toString();
 }
-async function startWorker(options) {
-  advanceWorkerAdmissionGeneration();
+async function startWorker(options, projectRuntime = {
+  projectConfigById: /* @__PURE__ */ new Map(),
+  readyProjectIds: /* @__PURE__ */ new Set(),
+  reconciledProjectConfigFingerprints: /* @__PURE__ */ new Map(),
+  gitIdentity: null
+}) {
   const config = readConfig(options.configPath);
   const { baseUrl, token } = resolveCredentials(options, config);
   initializeCredentialStoreDirectoryOnce();
   const label = requireValue(options.label, "Missing required --label <label>");
   validateLabel(label);
   const rootDir = path.resolve(options.rootDir ?? process.env.R5D_ROOT ?? defaultRootDir());
+  recoveryStore ??= new WorkerRecoveryStore(
+    path.join(rootDir, ".worker-recovery", createHash("sha256").update(`${baseUrl}\0${label}`).digest("hex"), "protocol.sqlite")
+  );
   const projectsRoot = path.join(rootDir, "projects");
   const syncRoot = path.join(rootDir, "sync");
   const workspaceShadowRoot = path.join(syncRoot, "workspace");
@@ -3459,7 +3611,7 @@ async function startWorker(options) {
 `);
   runGit(["--version"]);
   await verifyR5dctlAuth(baseUrl, token, label);
-  const initializedWorkspaceState = await workspaceSyncSingleFlight.runExclusive(() => {
+  const initializedWorkspaceState = projectRuntime.initializedWorkspaceState ?? await workspaceSyncSingleFlight.runExclusive(() => {
     fs.mkdirSync(projectsRoot, { recursive: true });
     fs.mkdirSync(syncRoot, { recursive: true });
     fs.mkdirSync(artifactRoot, { recursive: true });
@@ -3470,12 +3622,11 @@ async function startWorker(options) {
     });
     return { store, state: store.read() };
   });
+  projectRuntime.initializedWorkspaceState = initializedWorkspaceState;
   const projectWorkspaceStateStore = initializedWorkspaceState.store;
-  let projectWorkspaceState = initializedWorkspaceState.state;
-  const projectConfigById = /* @__PURE__ */ new Map();
+  let projectWorkspaceState = initializedWorkspaceState.store.read();
+  const { projectConfigById, readyProjectIds, reconciledProjectConfigFingerprints } = projectRuntime;
   const pendingCheckouts = /* @__PURE__ */ new Map();
-  const readyProjectIds = /* @__PURE__ */ new Set();
-  const reconciledProjectConfigFingerprints = /* @__PURE__ */ new Map();
   const lastObservedProjectHeads = /* @__PURE__ */ new Map();
   const advertisedProjectMirrorRefsTokens = /* @__PURE__ */ new Map();
   const advertisedProjectMirrorMountTokens = /* @__PURE__ */ new Map();
@@ -3505,7 +3656,7 @@ async function startWorker(options) {
   let workspaceRemoteUrl = null;
   let workspaceCredentialHelper = null;
   let workspaceCredentialUsername = null;
-  let workspaceGitIdentity = null;
+  let workspaceGitIdentity = projectRuntime.gitIdentity;
   let workspaceConfigured = false;
   let activeWorkspaceIncidentId = null;
   let deferredWorkspaceConfiguration = null;
@@ -3513,11 +3664,12 @@ async function startWorker(options) {
   let workspaceConfigurationReceiptGeneration = 0;
   let workspaceAutomaticTimer;
   let workspacePeriodicTimer;
-  let pendingAutomaticTrigger;
+  let pendingAutomaticTrigger = projectRuntime.pendingAutomaticTrigger;
   const pendingCreatedBranchPublicationNotBefore = /* @__PURE__ */ new Map();
   const workspaceProjectionLedger = new WorkspaceProjectionLedger();
   let automaticSyncInFlight = false;
   let terminalReplayTimer;
+  let outputReplayTimer;
   let workspaceSyncRequestsInFlight = 0;
   let lastWorkspaceSyncMirrorObservationMs = 0;
   let lastInventoriedMissingCheckouts;
@@ -3866,7 +4018,9 @@ async function startWorker(options) {
   const reportedMissingCheckouts = (inventory) => inventory.missingCheckouts.map(({ projectId, branchName, reason }) => ({ projectId, branchName, reason }));
   const logMissingCheckouts = (inventory) => {
     if (inventory.missingCheckouts.length === 0) return;
-    const described = inventory.missingCheckouts.map(({ checkoutPath, reason }) => `${checkoutPath} (${reason === "missing" ? "directory missing" : "not a linked worktree of the project"})`).join(", ");
+    const described = inventory.missingCheckouts.map(
+      ({ checkoutPath, reason }) => `${checkoutPath} (${reason === "missing" ? "directory missing" : "not a linked worktree of the project"})`
+    ).join(", ");
     process.stderr.write(
       `[r5d-worker] skipping ${inventory.missingCheckouts.length} configured project checkout(s) this cycle: ${described}; delete the branch through the platform or reconnect this worker to recreate the checkout
 `
@@ -4437,7 +4591,9 @@ async function startWorker(options) {
       error: error instanceof Error ? error.message : String(error)
     };
   };
-  const workspaceHydrationTransactionHooks = createWorkspaceHydrationLeaseHooks((message) => sendWorkerMessageFromCurrentSource(ws, message));
+  const workspaceHydrationTransactionHooks = createWorkspaceHydrationLeaseHooks(
+    (message) => sendWorkerMessageFromCurrentSource(ws, message)
+  );
   const performWorkspaceSync = async (input) => {
     if (!workspaceRemoteUrl || !workspaceCredentialHelper || !workspaceCredentialUsername || !workspaceGitIdentity) {
       throw new Error("Worker workspace configuration has not been received");
@@ -4494,7 +4650,11 @@ async function startWorker(options) {
     const outerRemediation = input.trigger.type === "remediation" || input.trigger.type === "remediation_confirm";
     const mounts = outerRemediation ? workspaceGitMountsForRemediation(configuredMounts) : configuredMounts;
     if (outerRemediation) workspaceProjectionLedger.reset();
-    const observedProjectHeads = outerRemediation ? [] : await observeProjectHeadsBeforeOuterWorkspace({ allowNonFastForward: false, failClosed: false, projects: checkoutInventory.projects });
+    const observedProjectHeads = outerRemediation ? [] : await observeProjectHeadsBeforeOuterWorkspace({
+      allowNonFastForward: false,
+      failClosed: false,
+      projects: checkoutInventory.projects
+    });
     const expectedMirrorHeads = /* @__PURE__ */ new Map();
     for (const bundle of observedProjectHeads) {
       for (const observation of bundle.observations) {
@@ -4601,7 +4761,7 @@ async function startWorker(options) {
   });
   const workspaceSyncAdmissionSnapshotsMatch = (left, right) => left.generation === right.generation && left.activeIncidentId === right.activeIncidentId && left.deferredIncidentId === right.deferredIncidentId;
   const assertWorkspaceSyncStillAdmitted = (admitted, trigger) => {
-    if (!workspaceSyncAdmissionSnapshotsMatch(admitted, captureWorkspaceSyncAdmission()) || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN || !deferredWorkspaceSyncTriggerIsAllowed(deferredWorkspaceConfiguration, activeWorkspaceIncidentId, trigger)) {
+    if (!workspaceSyncAdmissionSnapshotsMatch(admitted, captureWorkspaceSyncAdmission()) || !deferredWorkspaceSyncTriggerIsAllowed(deferredWorkspaceConfiguration, activeWorkspaceIncidentId, trigger)) {
       throw new WorkspaceSyncAdmissionChangedError("Workspace synchronization admission changed while asynchronous work was in flight");
     }
   };
@@ -4655,6 +4815,7 @@ async function startWorker(options) {
         bouncePending: unsafeWorkspaceSyncBouncePending,
         invalidateExecution: () => {
           workspaceConfigured = false;
+          projectRuntime.configuration = void 0;
           advanceWorkerAdmissionGeneration();
         },
         scheduleRecoveryBounce: () => {
@@ -4683,7 +4844,11 @@ async function startWorker(options) {
       };
     }
   };
-  const REMEDIATION_TRIGGER_TYPES = /* @__PURE__ */ new Set(["remediation", "remediation_confirm", "remediation_reset"]);
+  const REMEDIATION_TRIGGER_TYPES = /* @__PURE__ */ new Set([
+    "remediation",
+    "remediation_confirm",
+    "remediation_reset"
+  ]);
   const isOrdinaryWorkspaceSyncRequest = (input) => input.confirmedLargeDiff !== true && input.resetToCanonical !== true && !(input.requiredAncestorHeads && input.requiredAncestorHeads.length > 0) && !REMEDIATION_TRIGGER_TYPES.has(input.trigger.type);
   const LOCAL_ACTIVITY_TRIGGER_TYPES = /* @__PURE__ */ new Set([
     "write",
@@ -4724,7 +4889,14 @@ async function startWorker(options) {
         trigger: leader.trigger,
         assertStillAdmitted: () => assertWorkspaceSyncStillAdmitted(admission, leader.trigger)
       });
-      return { result, ranCycle: true, startedAtMs, finishedAtMs: Date.now(), prepareMs: lastWorkspaceSyncMirrorObservationMs, staleAttemptIds };
+      return {
+        result,
+        ranCycle: true,
+        startedAtMs,
+        finishedAtMs: Date.now(),
+        prepareMs: lastWorkspaceSyncMirrorObservationMs,
+        staleAttemptIds
+      };
     }
   });
   const runWorkspaceSync = async (input) => {
@@ -4754,7 +4926,7 @@ async function startWorker(options) {
       if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
         throw new Error("Cannot start autonomous workspace synchronization without an open worker control socket");
       }
-      ws.send(JSON.stringify({ type: "workspace_sync_started", attemptId }));
+      sendSerializedWorkerMessage(ws, JSON.stringify({ type: "workspace_sync_started", attemptId }));
     }
     let queueMs = 0;
     let prepareMs = 0;
@@ -4806,6 +4978,7 @@ async function startWorker(options) {
   };
   const scheduleAutomaticWorkspaceSync = (trigger, delayMs = WORKSPACE_GIT_QUIET_MS) => {
     pendingAutomaticTrigger = trigger;
+    projectRuntime.pendingAutomaticTrigger = trigger;
     const pendingBranchDeferral = () => pendingCreatedBranchAutomaticSyncDeferral(
       projectWorkspaceState.locallyPendingCreatedBranches,
       activeWorkspaceMutationTargets(),
@@ -4828,6 +5001,7 @@ async function startWorker(options) {
           return;
         }
         pendingAutomaticTrigger = void 0;
+        projectRuntime.pendingAutomaticTrigger = void 0;
         automaticSyncInFlight = true;
         void runWorkspaceSync({ trigger: scheduledTrigger }).then((result) => {
           if (result.outcome === "failed") {
@@ -4851,9 +5025,14 @@ async function startWorker(options) {
     workspaceAutomaticTimer.unref();
   };
   const markWorkspaceDirty = (trigger, force = false, target) => {
+    if (projectRuntime.markWorkspaceDirty && projectRuntime.markWorkspaceDirty !== markWorkspaceDirty) {
+      projectRuntime.markWorkspaceDirty(trigger, force, target);
+      return;
+    }
     if (target) recordVisibleWorkspaceMutation(target);
     scheduleAutomaticWorkspaceSync(trigger, force ? 0 : WORKSPACE_GIT_QUIET_MS);
   };
+  projectRuntime.markWorkspaceDirty = markWorkspaceDirty;
   const targetMayMutateVisibleWorkspace = (target) => target.type === "project" || target.rootProfile === "visible_projects";
   const resolveMessageTarget = (target) => {
     if (!workspaceConfigured) throw new Error("Worker workspace configuration has not completed successfully");
@@ -5156,6 +5335,7 @@ async function startWorker(options) {
         bootstrapCredentialGenerationFingerprintAtProcessStart = null;
         visibleGitIdentity = message.gitIdentity;
         workspaceGitIdentity = message.gitIdentity;
+        projectRuntime.gitIdentity = message.gitIdentity;
         workspaceRemoteUrl = message.workspaceRemoteUrl;
         workspaceCredentialHelper = nextWorkspaceCredentialHelper;
         workspaceCredentialUsername = workerCredentialUsername;
@@ -5370,6 +5550,10 @@ async function startWorker(options) {
       Authorization: `Bearer ${token}`
     }
   });
+  const connectionAttemptTimer = setTimeout(() => {
+    if (ws.readyState === WebSocket.CONNECTING) ws.terminate();
+  }, 3e4);
+  connectionAttemptTimer.unref();
   let lastServerControlMessageAtMs = performance.now();
   let serverHeartbeatProbeStartedAtMs;
   let lastServerHeartbeatSweepAtMs = performance.now();
@@ -5465,7 +5649,9 @@ async function startWorker(options) {
   process.on("SIGTERM", handleGracefulShutdown);
   process.on("SIGINT", handleGracefulShutdown);
   ws.addEventListener("open", () => {
+    clearTimeout(connectionAttemptTimer);
     currentWorkerSocket = ws;
+    workerRecoveryId = "connecting";
     noteServerControlActivity();
     lastServerHeartbeatSweepAtMs = performance.now();
     serverHeartbeatWatchdog = setInterval(() => {
@@ -5509,6 +5695,9 @@ async function startWorker(options) {
     });
     const hello = {
       type: "hello",
+      resumableProtocol: WORKER_RESUMABLE_PROTOCOL,
+      runtimeId: workerRuntimeId,
+      ledgerId: recoveryStore.ledgerId,
       startupReady: startupRecoveryComplete,
       hostInfo: {
         hostname: hostname(),
@@ -5533,15 +5722,17 @@ async function startWorker(options) {
         root: rootDir
       }
     };
-    ws.send(JSON.stringify(hello));
-    sendActiveProcessReport(ws);
-    for (const terminal of pendingProcessTerminals.values()) {
-      sendWorkerMessage(ws, terminal);
-    }
-    terminalReplayTimer = setInterval(() => {
-      for (const terminal of pendingProcessTerminals.values()) {
-        sendWorkerMessage(ws, terminal);
+    sendSerializedWorkerMessage(ws, JSON.stringify(hello));
+    outputReplayTimer = setInterval(() => {
+      if (workerRecoveryId !== void 0 || currentWorkerSocket !== ws) return;
+      for (const output of recoveryStore.output()) {
+        if (!sendReplayWorkerMessage(ws, output)) break;
       }
+    }, 250);
+    outputReplayTimer.unref();
+    terminalReplayTimer = setInterval(() => {
+      if (workerRecoveryId !== void 0) return;
+      for (const terminal of recoveryStore?.terminals() ?? []) sendReplayWorkerMessage(ws, terminal);
     }, 5e3);
     terminalReplayTimer.unref();
     process.stdout.write("[r5d-worker] connected\n");
@@ -5549,18 +5740,103 @@ async function startWorker(options) {
   ws.addEventListener("message", (event) => {
     noteServerControlActivity();
     void (async () => {
+      if (currentWorkerSocket !== ws) return;
       const message = JSON.parse(String(event.data));
+      if (message.type === "recovery_request") {
+        workerRecoveryId = message.recoveryId;
+        sendActiveProcessReport(ws);
+        sendWorkerMessage(ws, {
+          type: "recovery_state",
+          recoveryId: message.recoveryId,
+          runtimeId: workerRuntimeId,
+          ledgerId: recoveryStore.ledgerId,
+          operations: recoveryStore.operations()
+        });
+        const cursors = new Map(recoveryStore.outputCursors().map((cursor) => [cursor.runId, cursor]));
+        for (const cursor of message.outputOffsets) cursors.set(cursor.runId, cursor);
+        workerOutputWindow.reset([...cursors.values()]);
+        for (const response of recoveryStore.responses(message.requests)) sendReplayWorkerMessage(ws, response);
+        for (const output of recoveryStore.output(message.outputOffsets)) {
+          while (true) {
+            if (currentWorkerSocket !== ws || workerRecoveryId !== message.recoveryId || ws.readyState !== WebSocket.OPEN || workerLeaseExpired)
+              return;
+            if (sendReplayWorkerMessage(ws, output)) break;
+            await Bun.sleep(10);
+          }
+        }
+        if (currentWorkerSocket !== ws || workerRecoveryId !== message.recoveryId) return;
+        for (const terminal of recoveryStore.terminals()) sendReplayWorkerMessage(ws, terminal);
+        sendWorkerMessage(ws, { type: "recovery_complete", recoveryId: message.recoveryId });
+        return;
+      }
+      if (message.type === "recovery_ready") {
+        if (workerRecoveryId !== message.recoveryId) return;
+        workerRecoveryId = void 0;
+        workerReconnectAttempt = 0;
+        renewWorkerExecutionLease(message.leaseMs);
+        return;
+      }
+      if (message.type === "worker_execution_lease") {
+        renewWorkerExecutionLease(message.leaseMs);
+        return;
+      }
+      if (message.type === "operation_result_ack") {
+        recoveryStore.acknowledgeResult(message.requestId);
+        return;
+      }
+      if (message.type === "exec_output_ack") {
+        recoveryStore.acknowledgeOutput(message.runId, message.stream, message.nextOffset);
+        workerOutputWindow.acknowledge(message.runId, message.stream, message.nextOffset);
+        return;
+      }
+      if (message.type === "cancel_session") {
+        recoveryStore.cancelSession(message.sessionId);
+        await Promise.all(
+          [...activeProcesses].filter(([, active]) => active.sessionId === message.sessionId).map(async ([runId, active]) => {
+            cancelledProcessRuns.add(runId);
+            closeProcessStdin(active);
+            await terminateProcessTree(active.process);
+          })
+        );
+        sendWorkerMessage(ws, { type: "cancel_session_ack", cancellationId: message.cancellationId });
+        return;
+      }
       if (!startupRecoveryComplete && message.type !== "ping") await startupRecoveryPromise;
       if (!startupRecoveryComplete && message.type !== "ping") return;
+      if (currentWorkerSocket !== ws) return;
+      const operationRequestId = "requestId" in message && typeof message.requestId === "string" && message.type !== "workspace_config" ? message.requestId : void 0;
+      if (operationRequestId) {
+        const admission = recoveryStore.admit({ ...message, requestId: operationRequestId });
+        if (admission === "conflict") {
+          ws.close(1008, "Operation identity conflict");
+          return;
+        }
+        sendWorkerMessage(ws, { type: "operation_received", requestId: operationRequestId });
+        if (admission !== "new") {
+          const response = recoveryStore.response(operationRequestId);
+          if (response && admission !== "unknown") sendReplayWorkerMessage(ws, response);
+          return;
+        }
+      }
       const messageAdmissionGeneration = workerAdmissionGeneration;
-      const assertMessageAdmission = () => assertWorkerChildAdmission({
-        capturedGeneration: messageAdmissionGeneration,
-        currentGeneration: workerAdmissionGeneration,
-        sourceSocketIsCurrent: currentWorkerSocket === ws,
-        sourceSocketOpen: ws.readyState === WebSocket.OPEN,
-        workspaceConfigured,
-        runtimeHealthy: !shutdownAfterClose
-      });
+      const admissionCredentials = configuredCredentialGenerationFingerprint;
+      const assertMessageAdmission = () => {
+        if (operationRequestId) {
+          if (workerLeaseExpired || recoveryStore.isRequestCancelled(operationRequestId) || admissionCredentials !== configuredCredentialGenerationFingerprint || shutdownAfterClose || !workspaceConfigured) {
+            recoveryStore.unknown(operationRequestId);
+            throw new StaleWorkerAdmissionError();
+          }
+          return;
+        }
+        assertWorkerChildAdmission({
+          capturedGeneration: messageAdmissionGeneration,
+          currentGeneration: workerAdmissionGeneration,
+          sourceSocketIsCurrent: currentWorkerSocket === ws,
+          sourceSocketOpen: ws.readyState === WebSocket.OPEN,
+          workspaceConfigured,
+          runtimeHealthy: !shutdownAfterClose
+        });
+      };
       if (message.type === "connected") {
         return;
       }
@@ -5576,8 +5852,17 @@ async function startWorker(options) {
       if (message.type === "workspace_config") {
         const receiptGeneration = ++workspaceConfigurationReceiptGeneration;
         const refreshesDeferredConfiguration = deferredWorkspaceConfiguration !== null && activeWorkspaceIncidentId === null && message.deferWorkspaceSyncForIncidentId === void 0;
-        advanceWorkerAdmissionGeneration();
-        const configured = await configureWorkerWorkspace(message, receiptGeneration);
+        const digest = workerOperationDigest(message);
+        const cached = projectRuntime.configuration;
+        if (cached?.digest !== digest) advanceWorkerAdmissionGeneration();
+        const canReuseConfiguration = cached?.digest === digest && !unsafeWorkspaceSyncBouncePending && credentialGenerationFingerprint({ ...message, workerBaseUrl: baseUrl, workerAuthHeader: bearerAuthHeader }) === configuredCredentialGenerationFingerprint;
+        const configured = canReuseConfiguration ? cached.configured : await configureWorkerWorkspace(message, receiptGeneration);
+        if (canReuseConfiguration && configured.result.outcome !== "failed") {
+          workspaceConfigured = true;
+          activeWorkspaceIncidentId = cached.incidentId;
+          deferredWorkspaceConfiguration = cached.deferred;
+        }
+        projectRuntime.configuration = configured.result.outcome !== "failed" ? { digest, configured, incidentId: activeWorkspaceIncidentId, deferred: deferredWorkspaceConfiguration } : void 0;
         unsafeWorkspaceSyncBouncePending = false;
         if (configured.result.outcome !== "failed") unsafeWorkspaceSyncRecoveryAttempts = 0;
         sendWorkerMessageFromCurrentSource(ws, {
@@ -5623,16 +5908,31 @@ async function startWorker(options) {
         return;
       }
       if (message.type === "sync_workspace") {
-        await runWorkspaceSync({
-          requestId: message.requestId,
-          attemptId: message.attemptId,
-          trigger: message.trigger,
-          confirmedLargeDiff: message.confirmedLargeDiff,
-          confirmationReason: message.confirmationReason,
-          resetToCanonical: message.resetToCanonical,
-          requiredAncestorHeads: message.requiredAncestorHeads
-        });
-        return;
+        let retry = 0;
+        let recoveryDeadline;
+        while (true) {
+          const result = await runWorkspaceSync({
+            requestId: message.requestId,
+            attemptId: message.attemptId,
+            trigger: message.trigger,
+            confirmedLargeDiff: message.confirmedLargeDiff,
+            confirmationReason: message.confirmationReason,
+            resetToCanonical: message.resetToCanonical,
+            requiredAncestorHeads: message.requiredAncestorHeads,
+            sendResult: false
+          });
+          if (result.outcome !== "failed" || !isWorkerCommunicationFailure(result.error ?? "")) {
+            sendWorkspaceSyncResult(message.requestId, result);
+            return;
+          }
+          recoveryDeadline ??= Date.now() + WORKER_RECOVERY_TIMEOUT_MS;
+          if (workerLeaseExpired || Date.now() >= recoveryDeadline) {
+            recoveryStore.unknown(message.requestId);
+            return;
+          }
+          currentWorkerSocket?.close(1012, "Communication recovery");
+          await Bun.sleep(workerReconnectDelayMs(Math.random, retry++));
+        }
       }
       if (message.type === "create_project_branch") {
         if (workspaceOperationsAreFenced(deferredWorkspaceConfiguration, activeWorkspaceIncidentId)) {
@@ -5898,6 +6198,7 @@ async function startWorker(options) {
       }
       if (message.type === "exec_terminal_ack") {
         pendingProcessTerminals.delete(message.runId);
+        recoveryStore.acknowledgeTerminal(message.runId);
         return;
       }
       if (message.type === "port_forward_connect") {
@@ -5949,7 +6250,7 @@ async function startWorker(options) {
         return;
       }
       if (message.type === "ping") {
-        ws.send(JSON.stringify({ type: "pong" }));
+        sendSerializedWorkerMessage(ws, JSON.stringify({ type: "pong" }));
         return;
       }
       if (message.type === "cancel") {
@@ -5969,7 +6270,8 @@ async function startWorker(options) {
           cancelledProcessRuns.add(message.runId);
           cancelMessage = "Cancellation queued before command start";
         }
-        ws.send(
+        sendSerializedWorkerMessage(
+          ws,
           JSON.stringify({
             type: "cancel_result",
             requestId: message.requestId,
@@ -5981,7 +6283,8 @@ async function startWorker(options) {
         return;
       }
       if (message.type === "exec_stdin") {
-        const sendAck = (payload) => ws.send(
+        const sendAck = (payload) => sendSerializedWorkerMessage(
+          ws,
           JSON.stringify({
             type: "operation_result",
             requestId: message.requestId,
@@ -6104,16 +6407,19 @@ async function startWorker(options) {
             ptyId: message.ptyId,
             error: `Shell input is deferred for workspace incident ${deferredWorkspaceConfiguration?.incidentId ?? activeWorkspaceIncidentId ?? "unknown"}`
           });
+          sendWorkerMessage(ws, { type: "operation_completed", requestId: message.requestId });
           return;
         }
         if (activePty && targetMayMutateVisibleWorkspace(activePty.target)) {
           markWorkspaceDirty({ type: "shell_inline", detail: `pty ${message.ptyId}` }, false, activePty.target);
         }
         writePty(ws, message);
+        sendWorkerMessage(ws, { type: "operation_completed", requestId: message.requestId });
         return;
       }
       if (message.type === "pty_resize") {
         resizePty(message);
+        sendWorkerMessage(ws, { type: "operation_completed", requestId: message.requestId });
         return;
       }
       if (message.type === "pty_close") {
@@ -6122,6 +6428,7 @@ async function startWorker(options) {
         if (activePty && targetMayMutateVisibleWorkspace(activePty.target)) {
           markWorkspaceDirty({ type: "process_terminal", detail: `pty ${message.ptyId} closed` }, true, activePty.target);
         }
+        sendWorkerMessage(ws, { type: "operation_completed", requestId: message.requestId });
         return;
       }
       if (message.type === "exec") {
@@ -6203,7 +6510,11 @@ async function startWorker(options) {
         } finally {
           if (targetReserved) workspaceSyncPriorityProcessTargets.delete(message.runId);
         }
-        ws.send(JSON.stringify(result));
+        if (infrastructureStoppedRunIds.has(message.runId)) {
+          recoveryStore.unknown(message.requestId);
+          return;
+        }
+        sendSerializedWorkerMessage(ws, JSON.stringify(result));
         if (hasWorkspaceEffect && targetMayMutateVisibleWorkspace(message.target)) {
           markWorkspaceDirty(
             { type: "shell_inline", sessionId: message.sessionId, processRunId: message.runId, detail: "foreground command completed" },
@@ -6296,7 +6607,7 @@ async function startWorker(options) {
         });
         return;
       }
-      if (message.type === "read" || message.type === "write" || message.type === "edit" || message.type === "grep" || message.type === "find" || message.type === "ls" || message.type === "view_file_bytes" || message.type === "code_list" || message.type === "code_read") {
+      if (message.type === "read_plan" || message.type === "lint_plans" || message.type === "update_task_status" || message.type === "read" || message.type === "write" || message.type === "edit" || message.type === "grep" || message.type === "find" || message.type === "ls" || message.type === "view_file_bytes" || message.type === "code_list" || message.type === "code_read") {
         if (workspaceOperationsAreFenced(deferredWorkspaceConfiguration, activeWorkspaceIncidentId)) {
           sendWorkerMessage(ws, {
             type: "operation_result",
@@ -6306,7 +6617,7 @@ async function startWorker(options) {
           return;
         }
         const reservesVisibleWorkspace = targetMayMutateVisibleWorkspace(message.target);
-        const mutatesVisibleWorkspace = (message.type === "write" || message.type === "edit") && targetMayMutateVisibleWorkspace(message.target);
+        const mutatesVisibleWorkspace = (message.type === "write" || message.type === "edit" || message.type === "update_task_status" || message.type === "read_plan" && message.createIfMissing) && targetMayMutateVisibleWorkspace(message.target);
         if (reservesVisibleWorkspace) {
           await reserveWorkspaceCommandAfterCurrentSync(message.target, workspaceSyncSingleFlight, () => {
             workspaceSyncPriorityOperationTargets.set(message.requestId, message.target);
@@ -6326,16 +6637,17 @@ async function startWorker(options) {
               assertAdmission: assertMessageAdmission
             });
           });
-          ws.send(
+          sendSerializedWorkerMessage(
+            ws,
             JSON.stringify({
               type: "operation_result",
               requestId: message.requestId,
               result
             })
           );
-          if (mutatesVisibleWorkspace) {
+          if (mutatesVisibleWorkspace && (result.type !== "read_plan" || result.created)) {
             dirtyTrigger = {
-              type: message.type,
+              type: message.type === "edit" || message.type === "update_task_status" ? "edit" : "write",
               sessionId: message.sessionId,
               toolCallId: message.requestId,
               ...message.target.type === "project" ? { projectId: message.target.projectId, branchName: message.target.branchName } : {}
@@ -6343,7 +6655,8 @@ async function startWorker(options) {
             markWorkspaceDirty(dirtyTrigger, false, message.target);
           }
         } catch (error) {
-          ws.send(
+          sendSerializedWorkerMessage(
+            ws,
             JSON.stringify({
               type: "operation_result",
               requestId: message.requestId,
@@ -6362,7 +6675,8 @@ async function startWorker(options) {
       }
       const unhandledMessage = message;
       if (typeof unhandledMessage.requestId === "string") {
-        ws.send(
+        sendSerializedWorkerMessage(
+          ws,
           JSON.stringify({
             type: "operation_result",
             requestId: unhandledMessage.requestId,
@@ -6376,6 +6690,7 @@ async function startWorker(options) {
     });
   });
   ws.addEventListener("close", (event) => {
+    clearTimeout(connectionAttemptTimer);
     const processEvents = process;
     processEvents.off("SIGTERM", handleGracefulShutdown);
     processEvents.off("SIGINT", handleGracefulShutdown);
@@ -6392,15 +6707,20 @@ async function startWorker(options) {
       clearTimeout(deferredWorkspaceConfigurationRefreshTimer);
       deferredWorkspaceConfigurationRefreshTimer = void 0;
     }
+    if (outputReplayTimer) {
+      clearInterval(outputReplayTimer);
+      outputReplayTimer = void 0;
+    }
     if (terminalReplayTimer) {
       clearInterval(terminalReplayTimer);
       terminalReplayTimer = void 0;
     }
-    if (currentWorkerSocket === ws) {
-      advanceWorkerAdmissionGeneration();
-      currentWorkerSocket = null;
+    if (projectRuntime.configuration) {
+      projectRuntime.configuration.incidentId = activeWorkspaceIncidentId;
+      projectRuntime.configuration.deferred = deferredWorkspaceConfiguration;
     }
-    closeAllPtys();
+    if (currentWorkerSocket === ws) currentWorkerSocket = null;
+    workerRecoveryId = "reconnecting";
     const reason = event.reason ? `: ${event.reason}` : "";
     process.stderr.write(`[r5d-worker] disconnected (${event.code}${reason})
 `);
@@ -6410,50 +6730,19 @@ async function startWorker(options) {
     if (shutdownAfterClose) {
       process.exit(0);
     }
-    if (!startupRecoveryComplete) {
-      process.stderr.write("[r5d-worker] control connection closed during workspace recovery; finishing recovery before reconnecting\n");
-      void startupRecoveryPromise.then(() => {
-        if (!startupRecoveryComplete) {
-          process.exit(1);
-          return;
-        }
-        const reconnectDelayMs = workerReconnectDelayMs();
-        process.stderr.write(`[r5d-worker] workspace recovery complete; reconnecting in ${reconnectDelayMs}ms
+    const reconnect = () => {
+      const delay = workerReconnectDelayMs(Math.random, workerReconnectAttempt++);
+      process.stderr.write(`[r5d-worker] reconnecting in ${delay}ms
 `);
-        setTimeout(() => {
-          void startWorker(options).catch((error) => {
-            process.stderr.write(`[r5d-worker] reconnect failed: ${error instanceof Error ? error.message : String(error)}
-`);
-            process.exit(error instanceof WorkerServerUnavailableError ? WORKER_RECONNECT_EXIT_CODE : 1);
-          });
-        }, reconnectDelayMs);
-      });
-      return;
-    }
-    if (activeProcesses.size > 0 || credentialBearingProcessGroups.size > 0 || activePtys.size > 0 || pendingProcessTerminals.size > 0) {
-      const scheduleReconnect = (reconnect2) => {
-        const reconnectDelayMs = workerReconnectDelayMs();
-        process.stderr.write(
-          `[r5d-worker] ${activeProcesses.size} process(es), ${credentialBearingProcessGroups.size} owned process group(s), and ${activePtys.size} PTY(s) active with ${pendingProcessTerminals.size} terminal report(s) pending; reconnecting in ${reconnectDelayMs}ms
-`
-        );
-        setTimeout(reconnect2, reconnectDelayMs);
-      };
-      const reconnect = () => {
-        void startWorker(options).catch((error) => {
+      setTimeout(() => {
+        void startupRecoveryPromise.then(() => startWorker(options, projectRuntime)).catch((error) => {
           process.stderr.write(`[r5d-worker] reconnect failed: ${error instanceof Error ? error.message : String(error)}
 `);
-          if (activeProcesses.size > 0 || credentialBearingProcessGroups.size > 0 || activePtys.size > 0 || pendingProcessTerminals.size > 0) {
-            scheduleReconnect(reconnect);
-            return;
-          }
-          process.exit(error instanceof WorkerServerUnavailableError ? WORKER_RECONNECT_EXIT_CODE : 1);
+          reconnect();
         });
-      };
-      scheduleReconnect(reconnect);
-      return;
-    }
-    process.exit(WORKER_RECONNECT_EXIT_CODE);
+      }, delay);
+    };
+    reconnect();
   });
   ws.addEventListener("error", (event) => {
     const message = "message" in event && typeof event.message === "string" && event.message.trim() ? `: ${event.message.trim()}` : "";