@ricsam/r5d-worker 0.0.89 → 0.0.91

package/dist/cjs/project-worktrees.cjs

@@ -318,7 +318,7 @@ function normalizeSnapshotManifest(value) {
   if (record.version !== PROJECT_WORKTREE_SNAPSHOT_MANIFEST_VERSION) {
     throw new Error("Unsupported project worktree snapshot manifest version");
   }
-  if (record.state !== "building" && record.state !== "prepared" && record.state !== "consumed") {
+  if (record.state !== "building" && record.state !== "prepared" && record.state !== "restoring" && record.state !== "consumed") {
     throw new Error("Project worktree snapshot manifest has an invalid state");
   }
   if (!Number.isSafeInteger(record.ownerProcessId) || record.ownerProcessId <= 0) {
@@ -399,7 +399,7 @@ function assertSameSnapshotIdentity(expected, actual) {
 function advanceSnapshotState(snapshotRoot, expected, state) {
   const current = readSnapshotManifest(snapshotRoot);
   assertSameSnapshotIdentity(expected, current);
-  const allowed = current.state === "building" && state === "prepared" || current.state === "prepared" && state === "consumed" || current.state === state;
+  const allowed = current.state === "building" && state === "prepared" || current.state === "prepared" && state === "restoring" || current.state === "prepared" && state === "consumed" || current.state === "restoring" && state === "consumed" || current.state === state;
   if (!allowed) throw new Error(`Project worktree snapshot cannot advance from ${current.state} to ${state}`);
   return current.state === state ? current : writeSnapshotManifest(snapshotRoot, { ...current, state });
 }
@@ -423,6 +423,20 @@ function copyCheckoutTree(sourceRoot, targetRoot) {
     verbatimSymlinks: true
   });
 }
+function measureCheckoutTree(root) {
+  const stat = import_node_fs.default.lstatSync(root);
+  if (stat.isSymbolicLink()) return 0;
+  if (stat.isFile()) return stat.size;
+  if (!stat.isDirectory()) return 0;
+  let bytes = 0;
+  for (const entry of import_node_fs.default.readdirSync(root).sort()) bytes += measureCheckoutTree(import_node_path.default.join(root, entry));
+  return bytes;
+}
+function snapshotBranchMeasurements(branches, sourcePath) {
+  return new Map(
+    branches.map(({ branchName, snapshotPresent }) => [branchName, snapshotPresent ? measureCheckoutTree(sourcePath(branchName)) : 0])
+  );
+}
 function assertNoOutstandingSnapshotForProject(temporaryRoot, projectRoot) {
   let entries;
   try {
@@ -445,7 +459,7 @@ function assertNoOutstandingSnapshotForProject(temporaryRoot, projectRoot) {
     }
   }
 }
-function snapshotBranchTrees(projectRoot, branches, temporaryRoot) {
+function snapshotBranchTrees(projectRoot, branches, temporaryRoot, onProgress) {
   const resolvedProjectRoot = import_node_path.default.resolve(projectRoot);
   const resolvedTemporaryRoot = import_node_path.default.resolve(temporaryRoot ?? import_node_os.default.tmpdir());
   assertNoOutstandingSnapshotForProject(resolvedTemporaryRoot, resolvedProjectRoot);
@@ -472,13 +486,57 @@ function snapshotBranchTrees(projectRoot, branches, temporaryRoot) {
       branches: branchSnapshots,
       createdAt: (/* @__PURE__ */ new Date()).toISOString()
     });
+    onProgress?.({
+      operation: "snapshot",
+      phase: "scanning",
+      projectRoot: resolvedProjectRoot,
+      completedBranches: 0,
+      totalBranches: branchSnapshots.length,
+      completedBytes: 0
+    });
+    const branchBytes = snapshotBranchMeasurements(branchSnapshots, (branchName) => branchPath(resolvedProjectRoot, branchName));
+    const totalBytes = [...branchBytes.values()].reduce((total, bytes) => total + bytes, 0);
+    let completedBytes = 0;
+    let completedBranches = 0;
     for (const { branchName, snapshotPresent } of branchSnapshots) {
-      if (!snapshotPresent) continue;
-      const snapshotPath = fullSnapshotPath(root, branchName);
-      copyCheckoutTree(branchPath(resolvedProjectRoot, branchName), snapshotPath);
-      paths.set(branchName, snapshotPath);
+      onProgress?.({
+        operation: "snapshot",
+        phase: "copying",
+        projectRoot: resolvedProjectRoot,
+        branchName,
+        completedBranches,
+        totalBranches: branchSnapshots.length,
+        completedBytes,
+        totalBytes
+      });
+      if (snapshotPresent) {
+        const snapshotPath = fullSnapshotPath(root, branchName);
+        copyCheckoutTree(branchPath(resolvedProjectRoot, branchName), snapshotPath);
+        paths.set(branchName, snapshotPath);
+      }
+      completedBytes += branchBytes.get(branchName) ?? 0;
+      completedBranches += 1;
+      onProgress?.({
+        operation: "snapshot",
+        phase: "copying",
+        projectRoot: resolvedProjectRoot,
+        branchName,
+        completedBranches,
+        totalBranches: branchSnapshots.length,
+        completedBytes,
+        totalBytes
+      });
     }
     const treesRoot = import_node_path.default.join(root, "trees");
+    onProgress?.({
+      operation: "snapshot",
+      phase: "syncing",
+      projectRoot: resolvedProjectRoot,
+      completedBranches,
+      totalBranches: branchSnapshots.length,
+      completedBytes,
+      totalBytes
+    });
     if (import_node_fs.default.existsSync(treesRoot)) fsyncTree(treesRoot);
     manifest = advanceSnapshotState(root, manifest, "prepared");
     return { root, manifest, paths };
@@ -487,33 +545,127 @@ function snapshotBranchTrees(projectRoot, branches, temporaryRoot) {
     throw error;
   }
 }
-function restoreProjectWorktreeSnapshot(snapshotRoot, manifest) {
-  const current = readSnapshotManifest(snapshotRoot);
+function restoreProjectWorktreeSnapshot(snapshotRoot, manifest, onProgress) {
+  let current = readSnapshotManifest(snapshotRoot);
   assertSameSnapshotIdentity(manifest, current);
-  if (current.state !== "prepared") throw new Error(`Project worktree snapshot is ${current.state}, not prepared`);
+  if (current.state !== "prepared" && current.state !== "restoring") {
+    throw new Error(`Project worktree snapshot is ${current.state}, not prepared or restoring`);
+  }
+  const treesRoot = import_node_path.default.join(snapshotRoot, "trees");
+  const treesRootPresent = import_node_fs.default.existsSync(treesRoot);
+  const treesRootStat = treesRootPresent ? import_node_fs.default.lstatSync(treesRoot) : null;
+  if (treesRootStat && (!treesRootStat.isDirectory() || treesRootStat.isSymbolicLink())) {
+    throw new Error("Project worktree snapshot trees root must be a real directory");
+  }
+  const branchSourceRoot = current.state === "restoring" && !treesRootPresent ? current.projectRoot : treesRoot;
   for (const { branchName, snapshotPresent } of current.branches) {
     const targetPath = branchPath(current.projectRoot, branchName);
     assertNoSymlinkBranchPathComponents(current.projectRoot, targetPath);
-    if (!snapshotPresent) continue;
-    const sourcePath = fullSnapshotPath(snapshotRoot, branchName);
+    const sourcePath = import_node_path.default.join(branchSourceRoot, ...branchName.split("/"));
+    if (!snapshotPresent) {
+      if (current.state === "restoring" && !treesRootPresent && import_node_fs.default.existsSync(sourcePath)) {
+        throw new Error(`Restored project branch unexpectedly exists: ${branchName}`);
+      }
+      continue;
+    }
     const stat = import_node_fs.default.lstatSync(sourcePath);
     if (!stat.isDirectory() || stat.isSymbolicLink()) {
       throw new Error(`Project worktree snapshot branch must be a real directory: ${branchName}`);
     }
   }
+  onProgress?.({
+    operation: "restore",
+    phase: "scanning",
+    projectRoot: current.projectRoot,
+    completedBranches: 0,
+    totalBranches: current.branches.length,
+    completedBytes: 0
+  });
+  const branchBytes = snapshotBranchMeasurements(current.branches, (branchName) => import_node_path.default.join(branchSourceRoot, ...branchName.split("/")));
+  const totalBytes = [...branchBytes.values()].reduce((total, bytes) => total + bytes, 0);
+  const presentBranches = current.branches.filter(({ snapshotPresent }) => snapshotPresent);
+  const destinationParent = import_node_path.default.dirname(current.projectRoot);
+  import_node_fs.default.mkdirSync(destinationParent, { recursive: true });
+  const canMoveWholeSnapshot = presentBranches.length > 0 && (current.state === "restoring" || treesRootStat !== null && treesRootStat.dev === import_node_fs.default.lstatSync(destinationParent).dev);
+  if (canMoveWholeSnapshot) {
+    if (current.state === "prepared") current = advanceSnapshotState(snapshotRoot, current, "restoring");
+    onProgress?.({
+      operation: "restore",
+      phase: "moving",
+      projectRoot: current.projectRoot,
+      completedBranches: treesRootPresent ? 0 : current.branches.length,
+      totalBranches: current.branches.length,
+      completedBytes: treesRootPresent ? 0 : totalBytes,
+      totalBytes
+    });
+    if (treesRootPresent) {
+      import_node_fs.default.rmSync(current.projectRoot, { recursive: true, force: true });
+      fsyncDirectoryChain(destinationParent, import_node_path.default.dirname(destinationParent));
+      import_node_fs.default.renameSync(treesRoot, current.projectRoot);
+      fsyncDirectory(snapshotRoot);
+    }
+    fsyncManagedProjectRootEntry(current.projectRoot);
+    onProgress?.({
+      operation: "restore",
+      phase: "moving",
+      projectRoot: current.projectRoot,
+      completedBranches: current.branches.length,
+      totalBranches: current.branches.length,
+      completedBytes: totalBytes,
+      totalBytes
+    });
+    return;
+  }
+  if (current.state === "restoring") {
+    throw new Error("Project worktree snapshot entered atomic restore state without a movable snapshot tree");
+  }
   for (const { branchName } of [...current.branches].sort((left, right) => right.branchName.length - left.branchName.length)) {
     const targetPath = branchPath(current.projectRoot, branchName);
     const targetParent = import_node_path.default.dirname(targetPath);
     import_node_fs.default.rmSync(targetPath, { recursive: true, force: true });
     if (import_node_fs.default.existsSync(targetParent)) fsyncDirectoryChain(targetParent, current.projectRoot);
   }
+  let completedBytes = 0;
+  let completedBranches = 0;
   for (const { branchName, snapshotPresent } of current.branches) {
-    if (!snapshotPresent) continue;
-    const targetPath = branchPath(current.projectRoot, branchName);
-    copyCheckoutTree(fullSnapshotPath(snapshotRoot, branchName), targetPath);
-    fsyncTree(targetPath);
-    fsyncDirectoryChain(import_node_path.default.dirname(targetPath), current.projectRoot);
+    onProgress?.({
+      operation: "restore",
+      phase: "copying",
+      projectRoot: current.projectRoot,
+      branchName,
+      completedBranches,
+      totalBranches: current.branches.length,
+      completedBytes,
+      totalBytes
+    });
+    if (snapshotPresent) {
+      const targetPath = branchPath(current.projectRoot, branchName);
+      copyCheckoutTree(fullSnapshotPath(snapshotRoot, branchName), targetPath);
+      fsyncTree(targetPath);
+      fsyncDirectoryChain(import_node_path.default.dirname(targetPath), current.projectRoot);
+    }
+    completedBytes += branchBytes.get(branchName) ?? 0;
+    completedBranches += 1;
+    onProgress?.({
+      operation: "restore",
+      phase: "copying",
+      projectRoot: current.projectRoot,
+      branchName,
+      completedBranches,
+      totalBranches: current.branches.length,
+      completedBytes,
+      totalBytes
+    });
   }
+  onProgress?.({
+    operation: "restore",
+    phase: "syncing",
+    projectRoot: current.projectRoot,
+    completedBranches,
+    totalBranches: current.branches.length,
+    completedBytes,
+    totalBytes
+  });
   import_node_fs.default.mkdirSync(current.projectRoot, { recursive: true });
   fsyncManagedProjectRootEntry(current.projectRoot);
 }
@@ -534,7 +686,7 @@ function persistReconciledProjectAndConsumeSnapshot(snapshot, afterProjectTreeFs
 function rollbackProjectWorktreeSnapshot(snapshot) {
   const current = readSnapshotManifest(snapshot.root);
   assertSameSnapshotIdentity(snapshot.manifest, current);
-  if (current.state === "prepared") {
+  if (current.state === "prepared" || current.state === "restoring") {
     restoreProjectWorktreeSnapshot(snapshot.root, current);
     consumeAndRemoveProjectWorktreeSnapshot(snapshot.root, current);
     return;
@@ -616,8 +768,8 @@ function recoverStaleProjectWorktreeSnapshots(input) {
       assertNoSymlinkPathComponents(projectsRoot, manifest.projectRoot);
       const ownedByThisProcessSession = manifest.ownerProcessId === currentProcessId && manifest.ownerSessionId === currentOwnerSessionId;
       if (ownedByThisProcessSession || ownerProcessId !== currentProcessId && processAlive(ownerProcessId)) continue;
-      if (manifest.state === "prepared") {
-        restoreProjectWorktreeSnapshot(candidate, manifest);
+      if (manifest.state === "prepared" || manifest.state === "restoring") {
+        restoreProjectWorktreeSnapshot(candidate, manifest, input.onProgress);
         const consumed = advanceSnapshotState(candidate, manifest, "consumed");
         removeSnapshotStorage(candidate);
         assertSameSnapshotIdentity(manifest, consumed);
@@ -803,7 +955,11 @@ async function initializeLinkedLayout(input) {
     );
     const checkoutPath = branchPath(input.projectRoot, branch.branchName);
     import_node_fs.default.mkdirSync(import_node_path.default.dirname(checkoutPath), { recursive: true });
-    await gitAsync(primaryPath, ["worktree", "add", "--force", checkoutPath, branch.branchName], `create linked worktree ${branch.branchName}`);
+    await gitAsync(
+      primaryPath,
+      ["worktree", "add", "--force", checkoutPath, branch.branchName],
+      `create linked worktree ${branch.branchName}`
+    );
   }
   return mirrorHeads;
 }
@@ -837,7 +993,11 @@ async function reconcileLinkedLayout(input) {
     import_node_fs.default.rmSync(checkoutPath, { recursive: true, force: true });
     await gitAsync(primaryPath, ["branch", "-f", branch.branchName, resolved.seed], `anchor project branch ${branch.branchName}`);
     import_node_fs.default.mkdirSync(import_node_path.default.dirname(checkoutPath), { recursive: true });
-    await gitAsync(primaryPath, ["worktree", "add", "--force", checkoutPath, branch.branchName], `create linked worktree ${branch.branchName}`);
+    await gitAsync(
+      primaryPath,
+      ["worktree", "add", "--force", checkoutPath, branch.branchName],
+      `create linked worktree ${branch.branchName}`
+    );
   }
   return { mirrorHeads, mutatedBranches };
 }
@@ -864,7 +1024,7 @@ async function ensureProjectWorktrees(input) {
   assertNoOutstandingSnapshotForProject(snapshotStorageRoot, projectRoot);
   let snapshot;
   const ensureSnapshot = () => {
-    snapshot ??= snapshotBranchTrees(projectRoot, input.branches, snapshotStorageRoot);
+    snapshot ??= snapshotBranchTrees(projectRoot, input.branches, snapshotStorageRoot, input.onSnapshotProgress);
     return snapshot;
   };
   try {

package/dist/mjs/main.mjs

@@ -7,11 +7,6 @@ import { spawn as spawnChildProcess } from "node:child_process";
 import { Database } from "bun:sqlite";
 import { installCliUpdate, readInstalledCliVersion } from "./cli-update.mjs";
 import { gitTransportSecurityArgs, workerGitProcessEnvironment } from "./git-process-environment.mjs";
-import {
-  grantWorkerHeartbeatBusyGrace,
-  hasWorkerHeartbeatTimedOutWithBusyGrace,
-  WORKER_HEARTBEAT_INTERVAL_MS
-} from "./heartbeat.mjs";
 import { terminateProcessTree } from "./process-tree.mjs";
 import { createPtyOutputCoalescer } from "./pty-output-coalescer.mjs";
 import { openWorkerPortForwardRelay } from "./port-forward-client.mjs";
@@ -63,7 +58,6 @@ import { assertDisjointManagedRoots, managedProjectRoot, validateManagedBranchNa
 import {
   applyObservedProjectMirrorHeads,
   assertProjectMirrorHeadsPushed,
-  recoverStaleProjectWorktreeSnapshots,
   createOrRetryLinkedProjectBranch,
   deleteLinkedProjectBranch,
   deleteProjectMirrorBranch,
@@ -76,6 +70,11 @@ import {
   pushProjectMirrorHeads,
   removeProjectWorktrees
 } from "./project-worktrees.mjs";
+import {
+  PROJECT_SNAPSHOT_RECOVERY_HELPER_COMMAND,
+  recoverProjectSnapshotsInChild,
+  runProjectSnapshotRecoveryHelper
+} from "./project-snapshot-recovery-runner.mjs";
 import {
   ProjectWorkspacePendingBranchPathChangeError,
   ProjectWorkspaceStateStore,
@@ -3416,28 +3415,6 @@ async function startWorker(options) {
   runGit(["--version"]);
   await verifyR5dctlAuth(baseUrl, token, label);
   const initializedWorkspaceState = await workspaceSyncSingleFlight.runExclusive(() => {
-    if (!startupProjectSnapshotRecoveryCompleted) {
-      const snapshotRecovery = recoverStaleProjectWorktreeSnapshots({
-        projectsRoot,
-        temporaryRoot: projectWorktreeSnapshotsRoot
-      });
-      if (snapshotRecovery.restored.length > 0) {
-        process.stdout.write(`[r5d-worker] restored ${snapshotRecovery.restored.length} interrupted project snapshot(s)
-`);
-      }
-      if (snapshotRecovery.removed.length > 0) {
-        process.stdout.write(`[r5d-worker] removed ${snapshotRecovery.removed.length} completed/incomplete project snapshot(s)
-`);
-      }
-      for (const failure of snapshotRecovery.failed) {
-        process.stderr.write(`[r5d-worker] failed to recover stale project snapshot ${failure.path}: ${failure.error}
-`);
-      }
-      if (snapshotRecovery.failed.length > 0) {
-        throw new Error("Project snapshot recovery is incomplete; refusing to mutate project checkouts");
-      }
-      startupProjectSnapshotRecoveryCompleted = true;
-    }
     fs.mkdirSync(projectsRoot, { recursive: true });
     fs.mkdirSync(syncRoot, { recursive: true });
     fs.mkdirSync(artifactRoot, { recursive: true });
@@ -3476,6 +3453,8 @@ async function startWorker(options) {
   let cliUpdateInProgress = false;
   let reloadAfterClose = false;
   let shutdownAfterClose = false;
+  let startupRecoveryComplete = startupProjectSnapshotRecoveryCompleted;
+  let startupRecoveryPromise = Promise.resolve();
   const automaticWorkspaceSyncIsCurrentlyAdmitted = () => workspaceAutomaticSyncIsAdmitted({
     workspaceConfigured,
     activeIncidentId: activeWorkspaceIncidentId,
@@ -3892,8 +3871,14 @@ async function startWorker(options) {
       })
     )
   ].sort();
-  const ensureConfiguredProjects = async () => {
-    for (const project of projectConfigById.values()) {
+  const ensureConfiguredProjects = async (onProgress) => {
+    const projects = [...projectConfigById.values()];
+    for (const [projectIndex, project] of projects.entries()) {
+      onProgress?.({
+        detail: `Configuring ${project.projectPath}`,
+        completedItems: projectIndex,
+        totalItems: projects.length
+      });
       validateProjectId(project.projectId);
       assertRepositoryTransitionState(project);
       for (const branch of project.branches) validateBranchName(branch.branchName);
@@ -3941,7 +3926,14 @@ async function startWorker(options) {
           mirrorUrl: connection.mirrorUrl,
           mirrorCredentialUsername: connection.mirrorCredentialUsername,
           credentialHelper: connection.credentialHelper,
-          gitIdentity: workspaceGitIdentity
+          gitIdentity: workspaceGitIdentity,
+          onSnapshotProgress: (progress) => onProgress?.({
+            detail: progress.phase === "scanning" ? `Measuring ${project.projectPath} workspace snapshot` : progress.phase === "syncing" ? `Making ${project.projectPath} workspace snapshot durable` : `${progress.operation === "restore" ? "Restoring" : "Snapshotting"} ${project.projectPath}${progress.branchName ? `/${progress.branchName}` : ""}`,
+            completedBytes: progress.completedBytes,
+            completedItems: progress.completedBranches,
+            totalItems: progress.totalBranches,
+            ...progress.totalBytes === void 0 ? {} : { totalBytes: progress.totalBytes }
+          })
         });
         readyProjectIds.add(project.projectId);
         reconciledProjectConfigFingerprints.set(project.projectId, configFingerprint);
@@ -3976,6 +3968,11 @@ async function startWorker(options) {
 `
         );
       }
+      onProgress?.({
+        detail: `Configured ${project.projectPath}`,
+        completedItems: projectIndex + 1,
+        totalItems: projects.length
+      });
     }
   };
   const collectAheadOfOriginBranches = async () => {
@@ -4368,6 +4365,11 @@ async function startWorker(options) {
       if (input.sendResult !== false) sendWorkspaceSyncResult(input.requestId, result2);
       return result2;
     }
+    sendLifecycleProgress({
+      phase: "syncing",
+      operationId: attemptId,
+      detail: input.trigger.detail ?? `Synchronizing workspace (${input.trigger.type})`
+    });
     if (!input.requestId && input.sendResult !== false) {
       if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
         throw new Error("Cannot start autonomous workspace synchronization without an open worker control socket");
@@ -4470,6 +4472,9 @@ async function startWorker(options) {
       }
     };
     if (input.sendResult !== false) sendWorkspaceSyncResult(input.requestId, result);
+    sendLifecycleProgress(
+      result.outcome === "failed" ? { phase: "failed", operationId: attemptId, detail: result.error ?? "Workspace synchronization failed" } : { phase: "ready", operationId: attemptId, detail: "Workspace synchronized" }
+    );
     return result;
   };
   const scheduleAutomaticWorkspaceSync = (trigger, delayMs = WORKSPACE_GIT_QUIET_MS) => {
@@ -4509,7 +4514,6 @@ async function startWorker(options) {
           );
         }).finally(() => {
           automaticSyncInFlight = false;
-          heartbeatBusyGrace = grantWorkerHeartbeatBusyGrace(lastServerHeartbeatAt, heartbeatBusyGrace);
           if (pendingAutomaticTrigger && automaticWorkspaceSyncIsCurrentlyAdmitted()) {
             scheduleAutomaticWorkspaceSync(pendingAutomaticTrigger, WORKSPACE_GIT_QUIET_MS);
           }
@@ -4545,6 +4549,14 @@ async function startWorker(options) {
     });
   };
   const configureWorkerWorkspace = async (message, receiptGeneration) => {
+    const configurationOperationId = crypto.randomUUID();
+    sendLifecycleProgress({
+      phase: "configuring",
+      operationId: configurationOperationId,
+      detail: `Preparing ${message.projects.length} project(s)`,
+      completedItems: 0,
+      totalItems: message.projects.length
+    });
     const incidentDeferral = workspaceConfigurationIncidentDeferral({
       requestedIncidentId: message.deferWorkspaceSyncForIncidentId,
       activeIncidentId: activeWorkspaceIncidentId
@@ -4909,7 +4921,13 @@ async function startWorker(options) {
           // checkout initialization from erasing synchronized dirt.
           { ignoreBusy: true }
         );
-        await ensureConfiguredProjects();
+        await ensureConfiguredProjects((progress) => {
+          sendLifecycleProgress({ phase: "configuring", operationId: configurationOperationId, ...progress });
+          process.stdout.write(
+            `[r5d-worker] ${progress.detail ?? "Configuring workspace"}${progress.completedBytes === void 0 ? "" : `: ${progress.completedBytes}/${progress.totalBytes ?? "?"} bytes`}${progress.completedItems === void 0 ? "" : `, ${progress.completedItems}/${progress.totalItems ?? "?"} item(s)`}
+`
+          );
+        });
         const configurationSyncAdmissionGeneration = workerAdmissionGeneration;
         const assertConfigurationSyncStillAdmitted = () => {
           if (receiptGeneration !== workspaceConfigurationReceiptGeneration || configurationSyncAdmissionGeneration !== workerAdmissionGeneration || !workspaceConfigurationIncidentSnapshotIsCurrent(incidentDeferral.incidentId, activeWorkspaceIncidentId) || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
@@ -5008,10 +5026,60 @@ async function startWorker(options) {
       Authorization: `Bearer ${token}`
     }
   });
-  let lastServerHeartbeatAt = null;
-  let heartbeatBusyGrace = null;
-  let heartbeatTimedOut = false;
-  let heartbeatWatchdog;
+  const sendLifecycleProgress = (progress) => {
+    if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) return;
+    sendWorkerMessage(ws, { type: "worker_progress", progress });
+  };
+  const describeSnapshotProgress = (progress) => {
+    const project = path.relative(projectsRoot, progress.projectRoot) || path.basename(progress.projectRoot);
+    const branch = progress.branchName ? `/${progress.branchName}` : "";
+    if (progress.phase === "scanning") return `Scanning ${project} recovery snapshot`;
+    if (progress.phase === "syncing") return `Making restored ${project} files durable`;
+    if (progress.phase === "cleaning") return `Cleaning recovered ${project} snapshot`;
+    return `Restoring ${project}${branch}`;
+  };
+  const performStartupRecovery = async (operationId) => {
+    if (startupProjectSnapshotRecoveryCompleted) return;
+    sendLifecycleProgress({ phase: "recovering", operationId, detail: "Checking interrupted workspace transactions" });
+    const snapshotRecovery = await workspaceSyncSingleFlight.runExclusive(
+      async () => await recoverProjectSnapshotsInChild({
+        projectsRoot,
+        snapshotRoot: projectWorktreeSnapshotsRoot,
+        onProgress: (progress) => {
+          const lifecycleProgress = {
+            phase: "recovering",
+            operationId,
+            detail: describeSnapshotProgress(progress),
+            completedBytes: progress.completedBytes,
+            completedItems: progress.completedBranches,
+            totalItems: progress.totalBranches,
+            ...progress.totalBytes === void 0 ? {} : { totalBytes: progress.totalBytes }
+          };
+          sendLifecycleProgress(lifecycleProgress);
+          process.stdout.write(
+            `[r5d-worker] ${lifecycleProgress.detail}: ${progress.completedBranches}/${progress.totalBranches} branch(es), ${progress.completedBytes}/${progress.totalBytes ?? "?"} bytes
+`
+          );
+        }
+      })
+    );
+    if (snapshotRecovery.restored.length > 0) {
+      process.stdout.write(`[r5d-worker] restored ${snapshotRecovery.restored.length} interrupted project snapshot(s)
+`);
+    }
+    if (snapshotRecovery.removed.length > 0) {
+      process.stdout.write(`[r5d-worker] removed ${snapshotRecovery.removed.length} completed/incomplete project snapshot(s)
+`);
+    }
+    for (const failure of snapshotRecovery.failed) {
+      process.stderr.write(`[r5d-worker] failed to recover stale project snapshot ${failure.path}: ${failure.error}
+`);
+    }
+    if (snapshotRecovery.failed.length > 0) {
+      throw new Error("Project snapshot recovery is incomplete; refusing to mutate project checkouts");
+    }
+    startupProjectSnapshotRecoveryCompleted = true;
+  };
   const handleGracefulShutdown = () => {
     if (shutdownAfterClose) return;
     shutdownAfterClose = true;
@@ -5039,25 +5107,29 @@ async function startWorker(options) {
   };
   process.on("SIGTERM", handleGracefulShutdown);
   process.on("SIGINT", handleGracefulShutdown);
-  const stopHeartbeatWatchdog = () => {
-    if (heartbeatWatchdog) {
-      clearInterval(heartbeatWatchdog);
-      heartbeatWatchdog = void 0;
-    }
-  };
   ws.addEventListener("open", () => {
     currentWorkerSocket = ws;
-    heartbeatWatchdog = setInterval(() => {
-      if (heartbeatTimedOut || !hasWorkerHeartbeatTimedOutWithBusyGrace(lastServerHeartbeatAt, heartbeatBusyGrace)) {
-        return;
+    const startupOperationId = crypto.randomUUID();
+    startupRecoveryPromise = Promise.resolve().then(async () => await performStartupRecovery(startupOperationId)).then(() => {
+      startupRecoveryComplete = true;
+      if (currentWorkerSocket === ws && ws.readyState === WebSocket.OPEN) {
+        sendWorkerMessage(ws, { type: "startup_ready" });
+        sendLifecycleProgress({
+          phase: "configuring",
+          operationId: crypto.randomUUID(),
+          detail: "Waiting for workspace configuration"
+        });
       }
-      heartbeatTimedOut = true;
-      stopHeartbeatWatchdog();
-      process.stderr.write("[r5d-worker] server heartbeat timed out; reconnecting\n");
-      ws.terminate();
-    }, WORKER_HEARTBEAT_INTERVAL_MS);
+    }).catch((error) => {
+      const message = error instanceof Error ? error.message : String(error);
+      sendLifecycleProgress({ phase: "failed", operationId: startupOperationId, detail: message });
+      process.stderr.write(`[r5d-worker] startup recovery failed: ${message}
+`);
+      ws.close(1011, "Project snapshot recovery failed");
+    });
     const hello = {
       type: "hello",
+      startupReady: startupRecoveryComplete,
       hostInfo: {
         hostname: hostname(),
         platform: process.platform,
@@ -5095,6 +5167,8 @@ async function startWorker(options) {
   ws.addEventListener("message", (event) => {
     void (async () => {
       const message = JSON.parse(String(event.data));
+      if (!startupRecoveryComplete && message.type !== "ping") await startupRecoveryPromise;
+      if (!startupRecoveryComplete && message.type !== "ping") return;
       const messageAdmissionGeneration = workerAdmissionGeneration;
       const assertMessageAdmission = () => assertWorkerChildAdmission({
         capturedGeneration: messageAdmissionGeneration,
@@ -5102,7 +5176,7 @@ async function startWorker(options) {
         sourceSocketIsCurrent: currentWorkerSocket === ws,
         sourceSocketOpen: ws.readyState === WebSocket.OPEN,
         workspaceConfigured,
-        runtimeHealthy: !shutdownAfterClose && !heartbeatTimedOut
+        runtimeHealthy: !shutdownAfterClose
       });
       if (message.type === "connected") {
         return;
@@ -5122,6 +5196,19 @@ async function startWorker(options) {
           aheadOfOriginBranches: configured.aheadOfOriginBranches,
           ...configured.deferredWorkspaceSyncForIncidentId ? { deferredWorkspaceSyncForIncidentId: configured.deferredWorkspaceSyncForIncidentId } : {}
         });
+        sendLifecycleProgress(
+          configured.result.outcome === "failed" ? {
+            phase: "failed",
+            operationId: configured.result.attemptId,
+            detail: configured.result.error ?? "Workspace configuration failed"
+          } : {
+            phase: "ready",
+            operationId: configured.result.attemptId,
+            detail: `${message.projects.length} project(s) configured`,
+            completedItems: message.projects.length,
+            totalItems: message.projects.length
+          }
+        );
         process.stdout.write(
           `[r5d-worker] workspace configured: ${message.projects.length} project(s), ${configured.pending.length} pending checkout(s)
 `
@@ -5461,8 +5548,6 @@ async function startWorker(options) {
         return;
       }
       if (message.type === "ping") {
-        lastServerHeartbeatAt = Date.now();
-        heartbeatBusyGrace = null;
         ws.send(JSON.stringify({ type: "pong" }));
         return;
       }
@@ -5876,15 +5961,12 @@ async function startWorker(options) {
     })().catch((error) => {
       process.stderr.write(`[r5d-worker] message handling failed: ${error instanceof Error ? error.message : String(error)}
 `);
-    }).finally(() => {
-      heartbeatBusyGrace = grantWorkerHeartbeatBusyGrace(lastServerHeartbeatAt, heartbeatBusyGrace);
     });
   });
   ws.addEventListener("close", (event) => {
     const processEvents = process;
     processEvents.off("SIGTERM", handleGracefulShutdown);
     processEvents.off("SIGINT", handleGracefulShutdown);
-    stopHeartbeatWatchdog();
     if (workspaceAutomaticTimer) {
       clearTimeout(workspaceAutomaticTimer);
       workspaceAutomaticTimer = void 0;
@@ -5915,6 +5997,21 @@ async function startWorker(options) {
     if (shutdownAfterClose) {
       process.exit(0);
     }
+    if (!startupRecoveryComplete) {
+      process.stderr.write("[r5d-worker] control connection closed during workspace recovery; finishing recovery before reconnecting\n");
+      void startupRecoveryPromise.then(() => {
+        if (!startupRecoveryComplete) {
+          process.exit(1);
+          return;
+        }
+        void startWorker(options).catch((error) => {
+          process.stderr.write(`[r5d-worker] reconnect failed: ${error instanceof Error ? error.message : String(error)}
+`);
+          process.exit(error instanceof WorkerServerUnavailableError ? WORKER_RECONNECT_EXIT_CODE : 1);
+        });
+      });
+      return;
+    }
     if (activeProcesses.size > 0 || credentialBearingProcessGroups.size > 0 || activePtys.size > 0 || pendingProcessTerminals.size > 0) {
       process.stderr.write(
         `[r5d-worker] ${activeProcesses.size} process(es), ${credentialBearingProcessGroups.size} owned process group(s), and ${activePtys.size} PTY(s) active with ${pendingProcessTerminals.size} terminal report(s) pending; reconnecting in ${WORKER_RECONNECT_DELAY_MS}ms
@@ -5945,6 +6042,11 @@ async function startWorker(options) {
   });
 }
 async function main() {
+  if (process.argv[2] === PROJECT_SNAPSHOT_RECOVERY_HELPER_COMMAND) {
+    const exitCode2 = runProjectSnapshotRecoveryHelper(process.argv.slice(3));
+    if (exitCode2 !== 0) process.exit(exitCode2);
+    return;
+  }
   const parsed = parseArgs(process.argv.slice(2));
   if (parsed.command === "help") {
     printHelp();