@ricsam/r5d-worker 0.0.90 → 0.0.92

package/dist/mjs/project-snapshot-recovery-runner.mjs

@@ -0,0 +1,135 @@
+import path from "node:path";
+import { spawn } from "node:child_process";
+import { recoverStaleProjectWorktreeSnapshots } from "./project-worktrees.mjs";
+const PROJECT_SNAPSHOT_RECOVERY_HELPER_COMMAND = "__recover-project-snapshots";
+function requireAbsolutePath(value, label) {
+  if (!value) throw new Error(`Missing ${label}`);
+  const resolved = path.resolve(value);
+  if (resolved !== value) throw new Error(`${label} must be an absolute path`);
+  return resolved;
+}
+function parseHelperArguments(args) {
+  let projectsRoot;
+  let snapshotRoot;
+  for (let index = 0; index < args.length; index += 1) {
+    const argument = args[index];
+    if (argument === "--projects-root") {
+      projectsRoot = args[index + 1];
+      index += 1;
+      continue;
+    }
+    if (argument === "--snapshot-root") {
+      snapshotRoot = args[index + 1];
+      index += 1;
+      continue;
+    }
+    throw new Error(`Unknown project snapshot recovery helper argument: ${argument ?? "(missing)"}`);
+  }
+  return {
+    projectsRoot: requireAbsolutePath(projectsRoot, "projects root"),
+    snapshotRoot: requireAbsolutePath(snapshotRoot, "snapshot root")
+  };
+}
+function writeHelperMessage(message) {
+  process.stdout.write(`${JSON.stringify(message)}
+`);
+}
+function runProjectSnapshotRecoveryHelper(args) {
+  const { projectsRoot, snapshotRoot } = parseHelperArguments(args);
+  const result = recoverStaleProjectWorktreeSnapshots({
+    projectsRoot,
+    temporaryRoot: snapshotRoot,
+    onProgress: (progress) => writeHelperMessage({ type: "progress", progress })
+  });
+  writeHelperMessage({ type: "result", result });
+  return result.failed.length === 0 ? 0 : 1;
+}
+function isRecoveryProgress(value) {
+  if (!value || typeof value !== "object") return false;
+  const progress = value;
+  return progress.operation === "restore" && (progress.phase === "scanning" || progress.phase === "copying" || progress.phase === "moving" || progress.phase === "syncing" || progress.phase === "cleaning") && typeof progress.projectRoot === "string" && Number.isSafeInteger(progress.completedBranches) && Number.isSafeInteger(progress.totalBranches) && typeof progress.completedBytes === "number" && Number.isFinite(progress.completedBytes) && (progress.totalBytes === void 0 || typeof progress.totalBytes === "number" && Number.isFinite(progress.totalBytes));
+}
+function isRecoveryResult(value) {
+  if (!value || typeof value !== "object") return false;
+  const result = value;
+  return Array.isArray(result.restored) && Array.isArray(result.removed) && Array.isArray(result.failed);
+}
+function recoveryChildEnvironment() {
+  const environment = {};
+  for (const name of ["PATH", "TMPDIR", "TMP", "TEMP", "SystemRoot", "WINDIR"]) {
+    const value = process.env[name];
+    if (value !== void 0) environment[name] = value;
+  }
+  return environment;
+}
+async function recoverProjectSnapshotsInChild(input) {
+  const entrypoint = process.argv[1];
+  if (!entrypoint) throw new Error("Cannot locate the r5d-worker runtime entrypoint for snapshot recovery");
+  const child = spawn(
+    process.execPath,
+    [
+      entrypoint,
+      PROJECT_SNAPSHOT_RECOVERY_HELPER_COMMAND,
+      "--projects-root",
+      path.resolve(input.projectsRoot),
+      "--snapshot-root",
+      path.resolve(input.snapshotRoot)
+    ],
+    {
+      env: recoveryChildEnvironment(),
+      stdio: ["ignore", "pipe", "pipe"]
+    }
+  );
+  const terminateOnParentExit = () => {
+    if (child.exitCode === null && child.signalCode === null) child.kill("SIGTERM");
+  };
+  const parentProcessEvents = process;
+  parentProcessEvents.once("exit", terminateOnParentExit);
+  let stdout = "";
+  let stderr = "";
+  let result;
+  child.stdout.setEncoding("utf8");
+  child.stdout.on("data", (chunk) => {
+    stdout += chunk;
+    while (true) {
+      const newline = stdout.indexOf("\n");
+      if (newline < 0) break;
+      const line = stdout.slice(0, newline);
+      stdout = stdout.slice(newline + 1);
+      if (!line.trim()) continue;
+      let message;
+      try {
+        message = JSON.parse(line);
+      } catch {
+        stderr += `Invalid recovery helper output: ${line}
+`;
+        continue;
+      }
+      if (!message || typeof message !== "object") continue;
+      const record = message;
+      if (record.type === "progress" && isRecoveryProgress(record.progress)) input.onProgress?.(record.progress);
+      if (record.type === "result" && isRecoveryResult(record.result)) result = record.result;
+    }
+  });
+  child.stderr.setEncoding("utf8");
+  child.stderr.on("data", (chunk) => {
+    stderr = `${stderr}${chunk}`.slice(-32768);
+  });
+  const exitCode = await new Promise((resolve, reject) => {
+    child.once("error", reject);
+    child.once("close", (code, signal) => {
+      if (signal) reject(new Error(`Project snapshot recovery helper exited on signal ${signal}`));
+      else resolve(code ?? 1);
+    });
+  }).finally(() => parentProcessEvents.off("exit", terminateOnParentExit));
+  if (exitCode !== 0 || !result) {
+    const detail = stderr.trim();
+    throw new Error(detail || `Project snapshot recovery helper exited with status ${exitCode}`);
+  }
+  return result;
+}
+export {
+  PROJECT_SNAPSHOT_RECOVERY_HELPER_COMMAND,
+  recoverProjectSnapshotsInChild,
+  runProjectSnapshotRecoveryHelper
+};

package/dist/mjs/project-worktrees.mjs

@@ -265,7 +265,7 @@ function normalizeSnapshotManifest(value) {
   if (record.version !== PROJECT_WORKTREE_SNAPSHOT_MANIFEST_VERSION) {
     throw new Error("Unsupported project worktree snapshot manifest version");
   }
-  if (record.state !== "building" && record.state !== "prepared" && record.state !== "consumed") {
+  if (record.state !== "building" && record.state !== "prepared" && record.state !== "restoring" && record.state !== "consumed") {
     throw new Error("Project worktree snapshot manifest has an invalid state");
   }
   if (!Number.isSafeInteger(record.ownerProcessId) || record.ownerProcessId <= 0) {
@@ -346,7 +346,7 @@ function assertSameSnapshotIdentity(expected, actual) {
 function advanceSnapshotState(snapshotRoot, expected, state) {
   const current = readSnapshotManifest(snapshotRoot);
   assertSameSnapshotIdentity(expected, current);
-  const allowed = current.state === "building" && state === "prepared" || current.state === "prepared" && state === "consumed" || current.state === state;
+  const allowed = current.state === "building" && state === "prepared" || current.state === "prepared" && state === "restoring" || current.state === "prepared" && state === "consumed" || current.state === "restoring" && state === "consumed" || current.state === state;
   if (!allowed) throw new Error(`Project worktree snapshot cannot advance from ${current.state} to ${state}`);
   return current.state === state ? current : writeSnapshotManifest(snapshotRoot, { ...current, state });
 }
@@ -370,6 +370,20 @@ function copyCheckoutTree(sourceRoot, targetRoot) {
     verbatimSymlinks: true
   });
 }
+function measureCheckoutTree(root) {
+  const stat = fs.lstatSync(root);
+  if (stat.isSymbolicLink()) return 0;
+  if (stat.isFile()) return stat.size;
+  if (!stat.isDirectory()) return 0;
+  let bytes = 0;
+  for (const entry of fs.readdirSync(root).sort()) bytes += measureCheckoutTree(path.join(root, entry));
+  return bytes;
+}
+function snapshotBranchMeasurements(branches, sourcePath) {
+  return new Map(
+    branches.map(({ branchName, snapshotPresent }) => [branchName, snapshotPresent ? measureCheckoutTree(sourcePath(branchName)) : 0])
+  );
+}
 function assertNoOutstandingSnapshotForProject(temporaryRoot, projectRoot) {
   let entries;
   try {
@@ -392,7 +406,7 @@ function assertNoOutstandingSnapshotForProject(temporaryRoot, projectRoot) {
     }
   }
 }
-function snapshotBranchTrees(projectRoot, branches, temporaryRoot) {
+function snapshotBranchTrees(projectRoot, branches, temporaryRoot, onProgress) {
   const resolvedProjectRoot = path.resolve(projectRoot);
   const resolvedTemporaryRoot = path.resolve(temporaryRoot ?? os.tmpdir());
   assertNoOutstandingSnapshotForProject(resolvedTemporaryRoot, resolvedProjectRoot);
@@ -419,13 +433,57 @@ function snapshotBranchTrees(projectRoot, branches, temporaryRoot) {
       branches: branchSnapshots,
       createdAt: (/* @__PURE__ */ new Date()).toISOString()
     });
+    onProgress?.({
+      operation: "snapshot",
+      phase: "scanning",
+      projectRoot: resolvedProjectRoot,
+      completedBranches: 0,
+      totalBranches: branchSnapshots.length,
+      completedBytes: 0
+    });
+    const branchBytes = snapshotBranchMeasurements(branchSnapshots, (branchName) => branchPath(resolvedProjectRoot, branchName));
+    const totalBytes = [...branchBytes.values()].reduce((total, bytes) => total + bytes, 0);
+    let completedBytes = 0;
+    let completedBranches = 0;
     for (const { branchName, snapshotPresent } of branchSnapshots) {
-      if (!snapshotPresent) continue;
-      const snapshotPath = fullSnapshotPath(root, branchName);
-      copyCheckoutTree(branchPath(resolvedProjectRoot, branchName), snapshotPath);
-      paths.set(branchName, snapshotPath);
+      onProgress?.({
+        operation: "snapshot",
+        phase: "copying",
+        projectRoot: resolvedProjectRoot,
+        branchName,
+        completedBranches,
+        totalBranches: branchSnapshots.length,
+        completedBytes,
+        totalBytes
+      });
+      if (snapshotPresent) {
+        const snapshotPath = fullSnapshotPath(root, branchName);
+        copyCheckoutTree(branchPath(resolvedProjectRoot, branchName), snapshotPath);
+        paths.set(branchName, snapshotPath);
+      }
+      completedBytes += branchBytes.get(branchName) ?? 0;
+      completedBranches += 1;
+      onProgress?.({
+        operation: "snapshot",
+        phase: "copying",
+        projectRoot: resolvedProjectRoot,
+        branchName,
+        completedBranches,
+        totalBranches: branchSnapshots.length,
+        completedBytes,
+        totalBytes
+      });
     }
     const treesRoot = path.join(root, "trees");
+    onProgress?.({
+      operation: "snapshot",
+      phase: "syncing",
+      projectRoot: resolvedProjectRoot,
+      completedBranches,
+      totalBranches: branchSnapshots.length,
+      completedBytes,
+      totalBytes
+    });
     if (fs.existsSync(treesRoot)) fsyncTree(treesRoot);
     manifest = advanceSnapshotState(root, manifest, "prepared");
     return { root, manifest, paths };
@@ -434,33 +492,127 @@ function snapshotBranchTrees(projectRoot, branches, temporaryRoot) {
     throw error;
   }
 }
-function restoreProjectWorktreeSnapshot(snapshotRoot, manifest) {
-  const current = readSnapshotManifest(snapshotRoot);
+function restoreProjectWorktreeSnapshot(snapshotRoot, manifest, onProgress) {
+  let current = readSnapshotManifest(snapshotRoot);
   assertSameSnapshotIdentity(manifest, current);
-  if (current.state !== "prepared") throw new Error(`Project worktree snapshot is ${current.state}, not prepared`);
+  if (current.state !== "prepared" && current.state !== "restoring") {
+    throw new Error(`Project worktree snapshot is ${current.state}, not prepared or restoring`);
+  }
+  const treesRoot = path.join(snapshotRoot, "trees");
+  const treesRootPresent = fs.existsSync(treesRoot);
+  const treesRootStat = treesRootPresent ? fs.lstatSync(treesRoot) : null;
+  if (treesRootStat && (!treesRootStat.isDirectory() || treesRootStat.isSymbolicLink())) {
+    throw new Error("Project worktree snapshot trees root must be a real directory");
+  }
+  const branchSourceRoot = current.state === "restoring" && !treesRootPresent ? current.projectRoot : treesRoot;
   for (const { branchName, snapshotPresent } of current.branches) {
     const targetPath = branchPath(current.projectRoot, branchName);
     assertNoSymlinkBranchPathComponents(current.projectRoot, targetPath);
-    if (!snapshotPresent) continue;
-    const sourcePath = fullSnapshotPath(snapshotRoot, branchName);
+    const sourcePath = path.join(branchSourceRoot, ...branchName.split("/"));
+    if (!snapshotPresent) {
+      if (current.state === "restoring" && !treesRootPresent && fs.existsSync(sourcePath)) {
+        throw new Error(`Restored project branch unexpectedly exists: ${branchName}`);
+      }
+      continue;
+    }
     const stat = fs.lstatSync(sourcePath);
     if (!stat.isDirectory() || stat.isSymbolicLink()) {
       throw new Error(`Project worktree snapshot branch must be a real directory: ${branchName}`);
     }
   }
+  onProgress?.({
+    operation: "restore",
+    phase: "scanning",
+    projectRoot: current.projectRoot,
+    completedBranches: 0,
+    totalBranches: current.branches.length,
+    completedBytes: 0
+  });
+  const branchBytes = snapshotBranchMeasurements(current.branches, (branchName) => path.join(branchSourceRoot, ...branchName.split("/")));
+  const totalBytes = [...branchBytes.values()].reduce((total, bytes) => total + bytes, 0);
+  const presentBranches = current.branches.filter(({ snapshotPresent }) => snapshotPresent);
+  const destinationParent = path.dirname(current.projectRoot);
+  fs.mkdirSync(destinationParent, { recursive: true });
+  const canMoveWholeSnapshot = presentBranches.length > 0 && (current.state === "restoring" || treesRootStat !== null && treesRootStat.dev === fs.lstatSync(destinationParent).dev);
+  if (canMoveWholeSnapshot) {
+    if (current.state === "prepared") current = advanceSnapshotState(snapshotRoot, current, "restoring");
+    onProgress?.({
+      operation: "restore",
+      phase: "moving",
+      projectRoot: current.projectRoot,
+      completedBranches: treesRootPresent ? 0 : current.branches.length,
+      totalBranches: current.branches.length,
+      completedBytes: treesRootPresent ? 0 : totalBytes,
+      totalBytes
+    });
+    if (treesRootPresent) {
+      fs.rmSync(current.projectRoot, { recursive: true, force: true });
+      fsyncDirectoryChain(destinationParent, path.dirname(destinationParent));
+      fs.renameSync(treesRoot, current.projectRoot);
+      fsyncDirectory(snapshotRoot);
+    }
+    fsyncManagedProjectRootEntry(current.projectRoot);
+    onProgress?.({
+      operation: "restore",
+      phase: "moving",
+      projectRoot: current.projectRoot,
+      completedBranches: current.branches.length,
+      totalBranches: current.branches.length,
+      completedBytes: totalBytes,
+      totalBytes
+    });
+    return;
+  }
+  if (current.state === "restoring") {
+    throw new Error("Project worktree snapshot entered atomic restore state without a movable snapshot tree");
+  }
   for (const { branchName } of [...current.branches].sort((left, right) => right.branchName.length - left.branchName.length)) {
     const targetPath = branchPath(current.projectRoot, branchName);
     const targetParent = path.dirname(targetPath);
     fs.rmSync(targetPath, { recursive: true, force: true });
     if (fs.existsSync(targetParent)) fsyncDirectoryChain(targetParent, current.projectRoot);
   }
+  let completedBytes = 0;
+  let completedBranches = 0;
   for (const { branchName, snapshotPresent } of current.branches) {
-    if (!snapshotPresent) continue;
-    const targetPath = branchPath(current.projectRoot, branchName);
-    copyCheckoutTree(fullSnapshotPath(snapshotRoot, branchName), targetPath);
-    fsyncTree(targetPath);
-    fsyncDirectoryChain(path.dirname(targetPath), current.projectRoot);
+    onProgress?.({
+      operation: "restore",
+      phase: "copying",
+      projectRoot: current.projectRoot,
+      branchName,
+      completedBranches,
+      totalBranches: current.branches.length,
+      completedBytes,
+      totalBytes
+    });
+    if (snapshotPresent) {
+      const targetPath = branchPath(current.projectRoot, branchName);
+      copyCheckoutTree(fullSnapshotPath(snapshotRoot, branchName), targetPath);
+      fsyncTree(targetPath);
+      fsyncDirectoryChain(path.dirname(targetPath), current.projectRoot);
+    }
+    completedBytes += branchBytes.get(branchName) ?? 0;
+    completedBranches += 1;
+    onProgress?.({
+      operation: "restore",
+      phase: "copying",
+      projectRoot: current.projectRoot,
+      branchName,
+      completedBranches,
+      totalBranches: current.branches.length,
+      completedBytes,
+      totalBytes
+    });
   }
+  onProgress?.({
+    operation: "restore",
+    phase: "syncing",
+    projectRoot: current.projectRoot,
+    completedBranches,
+    totalBranches: current.branches.length,
+    completedBytes,
+    totalBytes
+  });
   fs.mkdirSync(current.projectRoot, { recursive: true });
   fsyncManagedProjectRootEntry(current.projectRoot);
 }
@@ -481,7 +633,7 @@ function persistReconciledProjectAndConsumeSnapshot(snapshot, afterProjectTreeFs
 function rollbackProjectWorktreeSnapshot(snapshot) {
   const current = readSnapshotManifest(snapshot.root);
   assertSameSnapshotIdentity(snapshot.manifest, current);
-  if (current.state === "prepared") {
+  if (current.state === "prepared" || current.state === "restoring") {
     restoreProjectWorktreeSnapshot(snapshot.root, current);
     consumeAndRemoveProjectWorktreeSnapshot(snapshot.root, current);
     return;
@@ -563,8 +715,8 @@ function recoverStaleProjectWorktreeSnapshots(input) {
       assertNoSymlinkPathComponents(projectsRoot, manifest.projectRoot);
       const ownedByThisProcessSession = manifest.ownerProcessId === currentProcessId && manifest.ownerSessionId === currentOwnerSessionId;
       if (ownedByThisProcessSession || ownerProcessId !== currentProcessId && processAlive(ownerProcessId)) continue;
-      if (manifest.state === "prepared") {
-        restoreProjectWorktreeSnapshot(candidate, manifest);
+      if (manifest.state === "prepared" || manifest.state === "restoring") {
+        restoreProjectWorktreeSnapshot(candidate, manifest, input.onProgress);
         const consumed = advanceSnapshotState(candidate, manifest, "consumed");
         removeSnapshotStorage(candidate);
         assertSameSnapshotIdentity(manifest, consumed);
@@ -750,7 +902,11 @@ async function initializeLinkedLayout(input) {
     );
     const checkoutPath = branchPath(input.projectRoot, branch.branchName);
     fs.mkdirSync(path.dirname(checkoutPath), { recursive: true });
-    await gitAsync(primaryPath, ["worktree", "add", "--force", checkoutPath, branch.branchName], `create linked worktree ${branch.branchName}`);
+    await gitAsync(
+      primaryPath,
+      ["worktree", "add", "--force", checkoutPath, branch.branchName],
+      `create linked worktree ${branch.branchName}`
+    );
   }
   return mirrorHeads;
 }
@@ -784,7 +940,11 @@ async function reconcileLinkedLayout(input) {
     fs.rmSync(checkoutPath, { recursive: true, force: true });
     await gitAsync(primaryPath, ["branch", "-f", branch.branchName, resolved.seed], `anchor project branch ${branch.branchName}`);
     fs.mkdirSync(path.dirname(checkoutPath), { recursive: true });
-    await gitAsync(primaryPath, ["worktree", "add", "--force", checkoutPath, branch.branchName], `create linked worktree ${branch.branchName}`);
+    await gitAsync(
+      primaryPath,
+      ["worktree", "add", "--force", checkoutPath, branch.branchName],
+      `create linked worktree ${branch.branchName}`
+    );
   }
   return { mirrorHeads, mutatedBranches };
 }
@@ -811,7 +971,7 @@ async function ensureProjectWorktrees(input) {
   assertNoOutstandingSnapshotForProject(snapshotStorageRoot, projectRoot);
   let snapshot;
   const ensureSnapshot = () => {
-    snapshot ??= snapshotBranchTrees(projectRoot, input.branches, snapshotStorageRoot);
+    snapshot ??= snapshotBranchTrees(projectRoot, input.branches, snapshotStorageRoot, input.onSnapshotProgress);
     return snapshot;
   };
   try {

package/dist/mjs/supervisor.mjs

@@ -5,6 +5,7 @@ const WORKER_RELOAD_EXIT_CODE = 75;
 const WORKER_RECONNECT_EXIT_CODE = 76;
 const WORKER_CREDENTIAL_RESTART_EXIT_CODE = 77;
 const WORKER_RECONNECT_DELAY_MS = 2e3;
+const WORKER_RECONNECT_JITTER_MS = 3e3;
 const SUPERVISOR_SIGNALS = ["SIGINT", "SIGTERM", "SIGHUP"];
 function defaultRuntimeSpawner(argv, options) {
   return Bun.spawn(argv, options);
@@ -12,6 +13,10 @@ function defaultRuntimeSpawner(argv, options) {
 function defaultRuntimeDelay(delayMs) {
   return new Promise((resolve) => setTimeout(resolve, delayMs));
 }
+function workerReconnectDelayMs(random = Math.random) {
+  const sample = Math.max(0, Math.min(1, random()));
+  return WORKER_RECONNECT_DELAY_MS + Math.floor(sample * WORKER_RECONNECT_JITTER_MS);
+}
 async function defaultRuntimeReaper(runtime) {
   await terminateProcessTree(runtime);
 }
@@ -24,7 +29,7 @@ function signalExitCode(signal) {
 function osSignalNumber(signal) {
   return osConstants.signals[signal];
 }
-async function superviseWorkerRuntime(argv, env = process.env, spawnRuntime = defaultRuntimeSpawner, waitBeforeReconnect = defaultRuntimeDelay, reapCredentialRuntime = defaultRuntimeReaper) {
+async function superviseWorkerRuntime(argv, env = process.env, spawnRuntime = defaultRuntimeSpawner, waitBeforeReconnect = defaultRuntimeDelay, reapCredentialRuntime = defaultRuntimeReaper, reconnectDelay = workerReconnectDelayMs) {
   if (!process.argv[1]) {
     throw new Error("Cannot locate the r5d-worker entrypoint");
   }
@@ -76,9 +81,10 @@ async function superviseWorkerRuntime(argv, env = process.env, spawnRuntime = de
         continue;
       }
       if (exitCode === WORKER_RECONNECT_EXIT_CODE) {
-        process.stderr.write(`[r5d-worker] reconnecting in ${WORKER_RECONNECT_DELAY_MS}ms
+        const reconnectDelayMs = reconnectDelay();
+        process.stderr.write(`[r5d-worker] reconnecting in ${reconnectDelayMs}ms
 `);
-        await waitBeforeReconnect(WORKER_RECONNECT_DELAY_MS);
+        await waitBeforeReconnect(reconnectDelayMs);
         if (receivedSignal) return signalExitCode(receivedSignal);
         continue;
       }
@@ -93,8 +99,10 @@ export {
   WORKER_CREDENTIAL_RESTART_EXIT_CODE,
   WORKER_RECONNECT_DELAY_MS,
   WORKER_RECONNECT_EXIT_CODE,
+  WORKER_RECONNECT_JITTER_MS,
   WORKER_RELOAD_EXIT_CODE,
   WORKER_RUNTIME_ENV,
   isRetryableWorkerServerStatus,
-  superviseWorkerRuntime
+  superviseWorkerRuntime,
+  workerReconnectDelayMs
 };

package/dist/types/main.d.ts

@@ -23,6 +23,26 @@ type WorkerProjectConfig = {
         baseCommitHash: string;
     }>;
 };
+type WorkerLifecycleProgress = {
+    phase: "starting" | "recovering" | "configuring" | "syncing" | "ready" | "failed";
+    operationId: string;
+    detail?: string;
+    completedBytes?: number;
+    totalBytes?: number;
+    completedItems?: number;
+    totalItems?: number;
+};
+declare function boundedWorkerLifecycleProgress(progress: WorkerLifecycleProgress): WorkerLifecycleProgress;
+declare function workerServerHeartbeatAction(input: {
+    elapsedMs: number;
+    probeElapsedMs?: number;
+    timeoutMs: number;
+    watchdogSweepDelayed: boolean;
+}): "wait" | "start_probe" | "terminate";
+export declare const workerLifecycleTestHarness: {
+    boundedWorkerLifecycleProgress: typeof boundedWorkerLifecycleProgress;
+    workerServerHeartbeatAction: typeof workerServerHeartbeatAction;
+};
 type WorkspaceSyncTrigger = {
     type: "connect" | "inbound_head" | "write" | "edit" | "shell_inline" | "process_terminal" | "process_cancel" | "periodic" | "manual" | "remediation" | "remediation_confirm" | "remediation_reset";
     sessionId?: string;
@@ -813,12 +833,16 @@ declare function createNodePtyBridge(options: {
 }): WorkerPty;
 export declare const workerPtyBridgeTestHarness: {
     create: typeof createNodePtyBridge;
+    closeAllBestEffort: typeof closePtysBestEffort;
 };
 export declare function resolveHostShell(command?: string, platform?: NodeJS.Platform): {
     file: string;
     args: string[];
     env?: Record<string, string>;
 };
+declare function closePtysBestEffort(ptys: Iterable<readonly [string, {
+    kill: () => void;
+}]>, reportFailure: (ptyId: string, error: unknown) => void): void;
 declare function terminateCredentialBearingChildren(children: readonly {
     label: string;
     terminate: () => void | Promise<void>;

package/dist/types/project-snapshot-recovery-runner.d.ts

@@ -0,0 +1,10 @@
+import { recoverStaleProjectWorktreeSnapshots, type ProjectWorktreeSnapshotProgress } from "./project-worktrees";
+export declare const PROJECT_SNAPSHOT_RECOVERY_HELPER_COMMAND = "__recover-project-snapshots";
+type ProjectSnapshotRecoveryResult = ReturnType<typeof recoverStaleProjectWorktreeSnapshots>;
+export declare function runProjectSnapshotRecoveryHelper(args: readonly string[]): number;
+export declare function recoverProjectSnapshotsInChild(input: {
+    projectsRoot: string;
+    snapshotRoot: string;
+    onProgress?: (progress: ProjectWorktreeSnapshotProgress) => void;
+}): Promise<ProjectSnapshotRecoveryResult>;
+export {};

package/dist/types/project-worktrees.d.ts

@@ -2,7 +2,7 @@ export declare const PROJECT_WORKTREE_SNAPSHOT_PREFIX = "r5d-project-worktrees-"
 declare const PROJECT_WORKTREE_SNAPSHOT_MANIFEST_VERSION: 1;
 type ProjectWorktreeSnapshotManifest = {
     version: typeof PROJECT_WORKTREE_SNAPSHOT_MANIFEST_VERSION;
-    state: "building" | "prepared" | "consumed";
+    state: "building" | "prepared" | "restoring" | "consumed";
     ownerProcessId: number;
     ownerSessionId: string;
     projectRoot: string;
@@ -12,6 +12,16 @@ type ProjectWorktreeSnapshotManifest = {
     }>;
     createdAt: string;
 };
+export type ProjectWorktreeSnapshotProgress = {
+    operation: "snapshot" | "restore";
+    phase: "scanning" | "copying" | "moving" | "syncing" | "cleaning";
+    projectRoot: string;
+    branchName?: string;
+    completedBranches: number;
+    totalBranches: number;
+    completedBytes: number;
+    totalBytes?: number;
+};
 export type ProjectWorktreeBranch = {
     branchId?: string;
     branchName: string;
@@ -68,6 +78,7 @@ export declare function recoverStaleProjectWorktreeSnapshots(input: {
     processAlive?: (processId: number) => boolean;
     currentProcessId?: number;
     currentOwnerSessionId?: string;
+    onProgress?: (progress: ProjectWorktreeSnapshotProgress) => void;
 }): {
     restored: string[];
     removed: string[];
@@ -110,6 +121,7 @@ export declare function ensureProjectWorktrees(input: {
     } | null;
     exactMirrorBranches?: ReadonlySet<string>;
     snapshotRoot?: string;
+    onSnapshotProgress?: (progress: ProjectWorktreeSnapshotProgress) => void;
 }): Promise<ProjectWorktreeState[]>;
 export declare function projectWorktreeOperationInProgress(checkoutPath: string): boolean;
 export declare function createLinkedProjectBranch(input: {

package/dist/types/supervisor.d.ts

@@ -4,6 +4,7 @@ export declare const WORKER_RELOAD_EXIT_CODE = 75;
 export declare const WORKER_RECONNECT_EXIT_CODE = 76;
 export declare const WORKER_CREDENTIAL_RESTART_EXIT_CODE = 77;
 export declare const WORKER_RECONNECT_DELAY_MS = 2000;
+export declare const WORKER_RECONNECT_JITTER_MS = 3000;
 type RuntimeProcess = KillableSubprocess;
 export type RuntimeSpawner = (argv: string[], options: {
     stdin: "inherit";
@@ -14,6 +15,8 @@ export type RuntimeSpawner = (argv: string[], options: {
 }) => RuntimeProcess;
 export type RuntimeDelay = (delayMs: number) => Promise<void>;
 export type RuntimeReaper = (runtime: RuntimeProcess) => Promise<void>;
+export type RuntimeReconnectDelay = () => number;
+export declare function workerReconnectDelayMs(random?: () => number): number;
 export declare function isRetryableWorkerServerStatus(status: number): boolean;
-export declare function superviseWorkerRuntime(argv: string[], env?: Record<string, string | undefined>, spawnRuntime?: RuntimeSpawner, waitBeforeReconnect?: RuntimeDelay, reapCredentialRuntime?: RuntimeReaper): Promise<number>;
+export declare function superviseWorkerRuntime(argv: string[], env?: Record<string, string | undefined>, spawnRuntime?: RuntimeSpawner, waitBeforeReconnect?: RuntimeDelay, reapCredentialRuntime?: RuntimeReaper, reconnectDelay?: RuntimeReconnectDelay): Promise<number>;
 export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ricsam/r5d-worker",
-  "version": "0.0.90",
+  "version": "0.0.92",
   "type": "module",
   "main": "./dist/cjs/main.cjs",
   "module": "./dist/mjs/main.mjs",