@ricsam/r5d-worker 0.0.90 → 0.0.92

package/dist/cjs/project-snapshot-recovery-runner.cjs

@@ -0,0 +1,171 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var project_snapshot_recovery_runner_exports = {};
+__export(project_snapshot_recovery_runner_exports, {
+  PROJECT_SNAPSHOT_RECOVERY_HELPER_COMMAND: () => PROJECT_SNAPSHOT_RECOVERY_HELPER_COMMAND,
+  recoverProjectSnapshotsInChild: () => recoverProjectSnapshotsInChild,
+  runProjectSnapshotRecoveryHelper: () => runProjectSnapshotRecoveryHelper
+});
+module.exports = __toCommonJS(project_snapshot_recovery_runner_exports);
+var import_node_path = __toESM(require("node:path"), 1);
+var import_node_child_process = require("node:child_process");
+var import_project_worktrees = require("./project-worktrees.cjs");
+const PROJECT_SNAPSHOT_RECOVERY_HELPER_COMMAND = "__recover-project-snapshots";
+function requireAbsolutePath(value, label) {
+  if (!value) throw new Error(`Missing ${label}`);
+  const resolved = import_node_path.default.resolve(value);
+  if (resolved !== value) throw new Error(`${label} must be an absolute path`);
+  return resolved;
+}
+function parseHelperArguments(args) {
+  let projectsRoot;
+  let snapshotRoot;
+  for (let index = 0; index < args.length; index += 1) {
+    const argument = args[index];
+    if (argument === "--projects-root") {
+      projectsRoot = args[index + 1];
+      index += 1;
+      continue;
+    }
+    if (argument === "--snapshot-root") {
+      snapshotRoot = args[index + 1];
+      index += 1;
+      continue;
+    }
+    throw new Error(`Unknown project snapshot recovery helper argument: ${argument ?? "(missing)"}`);
+  }
+  return {
+    projectsRoot: requireAbsolutePath(projectsRoot, "projects root"),
+    snapshotRoot: requireAbsolutePath(snapshotRoot, "snapshot root")
+  };
+}
+function writeHelperMessage(message) {
+  process.stdout.write(`${JSON.stringify(message)}
+`);
+}
+function runProjectSnapshotRecoveryHelper(args) {
+  const { projectsRoot, snapshotRoot } = parseHelperArguments(args);
+  const result = (0, import_project_worktrees.recoverStaleProjectWorktreeSnapshots)({
+    projectsRoot,
+    temporaryRoot: snapshotRoot,
+    onProgress: (progress) => writeHelperMessage({ type: "progress", progress })
+  });
+  writeHelperMessage({ type: "result", result });
+  return result.failed.length === 0 ? 0 : 1;
+}
+function isRecoveryProgress(value) {
+  if (!value || typeof value !== "object") return false;
+  const progress = value;
+  return progress.operation === "restore" && (progress.phase === "scanning" || progress.phase === "copying" || progress.phase === "moving" || progress.phase === "syncing" || progress.phase === "cleaning") && typeof progress.projectRoot === "string" && Number.isSafeInteger(progress.completedBranches) && Number.isSafeInteger(progress.totalBranches) && typeof progress.completedBytes === "number" && Number.isFinite(progress.completedBytes) && (progress.totalBytes === void 0 || typeof progress.totalBytes === "number" && Number.isFinite(progress.totalBytes));
+}
+function isRecoveryResult(value) {
+  if (!value || typeof value !== "object") return false;
+  const result = value;
+  return Array.isArray(result.restored) && Array.isArray(result.removed) && Array.isArray(result.failed);
+}
+function recoveryChildEnvironment() {
+  const environment = {};
+  for (const name of ["PATH", "TMPDIR", "TMP", "TEMP", "SystemRoot", "WINDIR"]) {
+    const value = process.env[name];
+    if (value !== void 0) environment[name] = value;
+  }
+  return environment;
+}
+async function recoverProjectSnapshotsInChild(input) {
+  const entrypoint = process.argv[1];
+  if (!entrypoint) throw new Error("Cannot locate the r5d-worker runtime entrypoint for snapshot recovery");
+  const child = (0, import_node_child_process.spawn)(
+    process.execPath,
+    [
+      entrypoint,
+      PROJECT_SNAPSHOT_RECOVERY_HELPER_COMMAND,
+      "--projects-root",
+      import_node_path.default.resolve(input.projectsRoot),
+      "--snapshot-root",
+      import_node_path.default.resolve(input.snapshotRoot)
+    ],
+    {
+      env: recoveryChildEnvironment(),
+      stdio: ["ignore", "pipe", "pipe"]
+    }
+  );
+  const terminateOnParentExit = () => {
+    if (child.exitCode === null && child.signalCode === null) child.kill("SIGTERM");
+  };
+  const parentProcessEvents = process;
+  parentProcessEvents.once("exit", terminateOnParentExit);
+  let stdout = "";
+  let stderr = "";
+  let result;
+  child.stdout.setEncoding("utf8");
+  child.stdout.on("data", (chunk) => {
+    stdout += chunk;
+    while (true) {
+      const newline = stdout.indexOf("\n");
+      if (newline < 0) break;
+      const line = stdout.slice(0, newline);
+      stdout = stdout.slice(newline + 1);
+      if (!line.trim()) continue;
+      let message;
+      try {
+        message = JSON.parse(line);
+      } catch {
+        stderr += `Invalid recovery helper output: ${line}
+`;
+        continue;
+      }
+      if (!message || typeof message !== "object") continue;
+      const record = message;
+      if (record.type === "progress" && isRecoveryProgress(record.progress)) input.onProgress?.(record.progress);
+      if (record.type === "result" && isRecoveryResult(record.result)) result = record.result;
+    }
+  });
+  child.stderr.setEncoding("utf8");
+  child.stderr.on("data", (chunk) => {
+    stderr = `${stderr}${chunk}`.slice(-32768);
+  });
+  const exitCode = await new Promise((resolve, reject) => {
+    child.once("error", reject);
+    child.once("close", (code, signal) => {
+      if (signal) reject(new Error(`Project snapshot recovery helper exited on signal ${signal}`));
+      else resolve(code ?? 1);
+    });
+  }).finally(() => parentProcessEvents.off("exit", terminateOnParentExit));
+  if (exitCode !== 0 || !result) {
+    const detail = stderr.trim();
+    throw new Error(detail || `Project snapshot recovery helper exited with status ${exitCode}`);
+  }
+  return result;
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  PROJECT_SNAPSHOT_RECOVERY_HELPER_COMMAND,
+  recoverProjectSnapshotsInChild,
+  runProjectSnapshotRecoveryHelper
+});

package/dist/cjs/project-worktrees.cjs

@@ -318,7 +318,7 @@ function normalizeSnapshotManifest(value) {
   if (record.version !== PROJECT_WORKTREE_SNAPSHOT_MANIFEST_VERSION) {
     throw new Error("Unsupported project worktree snapshot manifest version");
   }
-  if (record.state !== "building" && record.state !== "prepared" && record.state !== "consumed") {
+  if (record.state !== "building" && record.state !== "prepared" && record.state !== "restoring" && record.state !== "consumed") {
     throw new Error("Project worktree snapshot manifest has an invalid state");
   }
   if (!Number.isSafeInteger(record.ownerProcessId) || record.ownerProcessId <= 0) {
@@ -399,7 +399,7 @@ function assertSameSnapshotIdentity(expected, actual) {
 function advanceSnapshotState(snapshotRoot, expected, state) {
   const current = readSnapshotManifest(snapshotRoot);
   assertSameSnapshotIdentity(expected, current);
-  const allowed = current.state === "building" && state === "prepared" || current.state === "prepared" && state === "consumed" || current.state === state;
+  const allowed = current.state === "building" && state === "prepared" || current.state === "prepared" && state === "restoring" || current.state === "prepared" && state === "consumed" || current.state === "restoring" && state === "consumed" || current.state === state;
   if (!allowed) throw new Error(`Project worktree snapshot cannot advance from ${current.state} to ${state}`);
   return current.state === state ? current : writeSnapshotManifest(snapshotRoot, { ...current, state });
 }
@@ -423,6 +423,20 @@ function copyCheckoutTree(sourceRoot, targetRoot) {
     verbatimSymlinks: true
   });
 }
+function measureCheckoutTree(root) {
+  const stat = import_node_fs.default.lstatSync(root);
+  if (stat.isSymbolicLink()) return 0;
+  if (stat.isFile()) return stat.size;
+  if (!stat.isDirectory()) return 0;
+  let bytes = 0;
+  for (const entry of import_node_fs.default.readdirSync(root).sort()) bytes += measureCheckoutTree(import_node_path.default.join(root, entry));
+  return bytes;
+}
+function snapshotBranchMeasurements(branches, sourcePath) {
+  return new Map(
+    branches.map(({ branchName, snapshotPresent }) => [branchName, snapshotPresent ? measureCheckoutTree(sourcePath(branchName)) : 0])
+  );
+}
 function assertNoOutstandingSnapshotForProject(temporaryRoot, projectRoot) {
   let entries;
   try {
@@ -445,7 +459,7 @@ function assertNoOutstandingSnapshotForProject(temporaryRoot, projectRoot) {
     }
   }
 }
-function snapshotBranchTrees(projectRoot, branches, temporaryRoot) {
+function snapshotBranchTrees(projectRoot, branches, temporaryRoot, onProgress) {
   const resolvedProjectRoot = import_node_path.default.resolve(projectRoot);
   const resolvedTemporaryRoot = import_node_path.default.resolve(temporaryRoot ?? import_node_os.default.tmpdir());
   assertNoOutstandingSnapshotForProject(resolvedTemporaryRoot, resolvedProjectRoot);
@@ -472,13 +486,57 @@ function snapshotBranchTrees(projectRoot, branches, temporaryRoot) {
       branches: branchSnapshots,
       createdAt: (/* @__PURE__ */ new Date()).toISOString()
     });
+    onProgress?.({
+      operation: "snapshot",
+      phase: "scanning",
+      projectRoot: resolvedProjectRoot,
+      completedBranches: 0,
+      totalBranches: branchSnapshots.length,
+      completedBytes: 0
+    });
+    const branchBytes = snapshotBranchMeasurements(branchSnapshots, (branchName) => branchPath(resolvedProjectRoot, branchName));
+    const totalBytes = [...branchBytes.values()].reduce((total, bytes) => total + bytes, 0);
+    let completedBytes = 0;
+    let completedBranches = 0;
     for (const { branchName, snapshotPresent } of branchSnapshots) {
-      if (!snapshotPresent) continue;
-      const snapshotPath = fullSnapshotPath(root, branchName);
-      copyCheckoutTree(branchPath(resolvedProjectRoot, branchName), snapshotPath);
-      paths.set(branchName, snapshotPath);
+      onProgress?.({
+        operation: "snapshot",
+        phase: "copying",
+        projectRoot: resolvedProjectRoot,
+        branchName,
+        completedBranches,
+        totalBranches: branchSnapshots.length,
+        completedBytes,
+        totalBytes
+      });
+      if (snapshotPresent) {
+        const snapshotPath = fullSnapshotPath(root, branchName);
+        copyCheckoutTree(branchPath(resolvedProjectRoot, branchName), snapshotPath);
+        paths.set(branchName, snapshotPath);
+      }
+      completedBytes += branchBytes.get(branchName) ?? 0;
+      completedBranches += 1;
+      onProgress?.({
+        operation: "snapshot",
+        phase: "copying",
+        projectRoot: resolvedProjectRoot,
+        branchName,
+        completedBranches,
+        totalBranches: branchSnapshots.length,
+        completedBytes,
+        totalBytes
+      });
     }
     const treesRoot = import_node_path.default.join(root, "trees");
+    onProgress?.({
+      operation: "snapshot",
+      phase: "syncing",
+      projectRoot: resolvedProjectRoot,
+      completedBranches,
+      totalBranches: branchSnapshots.length,
+      completedBytes,
+      totalBytes
+    });
     if (import_node_fs.default.existsSync(treesRoot)) fsyncTree(treesRoot);
     manifest = advanceSnapshotState(root, manifest, "prepared");
     return { root, manifest, paths };
@@ -487,33 +545,127 @@ function snapshotBranchTrees(projectRoot, branches, temporaryRoot) {
     throw error;
   }
 }
-function restoreProjectWorktreeSnapshot(snapshotRoot, manifest) {
-  const current = readSnapshotManifest(snapshotRoot);
+function restoreProjectWorktreeSnapshot(snapshotRoot, manifest, onProgress) {
+  let current = readSnapshotManifest(snapshotRoot);
   assertSameSnapshotIdentity(manifest, current);
-  if (current.state !== "prepared") throw new Error(`Project worktree snapshot is ${current.state}, not prepared`);
+  if (current.state !== "prepared" && current.state !== "restoring") {
+    throw new Error(`Project worktree snapshot is ${current.state}, not prepared or restoring`);
+  }
+  const treesRoot = import_node_path.default.join(snapshotRoot, "trees");
+  const treesRootPresent = import_node_fs.default.existsSync(treesRoot);
+  const treesRootStat = treesRootPresent ? import_node_fs.default.lstatSync(treesRoot) : null;
+  if (treesRootStat && (!treesRootStat.isDirectory() || treesRootStat.isSymbolicLink())) {
+    throw new Error("Project worktree snapshot trees root must be a real directory");
+  }
+  const branchSourceRoot = current.state === "restoring" && !treesRootPresent ? current.projectRoot : treesRoot;
   for (const { branchName, snapshotPresent } of current.branches) {
     const targetPath = branchPath(current.projectRoot, branchName);
     assertNoSymlinkBranchPathComponents(current.projectRoot, targetPath);
-    if (!snapshotPresent) continue;
-    const sourcePath = fullSnapshotPath(snapshotRoot, branchName);
+    const sourcePath = import_node_path.default.join(branchSourceRoot, ...branchName.split("/"));
+    if (!snapshotPresent) {
+      if (current.state === "restoring" && !treesRootPresent && import_node_fs.default.existsSync(sourcePath)) {
+        throw new Error(`Restored project branch unexpectedly exists: ${branchName}`);
+      }
+      continue;
+    }
     const stat = import_node_fs.default.lstatSync(sourcePath);
     if (!stat.isDirectory() || stat.isSymbolicLink()) {
       throw new Error(`Project worktree snapshot branch must be a real directory: ${branchName}`);
     }
   }
+  onProgress?.({
+    operation: "restore",
+    phase: "scanning",
+    projectRoot: current.projectRoot,
+    completedBranches: 0,
+    totalBranches: current.branches.length,
+    completedBytes: 0
+  });
+  const branchBytes = snapshotBranchMeasurements(current.branches, (branchName) => import_node_path.default.join(branchSourceRoot, ...branchName.split("/")));
+  const totalBytes = [...branchBytes.values()].reduce((total, bytes) => total + bytes, 0);
+  const presentBranches = current.branches.filter(({ snapshotPresent }) => snapshotPresent);
+  const destinationParent = import_node_path.default.dirname(current.projectRoot);
+  import_node_fs.default.mkdirSync(destinationParent, { recursive: true });
+  const canMoveWholeSnapshot = presentBranches.length > 0 && (current.state === "restoring" || treesRootStat !== null && treesRootStat.dev === import_node_fs.default.lstatSync(destinationParent).dev);
+  if (canMoveWholeSnapshot) {
+    if (current.state === "prepared") current = advanceSnapshotState(snapshotRoot, current, "restoring");
+    onProgress?.({
+      operation: "restore",
+      phase: "moving",
+      projectRoot: current.projectRoot,
+      completedBranches: treesRootPresent ? 0 : current.branches.length,
+      totalBranches: current.branches.length,
+      completedBytes: treesRootPresent ? 0 : totalBytes,
+      totalBytes
+    });
+    if (treesRootPresent) {
+      import_node_fs.default.rmSync(current.projectRoot, { recursive: true, force: true });
+      fsyncDirectoryChain(destinationParent, import_node_path.default.dirname(destinationParent));
+      import_node_fs.default.renameSync(treesRoot, current.projectRoot);
+      fsyncDirectory(snapshotRoot);
+    }
+    fsyncManagedProjectRootEntry(current.projectRoot);
+    onProgress?.({
+      operation: "restore",
+      phase: "moving",
+      projectRoot: current.projectRoot,
+      completedBranches: current.branches.length,
+      totalBranches: current.branches.length,
+      completedBytes: totalBytes,
+      totalBytes
+    });
+    return;
+  }
+  if (current.state === "restoring") {
+    throw new Error("Project worktree snapshot entered atomic restore state without a movable snapshot tree");
+  }
   for (const { branchName } of [...current.branches].sort((left, right) => right.branchName.length - left.branchName.length)) {
     const targetPath = branchPath(current.projectRoot, branchName);
     const targetParent = import_node_path.default.dirname(targetPath);
     import_node_fs.default.rmSync(targetPath, { recursive: true, force: true });
     if (import_node_fs.default.existsSync(targetParent)) fsyncDirectoryChain(targetParent, current.projectRoot);
   }
+  let completedBytes = 0;
+  let completedBranches = 0;
   for (const { branchName, snapshotPresent } of current.branches) {
-    if (!snapshotPresent) continue;
-    const targetPath = branchPath(current.projectRoot, branchName);
-    copyCheckoutTree(fullSnapshotPath(snapshotRoot, branchName), targetPath);
-    fsyncTree(targetPath);
-    fsyncDirectoryChain(import_node_path.default.dirname(targetPath), current.projectRoot);
+    onProgress?.({
+      operation: "restore",
+      phase: "copying",
+      projectRoot: current.projectRoot,
+      branchName,
+      completedBranches,
+      totalBranches: current.branches.length,
+      completedBytes,
+      totalBytes
+    });
+    if (snapshotPresent) {
+      const targetPath = branchPath(current.projectRoot, branchName);
+      copyCheckoutTree(fullSnapshotPath(snapshotRoot, branchName), targetPath);
+      fsyncTree(targetPath);
+      fsyncDirectoryChain(import_node_path.default.dirname(targetPath), current.projectRoot);
+    }
+    completedBytes += branchBytes.get(branchName) ?? 0;
+    completedBranches += 1;
+    onProgress?.({
+      operation: "restore",
+      phase: "copying",
+      projectRoot: current.projectRoot,
+      branchName,
+      completedBranches,
+      totalBranches: current.branches.length,
+      completedBytes,
+      totalBytes
+    });
   }
+  onProgress?.({
+    operation: "restore",
+    phase: "syncing",
+    projectRoot: current.projectRoot,
+    completedBranches,
+    totalBranches: current.branches.length,
+    completedBytes,
+    totalBytes
+  });
   import_node_fs.default.mkdirSync(current.projectRoot, { recursive: true });
   fsyncManagedProjectRootEntry(current.projectRoot);
 }
@@ -534,7 +686,7 @@ function persistReconciledProjectAndConsumeSnapshot(snapshot, afterProjectTreeFs
 function rollbackProjectWorktreeSnapshot(snapshot) {
   const current = readSnapshotManifest(snapshot.root);
   assertSameSnapshotIdentity(snapshot.manifest, current);
-  if (current.state === "prepared") {
+  if (current.state === "prepared" || current.state === "restoring") {
     restoreProjectWorktreeSnapshot(snapshot.root, current);
     consumeAndRemoveProjectWorktreeSnapshot(snapshot.root, current);
     return;
@@ -616,8 +768,8 @@ function recoverStaleProjectWorktreeSnapshots(input) {
       assertNoSymlinkPathComponents(projectsRoot, manifest.projectRoot);
       const ownedByThisProcessSession = manifest.ownerProcessId === currentProcessId && manifest.ownerSessionId === currentOwnerSessionId;
       if (ownedByThisProcessSession || ownerProcessId !== currentProcessId && processAlive(ownerProcessId)) continue;
-      if (manifest.state === "prepared") {
-        restoreProjectWorktreeSnapshot(candidate, manifest);
+      if (manifest.state === "prepared" || manifest.state === "restoring") {
+        restoreProjectWorktreeSnapshot(candidate, manifest, input.onProgress);
         const consumed = advanceSnapshotState(candidate, manifest, "consumed");
         removeSnapshotStorage(candidate);
         assertSameSnapshotIdentity(manifest, consumed);
@@ -803,7 +955,11 @@ async function initializeLinkedLayout(input) {
     );
     const checkoutPath = branchPath(input.projectRoot, branch.branchName);
     import_node_fs.default.mkdirSync(import_node_path.default.dirname(checkoutPath), { recursive: true });
-    await gitAsync(primaryPath, ["worktree", "add", "--force", checkoutPath, branch.branchName], `create linked worktree ${branch.branchName}`);
+    await gitAsync(
+      primaryPath,
+      ["worktree", "add", "--force", checkoutPath, branch.branchName],
+      `create linked worktree ${branch.branchName}`
+    );
   }
   return mirrorHeads;
 }
@@ -837,7 +993,11 @@ async function reconcileLinkedLayout(input) {
     import_node_fs.default.rmSync(checkoutPath, { recursive: true, force: true });
     await gitAsync(primaryPath, ["branch", "-f", branch.branchName, resolved.seed], `anchor project branch ${branch.branchName}`);
     import_node_fs.default.mkdirSync(import_node_path.default.dirname(checkoutPath), { recursive: true });
-    await gitAsync(primaryPath, ["worktree", "add", "--force", checkoutPath, branch.branchName], `create linked worktree ${branch.branchName}`);
+    await gitAsync(
+      primaryPath,
+      ["worktree", "add", "--force", checkoutPath, branch.branchName],
+      `create linked worktree ${branch.branchName}`
+    );
   }
   return { mirrorHeads, mutatedBranches };
 }
@@ -864,7 +1024,7 @@ async function ensureProjectWorktrees(input) {
   assertNoOutstandingSnapshotForProject(snapshotStorageRoot, projectRoot);
   let snapshot;
   const ensureSnapshot = () => {
-    snapshot ??= snapshotBranchTrees(projectRoot, input.branches, snapshotStorageRoot);
+    snapshot ??= snapshotBranchTrees(projectRoot, input.branches, snapshotStorageRoot, input.onSnapshotProgress);
     return snapshot;
   };
   try {

package/dist/cjs/supervisor.cjs

@@ -21,10 +21,12 @@ __export(supervisor_exports, {
   WORKER_CREDENTIAL_RESTART_EXIT_CODE: () => WORKER_CREDENTIAL_RESTART_EXIT_CODE,
   WORKER_RECONNECT_DELAY_MS: () => WORKER_RECONNECT_DELAY_MS,
   WORKER_RECONNECT_EXIT_CODE: () => WORKER_RECONNECT_EXIT_CODE,
+  WORKER_RECONNECT_JITTER_MS: () => WORKER_RECONNECT_JITTER_MS,
   WORKER_RELOAD_EXIT_CODE: () => WORKER_RELOAD_EXIT_CODE,
   WORKER_RUNTIME_ENV: () => WORKER_RUNTIME_ENV,
   isRetryableWorkerServerStatus: () => isRetryableWorkerServerStatus,
-  superviseWorkerRuntime: () => superviseWorkerRuntime
+  superviseWorkerRuntime: () => superviseWorkerRuntime,
+  workerReconnectDelayMs: () => workerReconnectDelayMs
 });
 module.exports = __toCommonJS(supervisor_exports);
 var import_node_os = require("node:os");
@@ -34,6 +36,7 @@ const WORKER_RELOAD_EXIT_CODE = 75;
 const WORKER_RECONNECT_EXIT_CODE = 76;
 const WORKER_CREDENTIAL_RESTART_EXIT_CODE = 77;
 const WORKER_RECONNECT_DELAY_MS = 2e3;
+const WORKER_RECONNECT_JITTER_MS = 3e3;
 const SUPERVISOR_SIGNALS = ["SIGINT", "SIGTERM", "SIGHUP"];
 function defaultRuntimeSpawner(argv, options) {
   return Bun.spawn(argv, options);
@@ -41,6 +44,10 @@ function defaultRuntimeSpawner(argv, options) {
 function defaultRuntimeDelay(delayMs) {
   return new Promise((resolve) => setTimeout(resolve, delayMs));
 }
+function workerReconnectDelayMs(random = Math.random) {
+  const sample = Math.max(0, Math.min(1, random()));
+  return WORKER_RECONNECT_DELAY_MS + Math.floor(sample * WORKER_RECONNECT_JITTER_MS);
+}
 async function defaultRuntimeReaper(runtime) {
   await (0, import_process_tree.terminateProcessTree)(runtime);
 }
@@ -53,7 +60,7 @@ function signalExitCode(signal) {
 function osSignalNumber(signal) {
   return import_node_os.constants.signals[signal];
 }
-async function superviseWorkerRuntime(argv, env = process.env, spawnRuntime = defaultRuntimeSpawner, waitBeforeReconnect = defaultRuntimeDelay, reapCredentialRuntime = defaultRuntimeReaper) {
+async function superviseWorkerRuntime(argv, env = process.env, spawnRuntime = defaultRuntimeSpawner, waitBeforeReconnect = defaultRuntimeDelay, reapCredentialRuntime = defaultRuntimeReaper, reconnectDelay = workerReconnectDelayMs) {
   if (!process.argv[1]) {
     throw new Error("Cannot locate the r5d-worker entrypoint");
   }
@@ -105,9 +112,10 @@ async function superviseWorkerRuntime(argv, env = process.env, spawnRuntime = de
         continue;
       }
       if (exitCode === WORKER_RECONNECT_EXIT_CODE) {
-        process.stderr.write(`[r5d-worker] reconnecting in ${WORKER_RECONNECT_DELAY_MS}ms
+        const reconnectDelayMs = reconnectDelay();
+        process.stderr.write(`[r5d-worker] reconnecting in ${reconnectDelayMs}ms
 `);
-        await waitBeforeReconnect(WORKER_RECONNECT_DELAY_MS);
+        await waitBeforeReconnect(reconnectDelayMs);
         if (receivedSignal) return signalExitCode(receivedSignal);
         continue;
       }
@@ -123,8 +131,10 @@ async function superviseWorkerRuntime(argv, env = process.env, spawnRuntime = de
   WORKER_CREDENTIAL_RESTART_EXIT_CODE,
   WORKER_RECONNECT_DELAY_MS,
   WORKER_RECONNECT_EXIT_CODE,
+  WORKER_RECONNECT_JITTER_MS,
   WORKER_RELOAD_EXIT_CODE,
   WORKER_RUNTIME_ENV,
   isRetryableWorkerServerStatus,
-  superviseWorkerRuntime
+  superviseWorkerRuntime,
+  workerReconnectDelayMs
 });