@ricsam/r5d-worker 0.0.88 → 0.0.90

package/README.md

@@ -41,6 +41,8 @@ Worker labels are mandatory and unique per user. Choose labels that describe hos
 
 The supervisor also reconnects automatically after server rollouts and transient network failures. A brief bounded delay prevents a tight retry loop while the server is unavailable. Pressing Ctrl+C still stops the worker, and agent-managed processes retain their existing in-process recovery path while disconnected.
 
+The server registry is the sole heartbeat timeout authority. The worker answers server pings but does not run an independent connection watchdog, because synchronous bounded workspace operations can temporarily occupy its event loop. The server keeps those operations leased for their bounded lifetime and closes genuinely stale connections after the lease ends.
+
 Credential-generation restart fencing protects against stale or accidentally detached descendants. It is not a hostile same-UID sandbox: code running as the worker's OS user can read or tamper with that user's config and auth files, install a persistent user service, or return after reboot. Run untrusted agents in a disposable VM that is reset at the boundary, or use a separately privileged credential broker/service account. When rotating the worker's own local `r5dctl` credential, stop the worker and reboot/reset the host before writing the replacement credential so an old same-UID descendant cannot read the new config.
 
 One-click updates use the current user's global npm prefix and never invoke `sudo`. If that prefix is not writable, User Settings provides the exact command to run manually.

package/dist/cjs/main.cjs

@@ -63,7 +63,6 @@ var import_node_child_process = require("node:child_process");
 var import_bun_sqlite = require("bun:sqlite");
 var import_cli_update = require("./cli-update.cjs");
 var import_git_process_environment = require("./git-process-environment.cjs");
-var import_heartbeat = require("./heartbeat.cjs");
 var import_process_tree = require("./process-tree.cjs");
 var import_pty_output_coalescer = require("./pty-output-coalescer.cjs");
 var import_port_forward_client = require("./port-forward-client.cjs");
@@ -150,6 +149,11 @@ class StaleWorkerAdmissionError extends Error {
     this.name = "StaleWorkerAdmissionError";
   }
 }
+class WorkspaceSyncAdmissionChangedError extends Error {
+  name = "WorkspaceSyncAdmissionChangedError";
+}
+let unsafeWorkspaceSyncRecoveryAttempts = 0;
+let unsafeWorkspaceSyncBouncePending = false;
 function advanceWorkerAdmissionGeneration() {
   workerAdmissionGeneration += 1;
   return workerAdmissionGeneration;
@@ -1633,16 +1637,33 @@ function workspaceSyncFailureHydrationIsSafe(input) {
   if (input.error instanceof import_workspace_git_sync.WorkspaceRemediationAncestryError) return true;
   if (input.resetToCanonical) return false;
   try {
-    return input.inspectCurrentHydration();
+    return input.inspectRecoverableHydration();
   } catch {
     return false;
   }
 }
+const WORKSPACE_UNSAFE_SYNC_MAX_RECOVERY_ATTEMPTS = 3;
+const WORKSPACE_UNSAFE_SYNC_RECOVERY_CLOSE_CODE = 4008;
+function unsafeWorkspaceSyncRecoveryBounceDelayMs(attempt) {
+  return Math.min(1e3 * 2 ** Math.max(0, attempt - 1), 3e4);
+}
+function unsafeWorkspaceSyncFailureResponse(input) {
+  if (input.resetToCanonical) return "fatal_exit";
+  if (!input.hasRetainableWork) return "fatal_exit";
+  if (input.recoveryAttempts >= input.maxRecoveryAttempts) return "fatal_exit";
+  if (input.bouncePending) return "recovery_pending";
+  return "recovery_bounce";
+}
 function fenceUnsafeWorkspaceSyncFailure(input) {
-  if (input.hydrationCurrent) return false;
+  if (input.hydrationRecoverable) return "none";
   input.invalidateExecution();
-  requireCredentialGenerationRestart(true, input.fatalExit, input.exitCode);
-  return true;
+  const response = unsafeWorkspaceSyncFailureResponse(input);
+  if (response === "fatal_exit") {
+    requireCredentialGenerationRestart(true, input.fatalExit, input.exitCode);
+    throw new Error("Unsafe workspace synchronization fence continued past its fatal exit");
+  }
+  if (response === "recovery_bounce") input.scheduleRecoveryBounce();
+  return response;
 }
 const workerGitSecurityTestHarness = {
   commandArgs: workerGitCommand.commandArgs,
@@ -1670,6 +1691,11 @@ const workerGitSecurityTestHarness = {
   credentialReapContractStatus,
   verifiedCredentialReapContract,
   fenceUnsafeWorkspaceSyncFailure,
+  unsafeWorkspaceSyncFailureResponse,
+  unsafeWorkspaceSyncRecoveryBounceDelayMs,
+  WORKSPACE_UNSAFE_SYNC_MAX_RECOVERY_ATTEMPTS,
+  WORKSPACE_UNSAFE_SYNC_RECOVERY_CLOSE_CODE,
+  WorkspaceSyncAdmissionChangedError,
   workspaceConfigurationIncidentDeferral,
   workspaceIncidentTerminalClearDisposition,
   workspaceIncidentTerminalRefreshFence,
@@ -4286,6 +4312,28 @@ async function startWorker(options) {
       );
     }
   };
+  const scheduleUnsafeWorkspaceSyncRecoveryBounce = (cause) => {
+    unsafeWorkspaceSyncBouncePending = true;
+    const attempt = unsafeWorkspaceSyncRecoveryAttempts;
+    if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
+      process.stderr.write(
+        `[r5d-worker] workspace synchronization failed without a recoverable hydration basis; awaiting reconnect recovery (attempt ${attempt} of ${WORKSPACE_UNSAFE_SYNC_MAX_RECOVERY_ATTEMPTS}): ${cause}
+`
+      );
+      return;
+    }
+    process.stderr.write(
+      `[r5d-worker] workspace synchronization failed without a recoverable hydration basis; bouncing the worker socket for in-process recovery (attempt ${attempt} of ${WORKSPACE_UNSAFE_SYNC_MAX_RECOVERY_ATTEMPTS}): ${cause}
+`
+    );
+    const timer = setTimeout(() => {
+      if (!unsafeWorkspaceSyncBouncePending) return;
+      if (currentWorkerSocket === ws && (ws.readyState === WebSocket.OPEN || ws.readyState === WebSocket.CONNECTING)) {
+        ws.close(WORKSPACE_UNSAFE_SYNC_RECOVERY_CLOSE_CODE, "Workspace synchronization failed without a recoverable hydration basis");
+      }
+    }, unsafeWorkspaceSyncRecoveryBounceDelayMs(attempt));
+    timer.unref();
+  };
   const runWorkspaceSync = async (input) => {
     const attemptId = input.attemptId ?? crypto.randomUUID();
     const requestedAt = Date.now();
@@ -4294,7 +4342,7 @@ async function startWorker(options) {
     const admittedDeferredIncidentId = deferredWorkspaceConfiguration?.incidentId ?? null;
     const assertStillAdmitted = () => {
       if (admittedGeneration !== workerAdmissionGeneration || admittedActiveIncidentId !== activeWorkspaceIncidentId || admittedDeferredIncidentId !== (deferredWorkspaceConfiguration?.incidentId ?? null) || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN || !deferredWorkspaceSyncTriggerIsAllowed(deferredWorkspaceConfiguration, activeWorkspaceIncidentId, input.trigger)) {
-        throw new Error("Workspace synchronization admission changed while asynchronous work was in flight");
+        throw new WorkspaceSyncAdmissionChangedError("Workspace synchronization admission changed while asynchronous work was in flight");
       }
     };
     if (!deferredWorkspaceSyncTriggerIsAllowed(deferredWorkspaceConfiguration, activeWorkspaceIncidentId, input.trigger)) {
@@ -4346,31 +4394,62 @@ async function startWorker(options) {
             assertStillAdmitted
           });
         } catch (error) {
-          const hydrationCurrent = workspaceSyncFailureHydrationIsSafe({
+          const errorMessage = error instanceof Error ? error.message : String(error);
+          const hydrationRecoverable = workspaceSyncFailureHydrationIsSafe({
             error,
             resetToCanonical: input.resetToCanonical === true,
-            inspectCurrentHydration: () => (0, import_workspace_git_sync.workspaceGitHydrationIsCurrent)(workspaceShadowRoot, buildWorkspaceMounts())
+            inspectRecoverableHydration: () => (0, import_workspace_git_sync.workspaceGitHydrationIsRecoverable)(workspaceShadowRoot, buildWorkspaceMounts())
           });
-          if (!hydrationCurrent) {
-            process.stderr.write(
-              `[r5d-worker] workspace synchronization failed with an incompletely hydrated visible tree; exiting for exact recovery: ${error instanceof Error ? error.message : String(error)}
+          if (hydrationRecoverable) {
+            if (error instanceof WorkspaceSyncAdmissionChangedError) {
+              process.stderr.write(
+                `[r5d-worker] workspace synchronization was cancelled with every mount at a recorded basis; the next cycle converges it: ${errorMessage}
 `
-            );
+              );
+            }
+            return await failedWorkspaceSyncResult(attemptId, input.trigger, error);
           }
-          fenceUnsafeWorkspaceSyncFailure({
-            hydrationCurrent,
+          const fenced = fenceUnsafeWorkspaceSyncFailure({
+            hydrationRecoverable,
+            resetToCanonical: input.resetToCanonical === true,
+            recoveryAttempts: unsafeWorkspaceSyncRecoveryAttempts,
+            maxRecoveryAttempts: WORKSPACE_UNSAFE_SYNC_MAX_RECOVERY_ATTEMPTS,
+            hasRetainableWork: activeProcesses.size > 0 || credentialBearingProcessGroups.size > 0 || activePtys.size > 0 || pendingProcessTerminals.size > 0,
+            bouncePending: unsafeWorkspaceSyncBouncePending,
             invalidateExecution: () => {
               workspaceConfigured = false;
               advanceWorkerAdmissionGeneration();
             },
+            scheduleRecoveryBounce: () => {
+              unsafeWorkspaceSyncRecoveryAttempts += 1;
+              scheduleUnsafeWorkspaceSyncRecoveryBounce(errorMessage);
+            },
+            fatalExit: (exitCode) => {
+              process.stderr.write(
+                `[r5d-worker] workspace synchronization failed without a recoverable hydration basis; exiting for exact recovery after ${unsafeWorkspaceSyncRecoveryAttempts} in-process recovery attempt(s): ${errorMessage}
+`
+              );
+              process.exit(exitCode);
+            },
             exitCode: 1
           });
-          return await failedWorkspaceSyncResult(attemptId, input.trigger, error);
+          if (fenced === "recovery_pending") {
+            process.stderr.write(
+              `[r5d-worker] workspace synchronization failed without a recoverable hydration basis while in-process recovery is already pending: ${errorMessage}
+`
+            );
+          }
+          return {
+            ...await failedWorkspaceSyncResult(attemptId, input.trigger, error),
+            failureReason: "unsafe_hydration",
+            recoveryAttempt: unsafeWorkspaceSyncRecoveryAttempts
+          };
         }
       });
     } finally {
       workspaceSyncRequestsInFlight -= 1;
     }
+    if (result.outcome !== "failed") unsafeWorkspaceSyncRecoveryAttempts = 0;
     const finishedAt = Date.now();
     result = {
       ...result,
@@ -4421,7 +4500,6 @@ async function startWorker(options) {
           );
         }).finally(() => {
           automaticSyncInFlight = false;
-          heartbeatBusyGrace = (0, import_heartbeat.grantWorkerHeartbeatBusyGrace)(lastServerHeartbeatAt, heartbeatBusyGrace);
           if (pendingAutomaticTrigger && automaticWorkspaceSyncIsCurrentlyAdmitted()) {
             scheduleAutomaticWorkspaceSync(pendingAutomaticTrigger, WORKSPACE_GIT_QUIET_MS);
           }
@@ -4825,7 +4903,9 @@ async function startWorker(options) {
         const configurationSyncAdmissionGeneration = workerAdmissionGeneration;
         const assertConfigurationSyncStillAdmitted = () => {
           if (receiptGeneration !== workspaceConfigurationReceiptGeneration || configurationSyncAdmissionGeneration !== workerAdmissionGeneration || !workspaceConfigurationIncidentSnapshotIsCurrent(incidentDeferral.incidentId, activeWorkspaceIncidentId) || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
-            throw new Error("Workspace configuration synchronization admission changed while asynchronous work was in flight");
+            throw new WorkspaceSyncAdmissionChangedError(
+              "Workspace configuration synchronization admission changed while asynchronous work was in flight"
+            );
           }
         };
         const result = await performWorkspaceSync({
@@ -4918,10 +4998,6 @@ async function startWorker(options) {
       Authorization: `Bearer ${token}`
     }
   });
-  let lastServerHeartbeatAt = null;
-  let heartbeatBusyGrace = null;
-  let heartbeatTimedOut = false;
-  let heartbeatWatchdog;
   const handleGracefulShutdown = () => {
     if (shutdownAfterClose) return;
     shutdownAfterClose = true;
@@ -4949,23 +5025,8 @@ async function startWorker(options) {
   };
   process.on("SIGTERM", handleGracefulShutdown);
   process.on("SIGINT", handleGracefulShutdown);
-  const stopHeartbeatWatchdog = () => {
-    if (heartbeatWatchdog) {
-      clearInterval(heartbeatWatchdog);
-      heartbeatWatchdog = void 0;
-    }
-  };
   ws.addEventListener("open", () => {
     currentWorkerSocket = ws;
-    heartbeatWatchdog = setInterval(() => {
-      if (heartbeatTimedOut || !(0, import_heartbeat.hasWorkerHeartbeatTimedOutWithBusyGrace)(lastServerHeartbeatAt, heartbeatBusyGrace)) {
-        return;
-      }
-      heartbeatTimedOut = true;
-      stopHeartbeatWatchdog();
-      process.stderr.write("[r5d-worker] server heartbeat timed out; reconnecting\n");
-      ws.terminate();
-    }, import_heartbeat.WORKER_HEARTBEAT_INTERVAL_MS);
     const hello = {
       type: "hello",
       hostInfo: {
@@ -5012,7 +5073,7 @@ async function startWorker(options) {
         sourceSocketIsCurrent: currentWorkerSocket === ws,
         sourceSocketOpen: ws.readyState === WebSocket.OPEN,
         workspaceConfigured,
-        runtimeHealthy: !shutdownAfterClose && !heartbeatTimedOut
+        runtimeHealthy: !shutdownAfterClose
       });
       if (message.type === "connected") {
         return;
@@ -5022,6 +5083,8 @@ async function startWorker(options) {
         const refreshesDeferredConfiguration = deferredWorkspaceConfiguration !== null && activeWorkspaceIncidentId === null && message.deferWorkspaceSyncForIncidentId === void 0;
         advanceWorkerAdmissionGeneration();
         const configured = await configureWorkerWorkspace(message, receiptGeneration);
+        unsafeWorkspaceSyncBouncePending = false;
+        if (configured.result.outcome !== "failed") unsafeWorkspaceSyncRecoveryAttempts = 0;
         sendWorkerMessageFromCurrentSource(ws, {
           type: "workspace_configured",
           requestId: message.requestId,
@@ -5369,8 +5432,6 @@ async function startWorker(options) {
         return;
       }
       if (message.type === "ping") {
-        lastServerHeartbeatAt = Date.now();
-        heartbeatBusyGrace = null;
         ws.send(JSON.stringify({ type: "pong" }));
         return;
       }
@@ -5784,15 +5845,12 @@ async function startWorker(options) {
     })().catch((error) => {
       process.stderr.write(`[r5d-worker] message handling failed: ${error instanceof Error ? error.message : String(error)}
 `);
-    }).finally(() => {
-      heartbeatBusyGrace = (0, import_heartbeat.grantWorkerHeartbeatBusyGrace)(lastServerHeartbeatAt, heartbeatBusyGrace);
     });
   });
   ws.addEventListener("close", (event) => {
     const processEvents = process;
     processEvents.off("SIGTERM", handleGracefulShutdown);
     processEvents.off("SIGINT", handleGracefulShutdown);
-    stopHeartbeatWatchdog();
     if (workspaceAutomaticTimer) {
       clearTimeout(workspaceAutomaticTimer);
       workspaceAutomaticTimer = void 0;

package/dist/cjs/package.json

@@ -1,5 +1,5 @@
 {
   "name": "@ricsam/r5d-worker",
-  "version": "0.0.88",
+  "version": "0.0.90",
   "type": "commonjs"
 }

package/dist/cjs/workspace-git-sync.cjs

@@ -42,6 +42,7 @@ __export(workspace_git_sync_exports, {
   resetWorkspaceGit: () => resetWorkspaceGit,
   synchronizeWorkspaceGit: () => synchronizeWorkspaceGit,
   workspaceGitHydrationIsCurrent: () => workspaceGitHydrationIsCurrent,
+  workspaceGitHydrationIsRecoverable: () => workspaceGitHydrationIsRecoverable,
   workspaceGitMountsForRemediation: () => workspaceGitMountsForRemediation,
   workspaceGitSyncTestHarness: () => workspaceGitSyncTestHarness
 });
@@ -420,18 +421,25 @@ function hydratedWorkspaceReceiptFileIsV3(workspacePath) {
   }
 }
 function workspaceGitHydrationIsCurrent(workspacePath, mounts) {
+  return workspaceGitHydrationHolds(workspacePath, mounts, false);
+}
+function workspaceGitHydrationIsRecoverable(workspacePath, mounts) {
+  return workspaceGitHydrationHolds(workspacePath, mounts, true);
+}
+function workspaceGitHydrationHolds(workspacePath, mounts, tolerateIdleRecordedBasis) {
   const resolvedWorkspacePath = import_node_path.default.resolve(workspacePath);
   if (!import_node_fs.default.existsSync(import_node_path.default.join(resolvedWorkspacePath, ".git"))) return true;
   if (mounts) validateMounts(resolvedWorkspacePath, mounts);
   const head = revParse(resolvedWorkspacePath, "HEAD");
   if (!workspaceCheckoutDurabilityIsCurrent(resolvedWorkspacePath)) return false;
+  if (tolerateIdleRecordedBasis && lstatIfExists(hydrationTransactionPath(resolvedWorkspacePath)) !== null) return false;
   const receipt = readHydratedWorkspaceReceipt(resolvedWorkspacePath);
   if (!workspaceBasisRefsAreCurrent(resolvedWorkspacePath, receipt ?? { version: 3, mounts: [] })) return false;
   if (head === null) return true;
   if (!receipt) return false;
   return mounts ? configuredWorkspaceHydrationBasisMounts(resolvedWorkspacePath, mounts).every(
-    (mount) => workspaceHydrationReceiptCoversMount(receipt, head, mount) || Boolean(mount.busy?.() && workspaceHydrationReceiptMountBasisHead(receipt, mount))
-  ) : Boolean(receipt && receipt.mounts.every((mount) => mount.head === head));
+    (mount) => workspaceHydrationReceiptCoversMount(receipt, head, mount) || Boolean((tolerateIdleRecordedBasis || mount.busy?.()) && workspaceHydrationReceiptMountBasisHead(receipt, mount))
+  ) : Boolean(receipt && receipt.mounts.every((mount) => tolerateIdleRecordedBasis || mount.head === head));
 }
 const WORKSPACE_GIT_BASIS_REF_PREFIX = "refs/r5d/workspace-basis/";
 const WORKSPACE_GIT_BASIS_STAGING_REF_PREFIX = "refs/r5d/workspace-basis-staging/";
@@ -2190,6 +2198,7 @@ const workspaceGitSyncTestHarness = {
   resetWorkspaceGit,
   synchronizeWorkspaceGit,
   workspaceGitHydrationIsCurrent,
+  workspaceGitHydrationIsRecoverable,
   workspaceGitMountsForRemediation,
   workspaceGitSyncTestHarness
 });

package/dist/mjs/main.mjs

@@ -7,11 +7,6 @@ import { spawn as spawnChildProcess } from "node:child_process";
 import { Database } from "bun:sqlite";
 import { installCliUpdate, readInstalledCliVersion } from "./cli-update.mjs";
 import { gitTransportSecurityArgs, workerGitProcessEnvironment } from "./git-process-environment.mjs";
-import {
-  grantWorkerHeartbeatBusyGrace,
-  hasWorkerHeartbeatTimedOutWithBusyGrace,
-  WORKER_HEARTBEAT_INTERVAL_MS
-} from "./heartbeat.mjs";
 import { terminateProcessTree } from "./process-tree.mjs";
 import { createPtyOutputCoalescer } from "./pty-output-coalescer.mjs";
 import { openWorkerPortForwardRelay } from "./port-forward-client.mjs";
@@ -88,7 +83,7 @@ import {
   recoverWorkspaceGitHydration,
   resetWorkspaceGit,
   synchronizeWorkspaceGit,
-  workspaceGitHydrationIsCurrent,
+  workspaceGitHydrationIsRecoverable,
   workspaceGitMountsForRemediation,
   WorkspaceRemediationAncestryError
 } from "./workspace-git-sync.mjs";
@@ -158,6 +153,11 @@ class StaleWorkerAdmissionError extends Error {
     this.name = "StaleWorkerAdmissionError";
   }
 }
+class WorkspaceSyncAdmissionChangedError extends Error {
+  name = "WorkspaceSyncAdmissionChangedError";
+}
+let unsafeWorkspaceSyncRecoveryAttempts = 0;
+let unsafeWorkspaceSyncBouncePending = false;
 function advanceWorkerAdmissionGeneration() {
   workerAdmissionGeneration += 1;
   return workerAdmissionGeneration;
@@ -1641,16 +1641,33 @@ function workspaceSyncFailureHydrationIsSafe(input) {
   if (input.error instanceof WorkspaceRemediationAncestryError) return true;
   if (input.resetToCanonical) return false;
   try {
-    return input.inspectCurrentHydration();
+    return input.inspectRecoverableHydration();
   } catch {
     return false;
   }
 }
+const WORKSPACE_UNSAFE_SYNC_MAX_RECOVERY_ATTEMPTS = 3;
+const WORKSPACE_UNSAFE_SYNC_RECOVERY_CLOSE_CODE = 4008;
+function unsafeWorkspaceSyncRecoveryBounceDelayMs(attempt) {
+  return Math.min(1e3 * 2 ** Math.max(0, attempt - 1), 3e4);
+}
+function unsafeWorkspaceSyncFailureResponse(input) {
+  if (input.resetToCanonical) return "fatal_exit";
+  if (!input.hasRetainableWork) return "fatal_exit";
+  if (input.recoveryAttempts >= input.maxRecoveryAttempts) return "fatal_exit";
+  if (input.bouncePending) return "recovery_pending";
+  return "recovery_bounce";
+}
 function fenceUnsafeWorkspaceSyncFailure(input) {
-  if (input.hydrationCurrent) return false;
+  if (input.hydrationRecoverable) return "none";
   input.invalidateExecution();
-  requireCredentialGenerationRestart(true, input.fatalExit, input.exitCode);
-  return true;
+  const response = unsafeWorkspaceSyncFailureResponse(input);
+  if (response === "fatal_exit") {
+    requireCredentialGenerationRestart(true, input.fatalExit, input.exitCode);
+    throw new Error("Unsafe workspace synchronization fence continued past its fatal exit");
+  }
+  if (response === "recovery_bounce") input.scheduleRecoveryBounce();
+  return response;
 }
 const workerGitSecurityTestHarness = {
   commandArgs: workerGitCommand.commandArgs,
@@ -1678,6 +1695,11 @@ const workerGitSecurityTestHarness = {
   credentialReapContractStatus,
   verifiedCredentialReapContract,
   fenceUnsafeWorkspaceSyncFailure,
+  unsafeWorkspaceSyncFailureResponse,
+  unsafeWorkspaceSyncRecoveryBounceDelayMs,
+  WORKSPACE_UNSAFE_SYNC_MAX_RECOVERY_ATTEMPTS,
+  WORKSPACE_UNSAFE_SYNC_RECOVERY_CLOSE_CODE,
+  WorkspaceSyncAdmissionChangedError,
   workspaceConfigurationIncidentDeferral,
   workspaceIncidentTerminalClearDisposition,
   workspaceIncidentTerminalRefreshFence,
@@ -4294,6 +4316,28 @@ async function startWorker(options) {
       );
     }
   };
+  const scheduleUnsafeWorkspaceSyncRecoveryBounce = (cause) => {
+    unsafeWorkspaceSyncBouncePending = true;
+    const attempt = unsafeWorkspaceSyncRecoveryAttempts;
+    if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
+      process.stderr.write(
+        `[r5d-worker] workspace synchronization failed without a recoverable hydration basis; awaiting reconnect recovery (attempt ${attempt} of ${WORKSPACE_UNSAFE_SYNC_MAX_RECOVERY_ATTEMPTS}): ${cause}
+`
+      );
+      return;
+    }
+    process.stderr.write(
+      `[r5d-worker] workspace synchronization failed without a recoverable hydration basis; bouncing the worker socket for in-process recovery (attempt ${attempt} of ${WORKSPACE_UNSAFE_SYNC_MAX_RECOVERY_ATTEMPTS}): ${cause}
+`
+    );
+    const timer = setTimeout(() => {
+      if (!unsafeWorkspaceSyncBouncePending) return;
+      if (currentWorkerSocket === ws && (ws.readyState === WebSocket.OPEN || ws.readyState === WebSocket.CONNECTING)) {
+        ws.close(WORKSPACE_UNSAFE_SYNC_RECOVERY_CLOSE_CODE, "Workspace synchronization failed without a recoverable hydration basis");
+      }
+    }, unsafeWorkspaceSyncRecoveryBounceDelayMs(attempt));
+    timer.unref();
+  };
   const runWorkspaceSync = async (input) => {
     const attemptId = input.attemptId ?? crypto.randomUUID();
     const requestedAt = Date.now();
@@ -4302,7 +4346,7 @@ async function startWorker(options) {
     const admittedDeferredIncidentId = deferredWorkspaceConfiguration?.incidentId ?? null;
     const assertStillAdmitted = () => {
       if (admittedGeneration !== workerAdmissionGeneration || admittedActiveIncidentId !== activeWorkspaceIncidentId || admittedDeferredIncidentId !== (deferredWorkspaceConfiguration?.incidentId ?? null) || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN || !deferredWorkspaceSyncTriggerIsAllowed(deferredWorkspaceConfiguration, activeWorkspaceIncidentId, input.trigger)) {
-        throw new Error("Workspace synchronization admission changed while asynchronous work was in flight");
+        throw new WorkspaceSyncAdmissionChangedError("Workspace synchronization admission changed while asynchronous work was in flight");
       }
     };
     if (!deferredWorkspaceSyncTriggerIsAllowed(deferredWorkspaceConfiguration, activeWorkspaceIncidentId, input.trigger)) {
@@ -4354,31 +4398,62 @@ async function startWorker(options) {
             assertStillAdmitted
           });
         } catch (error) {
-          const hydrationCurrent = workspaceSyncFailureHydrationIsSafe({
+          const errorMessage = error instanceof Error ? error.message : String(error);
+          const hydrationRecoverable = workspaceSyncFailureHydrationIsSafe({
             error,
             resetToCanonical: input.resetToCanonical === true,
-            inspectCurrentHydration: () => workspaceGitHydrationIsCurrent(workspaceShadowRoot, buildWorkspaceMounts())
+            inspectRecoverableHydration: () => workspaceGitHydrationIsRecoverable(workspaceShadowRoot, buildWorkspaceMounts())
           });
-          if (!hydrationCurrent) {
-            process.stderr.write(
-              `[r5d-worker] workspace synchronization failed with an incompletely hydrated visible tree; exiting for exact recovery: ${error instanceof Error ? error.message : String(error)}
+          if (hydrationRecoverable) {
+            if (error instanceof WorkspaceSyncAdmissionChangedError) {
+              process.stderr.write(
+                `[r5d-worker] workspace synchronization was cancelled with every mount at a recorded basis; the next cycle converges it: ${errorMessage}
 `
-            );
+              );
+            }
+            return await failedWorkspaceSyncResult(attemptId, input.trigger, error);
           }
-          fenceUnsafeWorkspaceSyncFailure({
-            hydrationCurrent,
+          const fenced = fenceUnsafeWorkspaceSyncFailure({
+            hydrationRecoverable,
+            resetToCanonical: input.resetToCanonical === true,
+            recoveryAttempts: unsafeWorkspaceSyncRecoveryAttempts,
+            maxRecoveryAttempts: WORKSPACE_UNSAFE_SYNC_MAX_RECOVERY_ATTEMPTS,
+            hasRetainableWork: activeProcesses.size > 0 || credentialBearingProcessGroups.size > 0 || activePtys.size > 0 || pendingProcessTerminals.size > 0,
+            bouncePending: unsafeWorkspaceSyncBouncePending,
             invalidateExecution: () => {
               workspaceConfigured = false;
               advanceWorkerAdmissionGeneration();
             },
+            scheduleRecoveryBounce: () => {
+              unsafeWorkspaceSyncRecoveryAttempts += 1;
+              scheduleUnsafeWorkspaceSyncRecoveryBounce(errorMessage);
+            },
+            fatalExit: (exitCode) => {
+              process.stderr.write(
+                `[r5d-worker] workspace synchronization failed without a recoverable hydration basis; exiting for exact recovery after ${unsafeWorkspaceSyncRecoveryAttempts} in-process recovery attempt(s): ${errorMessage}
+`
+              );
+              process.exit(exitCode);
+            },
             exitCode: 1
           });
-          return await failedWorkspaceSyncResult(attemptId, input.trigger, error);
+          if (fenced === "recovery_pending") {
+            process.stderr.write(
+              `[r5d-worker] workspace synchronization failed without a recoverable hydration basis while in-process recovery is already pending: ${errorMessage}
+`
+            );
+          }
+          return {
+            ...await failedWorkspaceSyncResult(attemptId, input.trigger, error),
+            failureReason: "unsafe_hydration",
+            recoveryAttempt: unsafeWorkspaceSyncRecoveryAttempts
+          };
         }
       });
     } finally {
       workspaceSyncRequestsInFlight -= 1;
     }
+    if (result.outcome !== "failed") unsafeWorkspaceSyncRecoveryAttempts = 0;
     const finishedAt = Date.now();
     result = {
       ...result,
@@ -4429,7 +4504,6 @@ async function startWorker(options) {
           );
         }).finally(() => {
           automaticSyncInFlight = false;
-          heartbeatBusyGrace = grantWorkerHeartbeatBusyGrace(lastServerHeartbeatAt, heartbeatBusyGrace);
           if (pendingAutomaticTrigger && automaticWorkspaceSyncIsCurrentlyAdmitted()) {
             scheduleAutomaticWorkspaceSync(pendingAutomaticTrigger, WORKSPACE_GIT_QUIET_MS);
           }
@@ -4833,7 +4907,9 @@ async function startWorker(options) {
         const configurationSyncAdmissionGeneration = workerAdmissionGeneration;
         const assertConfigurationSyncStillAdmitted = () => {
           if (receiptGeneration !== workspaceConfigurationReceiptGeneration || configurationSyncAdmissionGeneration !== workerAdmissionGeneration || !workspaceConfigurationIncidentSnapshotIsCurrent(incidentDeferral.incidentId, activeWorkspaceIncidentId) || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
-            throw new Error("Workspace configuration synchronization admission changed while asynchronous work was in flight");
+            throw new WorkspaceSyncAdmissionChangedError(
+              "Workspace configuration synchronization admission changed while asynchronous work was in flight"
+            );
           }
         };
         const result = await performWorkspaceSync({
@@ -4926,10 +5002,6 @@ async function startWorker(options) {
       Authorization: `Bearer ${token}`
     }
   });
-  let lastServerHeartbeatAt = null;
-  let heartbeatBusyGrace = null;
-  let heartbeatTimedOut = false;
-  let heartbeatWatchdog;
   const handleGracefulShutdown = () => {
     if (shutdownAfterClose) return;
     shutdownAfterClose = true;
@@ -4957,23 +5029,8 @@ async function startWorker(options) {
   };
   process.on("SIGTERM", handleGracefulShutdown);
   process.on("SIGINT", handleGracefulShutdown);
-  const stopHeartbeatWatchdog = () => {
-    if (heartbeatWatchdog) {
-      clearInterval(heartbeatWatchdog);
-      heartbeatWatchdog = void 0;
-    }
-  };
   ws.addEventListener("open", () => {
     currentWorkerSocket = ws;
-    heartbeatWatchdog = setInterval(() => {
-      if (heartbeatTimedOut || !hasWorkerHeartbeatTimedOutWithBusyGrace(lastServerHeartbeatAt, heartbeatBusyGrace)) {
-        return;
-      }
-      heartbeatTimedOut = true;
-      stopHeartbeatWatchdog();
-      process.stderr.write("[r5d-worker] server heartbeat timed out; reconnecting\n");
-      ws.terminate();
-    }, WORKER_HEARTBEAT_INTERVAL_MS);
     const hello = {
       type: "hello",
       hostInfo: {
@@ -5020,7 +5077,7 @@ async function startWorker(options) {
         sourceSocketIsCurrent: currentWorkerSocket === ws,
         sourceSocketOpen: ws.readyState === WebSocket.OPEN,
         workspaceConfigured,
-        runtimeHealthy: !shutdownAfterClose && !heartbeatTimedOut
+        runtimeHealthy: !shutdownAfterClose
       });
       if (message.type === "connected") {
         return;
@@ -5030,6 +5087,8 @@ async function startWorker(options) {
         const refreshesDeferredConfiguration = deferredWorkspaceConfiguration !== null && activeWorkspaceIncidentId === null && message.deferWorkspaceSyncForIncidentId === void 0;
         advanceWorkerAdmissionGeneration();
         const configured = await configureWorkerWorkspace(message, receiptGeneration);
+        unsafeWorkspaceSyncBouncePending = false;
+        if (configured.result.outcome !== "failed") unsafeWorkspaceSyncRecoveryAttempts = 0;
         sendWorkerMessageFromCurrentSource(ws, {
           type: "workspace_configured",
           requestId: message.requestId,
@@ -5377,8 +5436,6 @@ async function startWorker(options) {
         return;
       }
       if (message.type === "ping") {
-        lastServerHeartbeatAt = Date.now();
-        heartbeatBusyGrace = null;
         ws.send(JSON.stringify({ type: "pong" }));
         return;
       }
@@ -5792,15 +5849,12 @@ async function startWorker(options) {
     })().catch((error) => {
       process.stderr.write(`[r5d-worker] message handling failed: ${error instanceof Error ? error.message : String(error)}
 `);
-    }).finally(() => {
-      heartbeatBusyGrace = grantWorkerHeartbeatBusyGrace(lastServerHeartbeatAt, heartbeatBusyGrace);
     });
   });
   ws.addEventListener("close", (event) => {
     const processEvents = process;
     processEvents.off("SIGTERM", handleGracefulShutdown);
     processEvents.off("SIGINT", handleGracefulShutdown);
-    stopHeartbeatWatchdog();
     if (workspaceAutomaticTimer) {
       clearTimeout(workspaceAutomaticTimer);
       workspaceAutomaticTimer = void 0;

package/dist/mjs/package.json

@@ -1,5 +1,5 @@
 {
   "name": "@ricsam/r5d-worker",
-  "version": "0.0.88",
+  "version": "0.0.90",
   "type": "module"
 }

package/dist/mjs/workspace-git-sync.mjs

@@ -376,18 +376,25 @@ function hydratedWorkspaceReceiptFileIsV3(workspacePath) {
   }
 }
 function workspaceGitHydrationIsCurrent(workspacePath, mounts) {
+  return workspaceGitHydrationHolds(workspacePath, mounts, false);
+}
+function workspaceGitHydrationIsRecoverable(workspacePath, mounts) {
+  return workspaceGitHydrationHolds(workspacePath, mounts, true);
+}
+function workspaceGitHydrationHolds(workspacePath, mounts, tolerateIdleRecordedBasis) {
   const resolvedWorkspacePath = path.resolve(workspacePath);
   if (!fs.existsSync(path.join(resolvedWorkspacePath, ".git"))) return true;
   if (mounts) validateMounts(resolvedWorkspacePath, mounts);
   const head = revParse(resolvedWorkspacePath, "HEAD");
   if (!workspaceCheckoutDurabilityIsCurrent(resolvedWorkspacePath)) return false;
+  if (tolerateIdleRecordedBasis && lstatIfExists(hydrationTransactionPath(resolvedWorkspacePath)) !== null) return false;
   const receipt = readHydratedWorkspaceReceipt(resolvedWorkspacePath);
   if (!workspaceBasisRefsAreCurrent(resolvedWorkspacePath, receipt ?? { version: 3, mounts: [] })) return false;
   if (head === null) return true;
   if (!receipt) return false;
   return mounts ? configuredWorkspaceHydrationBasisMounts(resolvedWorkspacePath, mounts).every(
-    (mount) => workspaceHydrationReceiptCoversMount(receipt, head, mount) || Boolean(mount.busy?.() && workspaceHydrationReceiptMountBasisHead(receipt, mount))
-  ) : Boolean(receipt && receipt.mounts.every((mount) => mount.head === head));
+    (mount) => workspaceHydrationReceiptCoversMount(receipt, head, mount) || Boolean((tolerateIdleRecordedBasis || mount.busy?.()) && workspaceHydrationReceiptMountBasisHead(receipt, mount))
+  ) : Boolean(receipt && receipt.mounts.every((mount) => tolerateIdleRecordedBasis || mount.head === head));
 }
 const WORKSPACE_GIT_BASIS_REF_PREFIX = "refs/r5d/workspace-basis/";
 const WORKSPACE_GIT_BASIS_STAGING_REF_PREFIX = "refs/r5d/workspace-basis-staging/";
@@ -2145,6 +2152,7 @@ export {
   resetWorkspaceGit,
   synchronizeWorkspaceGit,
   workspaceGitHydrationIsCurrent,
+  workspaceGitHydrationIsRecoverable,
   workspaceGitMountsForRemediation,
   workspaceGitSyncTestHarness
 };

package/dist/types/main.d.ts

@@ -69,6 +69,10 @@ type WorkspaceSyncResult = {
         synchronizeMs: number;
     };
     error?: string;
+    /** Typed reason for a failed outcome the server should surface loudly. */
+    failureReason?: "unsafe_hydration";
+    /** Consecutive in-process recovery bounces taken for unsafe hydration failures. */
+    recoveryAttempt?: number;
 };
 export type WorkerSessionTarget = {
     type: "project";
@@ -356,6 +360,9 @@ type WorkerCodeReadResult = {
     mtime: string;
     base64: string;
 };
+declare class WorkspaceSyncAdmissionChangedError extends Error {
+    readonly name = "WorkspaceSyncAdmissionChangedError";
+}
 declare function assertWorkerChildAdmission(input: {
     capturedGeneration: number;
     currentGeneration: number;
@@ -609,14 +616,29 @@ declare function deferredIncidentWorkspaceConfigurationResult(input: {
 declare function workspaceSyncFailureHydrationIsSafe(input: {
     error: unknown;
     resetToCanonical: boolean;
-    inspectCurrentHydration: () => boolean;
+    inspectRecoverableHydration: () => boolean;
 }): boolean;
+declare function unsafeWorkspaceSyncRecoveryBounceDelayMs(attempt: number): number;
+type UnsafeWorkspaceSyncFailureResponse = "fatal_exit" | "recovery_bounce" | "recovery_pending";
+declare function unsafeWorkspaceSyncFailureResponse(input: {
+    resetToCanonical: boolean;
+    recoveryAttempts: number;
+    maxRecoveryAttempts: number;
+    hasRetainableWork: boolean;
+    bouncePending: boolean;
+}): UnsafeWorkspaceSyncFailureResponse;
 declare function fenceUnsafeWorkspaceSyncFailure(input: {
-    hydrationCurrent: boolean;
+    hydrationRecoverable: boolean;
+    resetToCanonical: boolean;
+    recoveryAttempts: number;
+    maxRecoveryAttempts: number;
+    hasRetainableWork: boolean;
+    bouncePending: boolean;
     invalidateExecution: () => void;
+    scheduleRecoveryBounce: () => void;
     fatalExit?: (exitCode: number) => void;
     exitCode?: number;
-}): boolean;
+}): "none" | "recovery_bounce" | "recovery_pending";
 export declare const workerGitSecurityTestHarness: {
     commandArgs: (args: string[]) => string[];
     credentialHelperForRemotes: typeof credentialHelperForRemotes;
@@ -641,6 +663,11 @@ export declare const workerGitSecurityTestHarness: {
     credentialReapContractStatus: typeof credentialReapContractStatus;
     verifiedCredentialReapContract: typeof verifiedCredentialReapContract;
     fenceUnsafeWorkspaceSyncFailure: typeof fenceUnsafeWorkspaceSyncFailure;
+    unsafeWorkspaceSyncFailureResponse: typeof unsafeWorkspaceSyncFailureResponse;
+    unsafeWorkspaceSyncRecoveryBounceDelayMs: typeof unsafeWorkspaceSyncRecoveryBounceDelayMs;
+    WORKSPACE_UNSAFE_SYNC_MAX_RECOVERY_ATTEMPTS: number;
+    WORKSPACE_UNSAFE_SYNC_RECOVERY_CLOSE_CODE: number;
+    WorkspaceSyncAdmissionChangedError: typeof WorkspaceSyncAdmissionChangedError;
     workspaceConfigurationIncidentDeferral: typeof workspaceConfigurationIncidentDeferral;
     workspaceIncidentTerminalClearDisposition: typeof workspaceIncidentTerminalClearDisposition;
     workspaceIncidentTerminalRefreshFence: typeof workspaceIncidentTerminalRefreshFence;

package/dist/types/workspace-git-sync.d.ts

@@ -71,6 +71,16 @@ export declare class WorkspaceRemediationAncestryError extends Error {
 declare function gitCommandArgs(args: string[]): string[];
 declare function workspaceCloneCommandArgs(args: string[], remoteUrl: string, credentialHelper?: string | null, credentialUsername?: string | null): string[];
 export declare function workspaceGitHydrationIsCurrent(workspacePath: string, mounts?: readonly WorkspaceGitMount[]): boolean;
+/**
+ * Failure-time variant of the currency check. A mount pinned at a valid
+ * recorded basis is convergent whether or not it is busy right now: the next
+ * cycle merges it forward from that exact basis and a genuine collision
+ * surfaces as a projection_merge conflict, never a silent overwrite. Live
+ * busy-ness must not decide safety here because the same disconnect that
+ * aborts a sync also clears the process/PTY maps busy-ness is computed from.
+ * Configuration-time recovery keeps the strict predicate.
+ */
+export declare function workspaceGitHydrationIsRecoverable(workspacePath: string, mounts?: readonly WorkspaceGitMount[]): boolean;
 declare function configureWorkspaceRepository(input: {
     workspacePath: string;
     remoteUrl: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ricsam/r5d-worker",
-  "version": "0.0.88",
+  "version": "0.0.90",
   "type": "module",
   "main": "./dist/cjs/main.cjs",
   "module": "./dist/mjs/main.mjs",

package/dist/cjs/heartbeat.cjs

@@ -1,49 +0,0 @@
-"use strict";
-var __defProp = Object.defineProperty;
-var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
-var __getOwnPropNames = Object.getOwnPropertyNames;
-var __hasOwnProp = Object.prototype.hasOwnProperty;
-var __export = (target, all) => {
-  for (var name in all)
-    __defProp(target, name, { get: all[name], enumerable: true });
-};
-var __copyProps = (to, from, except, desc) => {
-  if (from && typeof from === "object" || typeof from === "function") {
-    for (let key of __getOwnPropNames(from))
-      if (!__hasOwnProp.call(to, key) && key !== except)
-        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
-  }
-  return to;
-};
-var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
-var heartbeat_exports = {};
-__export(heartbeat_exports, {
-  WORKER_HEARTBEAT_INTERVAL_MS: () => WORKER_HEARTBEAT_INTERVAL_MS,
-  WORKER_HEARTBEAT_TIMEOUT_MS: () => WORKER_HEARTBEAT_TIMEOUT_MS,
-  grantWorkerHeartbeatBusyGrace: () => grantWorkerHeartbeatBusyGrace,
-  hasWorkerHeartbeatTimedOut: () => hasWorkerHeartbeatTimedOut,
-  hasWorkerHeartbeatTimedOutWithBusyGrace: () => hasWorkerHeartbeatTimedOutWithBusyGrace
-});
-module.exports = __toCommonJS(heartbeat_exports);
-const WORKER_HEARTBEAT_INTERVAL_MS = 15e3;
-const WORKER_HEARTBEAT_TIMEOUT_MS = 45e3;
-function hasWorkerHeartbeatTimedOut(lastHeartbeatAt, now = Date.now(), timeoutMs = WORKER_HEARTBEAT_TIMEOUT_MS) {
-  return lastHeartbeatAt !== null && now - lastHeartbeatAt >= timeoutMs;
-}
-function grantWorkerHeartbeatBusyGrace(lastHeartbeatAt, currentGrace, now = Date.now(), timeoutMs = WORKER_HEARTBEAT_TIMEOUT_MS, graceMs = WORKER_HEARTBEAT_INTERVAL_MS) {
-  if (!hasWorkerHeartbeatTimedOut(lastHeartbeatAt, now, timeoutMs)) return currentGrace;
-  if (currentGrace?.heartbeatAt === lastHeartbeatAt) return currentGrace;
-  return { heartbeatAt: lastHeartbeatAt, expiresAt: now + graceMs };
-}
-function hasWorkerHeartbeatTimedOutWithBusyGrace(lastHeartbeatAt, busyGrace, now = Date.now(), timeoutMs = WORKER_HEARTBEAT_TIMEOUT_MS) {
-  if (lastHeartbeatAt !== null && busyGrace?.heartbeatAt === lastHeartbeatAt && now < busyGrace.expiresAt) return false;
-  return hasWorkerHeartbeatTimedOut(lastHeartbeatAt, now, timeoutMs);
-}
-// Annotate the CommonJS export names for ESM import in node:
-0 && (module.exports = {
-  WORKER_HEARTBEAT_INTERVAL_MS,
-  WORKER_HEARTBEAT_TIMEOUT_MS,
-  grantWorkerHeartbeatBusyGrace,
-  hasWorkerHeartbeatTimedOut,
-  hasWorkerHeartbeatTimedOutWithBusyGrace
-});

package/dist/mjs/heartbeat.mjs

@@ -1,21 +0,0 @@
-const WORKER_HEARTBEAT_INTERVAL_MS = 15e3;
-const WORKER_HEARTBEAT_TIMEOUT_MS = 45e3;
-function hasWorkerHeartbeatTimedOut(lastHeartbeatAt, now = Date.now(), timeoutMs = WORKER_HEARTBEAT_TIMEOUT_MS) {
-  return lastHeartbeatAt !== null && now - lastHeartbeatAt >= timeoutMs;
-}
-function grantWorkerHeartbeatBusyGrace(lastHeartbeatAt, currentGrace, now = Date.now(), timeoutMs = WORKER_HEARTBEAT_TIMEOUT_MS, graceMs = WORKER_HEARTBEAT_INTERVAL_MS) {
-  if (!hasWorkerHeartbeatTimedOut(lastHeartbeatAt, now, timeoutMs)) return currentGrace;
-  if (currentGrace?.heartbeatAt === lastHeartbeatAt) return currentGrace;
-  return { heartbeatAt: lastHeartbeatAt, expiresAt: now + graceMs };
-}
-function hasWorkerHeartbeatTimedOutWithBusyGrace(lastHeartbeatAt, busyGrace, now = Date.now(), timeoutMs = WORKER_HEARTBEAT_TIMEOUT_MS) {
-  if (lastHeartbeatAt !== null && busyGrace?.heartbeatAt === lastHeartbeatAt && now < busyGrace.expiresAt) return false;
-  return hasWorkerHeartbeatTimedOut(lastHeartbeatAt, now, timeoutMs);
-}
-export {
-  WORKER_HEARTBEAT_INTERVAL_MS,
-  WORKER_HEARTBEAT_TIMEOUT_MS,
-  grantWorkerHeartbeatBusyGrace,
-  hasWorkerHeartbeatTimedOut,
-  hasWorkerHeartbeatTimedOutWithBusyGrace
-};

package/dist/types/heartbeat.d.ts

@@ -1,10 +0,0 @@
-export declare const WORKER_HEARTBEAT_INTERVAL_MS = 15000;
-export declare const WORKER_HEARTBEAT_TIMEOUT_MS = 45000;
-export declare function hasWorkerHeartbeatTimedOut(lastHeartbeatAt: number | null, now?: number, timeoutMs?: number): boolean;
-export type WorkerHeartbeatBusyGrace = {
-    heartbeatAt: number;
-    expiresAt: number;
-};
-/** Grant one bounded grace period for one actual server-heartbeat generation. */
-export declare function grantWorkerHeartbeatBusyGrace(lastHeartbeatAt: number | null, currentGrace: WorkerHeartbeatBusyGrace | null, now?: number, timeoutMs?: number, graceMs?: number): WorkerHeartbeatBusyGrace | null;
-export declare function hasWorkerHeartbeatTimedOutWithBusyGrace(lastHeartbeatAt: number | null, busyGrace: WorkerHeartbeatBusyGrace | null, now?: number, timeoutMs?: number): boolean;