@ricsam/r5d-worker 0.0.88 → 0.0.89

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cjs/main.cjs CHANGED
@@ -150,6 +150,11 @@ class StaleWorkerAdmissionError extends Error {
150
150
  this.name = "StaleWorkerAdmissionError";
151
151
  }
152
152
  }
153
+ class WorkspaceSyncAdmissionChangedError extends Error {
154
+ name = "WorkspaceSyncAdmissionChangedError";
155
+ }
156
+ let unsafeWorkspaceSyncRecoveryAttempts = 0;
157
+ let unsafeWorkspaceSyncBouncePending = false;
153
158
  function advanceWorkerAdmissionGeneration() {
154
159
  workerAdmissionGeneration += 1;
155
160
  return workerAdmissionGeneration;
@@ -1633,16 +1638,33 @@ function workspaceSyncFailureHydrationIsSafe(input) {
1633
1638
  if (input.error instanceof import_workspace_git_sync.WorkspaceRemediationAncestryError) return true;
1634
1639
  if (input.resetToCanonical) return false;
1635
1640
  try {
1636
- return input.inspectCurrentHydration();
1641
+ return input.inspectRecoverableHydration();
1637
1642
  } catch {
1638
1643
  return false;
1639
1644
  }
1640
1645
  }
1646
+ const WORKSPACE_UNSAFE_SYNC_MAX_RECOVERY_ATTEMPTS = 3;
1647
+ const WORKSPACE_UNSAFE_SYNC_RECOVERY_CLOSE_CODE = 4008;
1648
+ function unsafeWorkspaceSyncRecoveryBounceDelayMs(attempt) {
1649
+ return Math.min(1e3 * 2 ** Math.max(0, attempt - 1), 3e4);
1650
+ }
1651
+ function unsafeWorkspaceSyncFailureResponse(input) {
1652
+ if (input.resetToCanonical) return "fatal_exit";
1653
+ if (!input.hasRetainableWork) return "fatal_exit";
1654
+ if (input.recoveryAttempts >= input.maxRecoveryAttempts) return "fatal_exit";
1655
+ if (input.bouncePending) return "recovery_pending";
1656
+ return "recovery_bounce";
1657
+ }
1641
1658
  function fenceUnsafeWorkspaceSyncFailure(input) {
1642
- if (input.hydrationCurrent) return false;
1659
+ if (input.hydrationRecoverable) return "none";
1643
1660
  input.invalidateExecution();
1644
- requireCredentialGenerationRestart(true, input.fatalExit, input.exitCode);
1645
- return true;
1661
+ const response = unsafeWorkspaceSyncFailureResponse(input);
1662
+ if (response === "fatal_exit") {
1663
+ requireCredentialGenerationRestart(true, input.fatalExit, input.exitCode);
1664
+ throw new Error("Unsafe workspace synchronization fence continued past its fatal exit");
1665
+ }
1666
+ if (response === "recovery_bounce") input.scheduleRecoveryBounce();
1667
+ return response;
1646
1668
  }
1647
1669
  const workerGitSecurityTestHarness = {
1648
1670
  commandArgs: workerGitCommand.commandArgs,
@@ -1670,6 +1692,11 @@ const workerGitSecurityTestHarness = {
1670
1692
  credentialReapContractStatus,
1671
1693
  verifiedCredentialReapContract,
1672
1694
  fenceUnsafeWorkspaceSyncFailure,
1695
+ unsafeWorkspaceSyncFailureResponse,
1696
+ unsafeWorkspaceSyncRecoveryBounceDelayMs,
1697
+ WORKSPACE_UNSAFE_SYNC_MAX_RECOVERY_ATTEMPTS,
1698
+ WORKSPACE_UNSAFE_SYNC_RECOVERY_CLOSE_CODE,
1699
+ WorkspaceSyncAdmissionChangedError,
1673
1700
  workspaceConfigurationIncidentDeferral,
1674
1701
  workspaceIncidentTerminalClearDisposition,
1675
1702
  workspaceIncidentTerminalRefreshFence,
@@ -4286,6 +4313,28 @@ async function startWorker(options) {
4286
4313
  );
4287
4314
  }
4288
4315
  };
4316
+ const scheduleUnsafeWorkspaceSyncRecoveryBounce = (cause) => {
4317
+ unsafeWorkspaceSyncBouncePending = true;
4318
+ const attempt = unsafeWorkspaceSyncRecoveryAttempts;
4319
+ if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
4320
+ process.stderr.write(
4321
+ `[r5d-worker] workspace synchronization failed without a recoverable hydration basis; awaiting reconnect recovery (attempt ${attempt} of ${WORKSPACE_UNSAFE_SYNC_MAX_RECOVERY_ATTEMPTS}): ${cause}
4322
+ `
4323
+ );
4324
+ return;
4325
+ }
4326
+ process.stderr.write(
4327
+ `[r5d-worker] workspace synchronization failed without a recoverable hydration basis; bouncing the worker socket for in-process recovery (attempt ${attempt} of ${WORKSPACE_UNSAFE_SYNC_MAX_RECOVERY_ATTEMPTS}): ${cause}
4328
+ `
4329
+ );
4330
+ const timer = setTimeout(() => {
4331
+ if (!unsafeWorkspaceSyncBouncePending) return;
4332
+ if (currentWorkerSocket === ws && (ws.readyState === WebSocket.OPEN || ws.readyState === WebSocket.CONNECTING)) {
4333
+ ws.close(WORKSPACE_UNSAFE_SYNC_RECOVERY_CLOSE_CODE, "Workspace synchronization failed without a recoverable hydration basis");
4334
+ }
4335
+ }, unsafeWorkspaceSyncRecoveryBounceDelayMs(attempt));
4336
+ timer.unref();
4337
+ };
4289
4338
  const runWorkspaceSync = async (input) => {
4290
4339
  const attemptId = input.attemptId ?? crypto.randomUUID();
4291
4340
  const requestedAt = Date.now();
@@ -4294,7 +4343,7 @@ async function startWorker(options) {
4294
4343
  const admittedDeferredIncidentId = deferredWorkspaceConfiguration?.incidentId ?? null;
4295
4344
  const assertStillAdmitted = () => {
4296
4345
  if (admittedGeneration !== workerAdmissionGeneration || admittedActiveIncidentId !== activeWorkspaceIncidentId || admittedDeferredIncidentId !== (deferredWorkspaceConfiguration?.incidentId ?? null) || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN || !deferredWorkspaceSyncTriggerIsAllowed(deferredWorkspaceConfiguration, activeWorkspaceIncidentId, input.trigger)) {
4297
- throw new Error("Workspace synchronization admission changed while asynchronous work was in flight");
4346
+ throw new WorkspaceSyncAdmissionChangedError("Workspace synchronization admission changed while asynchronous work was in flight");
4298
4347
  }
4299
4348
  };
4300
4349
  if (!deferredWorkspaceSyncTriggerIsAllowed(deferredWorkspaceConfiguration, activeWorkspaceIncidentId, input.trigger)) {
@@ -4346,31 +4395,62 @@ async function startWorker(options) {
4346
4395
  assertStillAdmitted
4347
4396
  });
4348
4397
  } catch (error) {
4349
- const hydrationCurrent = workspaceSyncFailureHydrationIsSafe({
4398
+ const errorMessage = error instanceof Error ? error.message : String(error);
4399
+ const hydrationRecoverable = workspaceSyncFailureHydrationIsSafe({
4350
4400
  error,
4351
4401
  resetToCanonical: input.resetToCanonical === true,
4352
- inspectCurrentHydration: () => (0, import_workspace_git_sync.workspaceGitHydrationIsCurrent)(workspaceShadowRoot, buildWorkspaceMounts())
4402
+ inspectRecoverableHydration: () => (0, import_workspace_git_sync.workspaceGitHydrationIsRecoverable)(workspaceShadowRoot, buildWorkspaceMounts())
4353
4403
  });
4354
- if (!hydrationCurrent) {
4355
- process.stderr.write(
4356
- `[r5d-worker] workspace synchronization failed with an incompletely hydrated visible tree; exiting for exact recovery: ${error instanceof Error ? error.message : String(error)}
4404
+ if (hydrationRecoverable) {
4405
+ if (error instanceof WorkspaceSyncAdmissionChangedError) {
4406
+ process.stderr.write(
4407
+ `[r5d-worker] workspace synchronization was cancelled with every mount at a recorded basis; the next cycle converges it: ${errorMessage}
4357
4408
  `
4358
- );
4409
+ );
4410
+ }
4411
+ return await failedWorkspaceSyncResult(attemptId, input.trigger, error);
4359
4412
  }
4360
- fenceUnsafeWorkspaceSyncFailure({
4361
- hydrationCurrent,
4413
+ const fenced = fenceUnsafeWorkspaceSyncFailure({
4414
+ hydrationRecoverable,
4415
+ resetToCanonical: input.resetToCanonical === true,
4416
+ recoveryAttempts: unsafeWorkspaceSyncRecoveryAttempts,
4417
+ maxRecoveryAttempts: WORKSPACE_UNSAFE_SYNC_MAX_RECOVERY_ATTEMPTS,
4418
+ hasRetainableWork: activeProcesses.size > 0 || credentialBearingProcessGroups.size > 0 || activePtys.size > 0 || pendingProcessTerminals.size > 0,
4419
+ bouncePending: unsafeWorkspaceSyncBouncePending,
4362
4420
  invalidateExecution: () => {
4363
4421
  workspaceConfigured = false;
4364
4422
  advanceWorkerAdmissionGeneration();
4365
4423
  },
4424
+ scheduleRecoveryBounce: () => {
4425
+ unsafeWorkspaceSyncRecoveryAttempts += 1;
4426
+ scheduleUnsafeWorkspaceSyncRecoveryBounce(errorMessage);
4427
+ },
4428
+ fatalExit: (exitCode) => {
4429
+ process.stderr.write(
4430
+ `[r5d-worker] workspace synchronization failed without a recoverable hydration basis; exiting for exact recovery after ${unsafeWorkspaceSyncRecoveryAttempts} in-process recovery attempt(s): ${errorMessage}
4431
+ `
4432
+ );
4433
+ process.exit(exitCode);
4434
+ },
4366
4435
  exitCode: 1
4367
4436
  });
4368
- return await failedWorkspaceSyncResult(attemptId, input.trigger, error);
4437
+ if (fenced === "recovery_pending") {
4438
+ process.stderr.write(
4439
+ `[r5d-worker] workspace synchronization failed without a recoverable hydration basis while in-process recovery is already pending: ${errorMessage}
4440
+ `
4441
+ );
4442
+ }
4443
+ return {
4444
+ ...await failedWorkspaceSyncResult(attemptId, input.trigger, error),
4445
+ failureReason: "unsafe_hydration",
4446
+ recoveryAttempt: unsafeWorkspaceSyncRecoveryAttempts
4447
+ };
4369
4448
  }
4370
4449
  });
4371
4450
  } finally {
4372
4451
  workspaceSyncRequestsInFlight -= 1;
4373
4452
  }
4453
+ if (result.outcome !== "failed") unsafeWorkspaceSyncRecoveryAttempts = 0;
4374
4454
  const finishedAt = Date.now();
4375
4455
  result = {
4376
4456
  ...result,
@@ -4825,7 +4905,9 @@ async function startWorker(options) {
4825
4905
  const configurationSyncAdmissionGeneration = workerAdmissionGeneration;
4826
4906
  const assertConfigurationSyncStillAdmitted = () => {
4827
4907
  if (receiptGeneration !== workspaceConfigurationReceiptGeneration || configurationSyncAdmissionGeneration !== workerAdmissionGeneration || !workspaceConfigurationIncidentSnapshotIsCurrent(incidentDeferral.incidentId, activeWorkspaceIncidentId) || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
4828
- throw new Error("Workspace configuration synchronization admission changed while asynchronous work was in flight");
4908
+ throw new WorkspaceSyncAdmissionChangedError(
4909
+ "Workspace configuration synchronization admission changed while asynchronous work was in flight"
4910
+ );
4829
4911
  }
4830
4912
  };
4831
4913
  const result = await performWorkspaceSync({
@@ -5022,6 +5104,8 @@ async function startWorker(options) {
5022
5104
  const refreshesDeferredConfiguration = deferredWorkspaceConfiguration !== null && activeWorkspaceIncidentId === null && message.deferWorkspaceSyncForIncidentId === void 0;
5023
5105
  advanceWorkerAdmissionGeneration();
5024
5106
  const configured = await configureWorkerWorkspace(message, receiptGeneration);
5107
+ unsafeWorkspaceSyncBouncePending = false;
5108
+ if (configured.result.outcome !== "failed") unsafeWorkspaceSyncRecoveryAttempts = 0;
5025
5109
  sendWorkerMessageFromCurrentSource(ws, {
5026
5110
  type: "workspace_configured",
5027
5111
  requestId: message.requestId,
@@ -1,5 +1,5 @@
1
1
  {
2
2
  "name": "@ricsam/r5d-worker",
3
- "version": "0.0.88",
3
+ "version": "0.0.89",
4
4
  "type": "commonjs"
5
5
  }
@@ -42,6 +42,7 @@ __export(workspace_git_sync_exports, {
42
42
  resetWorkspaceGit: () => resetWorkspaceGit,
43
43
  synchronizeWorkspaceGit: () => synchronizeWorkspaceGit,
44
44
  workspaceGitHydrationIsCurrent: () => workspaceGitHydrationIsCurrent,
45
+ workspaceGitHydrationIsRecoverable: () => workspaceGitHydrationIsRecoverable,
45
46
  workspaceGitMountsForRemediation: () => workspaceGitMountsForRemediation,
46
47
  workspaceGitSyncTestHarness: () => workspaceGitSyncTestHarness
47
48
  });
@@ -420,18 +421,25 @@ function hydratedWorkspaceReceiptFileIsV3(workspacePath) {
420
421
  }
421
422
  }
422
423
  function workspaceGitHydrationIsCurrent(workspacePath, mounts) {
424
+ return workspaceGitHydrationHolds(workspacePath, mounts, false);
425
+ }
426
+ function workspaceGitHydrationIsRecoverable(workspacePath, mounts) {
427
+ return workspaceGitHydrationHolds(workspacePath, mounts, true);
428
+ }
429
+ function workspaceGitHydrationHolds(workspacePath, mounts, tolerateIdleRecordedBasis) {
423
430
  const resolvedWorkspacePath = import_node_path.default.resolve(workspacePath);
424
431
  if (!import_node_fs.default.existsSync(import_node_path.default.join(resolvedWorkspacePath, ".git"))) return true;
425
432
  if (mounts) validateMounts(resolvedWorkspacePath, mounts);
426
433
  const head = revParse(resolvedWorkspacePath, "HEAD");
427
434
  if (!workspaceCheckoutDurabilityIsCurrent(resolvedWorkspacePath)) return false;
435
+ if (tolerateIdleRecordedBasis && lstatIfExists(hydrationTransactionPath(resolvedWorkspacePath)) !== null) return false;
428
436
  const receipt = readHydratedWorkspaceReceipt(resolvedWorkspacePath);
429
437
  if (!workspaceBasisRefsAreCurrent(resolvedWorkspacePath, receipt ?? { version: 3, mounts: [] })) return false;
430
438
  if (head === null) return true;
431
439
  if (!receipt) return false;
432
440
  return mounts ? configuredWorkspaceHydrationBasisMounts(resolvedWorkspacePath, mounts).every(
433
- (mount) => workspaceHydrationReceiptCoversMount(receipt, head, mount) || Boolean(mount.busy?.() && workspaceHydrationReceiptMountBasisHead(receipt, mount))
434
- ) : Boolean(receipt && receipt.mounts.every((mount) => mount.head === head));
441
+ (mount) => workspaceHydrationReceiptCoversMount(receipt, head, mount) || Boolean((tolerateIdleRecordedBasis || mount.busy?.()) && workspaceHydrationReceiptMountBasisHead(receipt, mount))
442
+ ) : Boolean(receipt && receipt.mounts.every((mount) => tolerateIdleRecordedBasis || mount.head === head));
435
443
  }
436
444
  const WORKSPACE_GIT_BASIS_REF_PREFIX = "refs/r5d/workspace-basis/";
437
445
  const WORKSPACE_GIT_BASIS_STAGING_REF_PREFIX = "refs/r5d/workspace-basis-staging/";
@@ -2190,6 +2198,7 @@ const workspaceGitSyncTestHarness = {
2190
2198
  resetWorkspaceGit,
2191
2199
  synchronizeWorkspaceGit,
2192
2200
  workspaceGitHydrationIsCurrent,
2201
+ workspaceGitHydrationIsRecoverable,
2193
2202
  workspaceGitMountsForRemediation,
2194
2203
  workspaceGitSyncTestHarness
2195
2204
  });
package/dist/mjs/main.mjs CHANGED
@@ -88,7 +88,7 @@ import {
88
88
  recoverWorkspaceGitHydration,
89
89
  resetWorkspaceGit,
90
90
  synchronizeWorkspaceGit,
91
- workspaceGitHydrationIsCurrent,
91
+ workspaceGitHydrationIsRecoverable,
92
92
  workspaceGitMountsForRemediation,
93
93
  WorkspaceRemediationAncestryError
94
94
  } from "./workspace-git-sync.mjs";
@@ -158,6 +158,11 @@ class StaleWorkerAdmissionError extends Error {
158
158
  this.name = "StaleWorkerAdmissionError";
159
159
  }
160
160
  }
161
+ class WorkspaceSyncAdmissionChangedError extends Error {
162
+ name = "WorkspaceSyncAdmissionChangedError";
163
+ }
164
+ let unsafeWorkspaceSyncRecoveryAttempts = 0;
165
+ let unsafeWorkspaceSyncBouncePending = false;
161
166
  function advanceWorkerAdmissionGeneration() {
162
167
  workerAdmissionGeneration += 1;
163
168
  return workerAdmissionGeneration;
@@ -1641,16 +1646,33 @@ function workspaceSyncFailureHydrationIsSafe(input) {
1641
1646
  if (input.error instanceof WorkspaceRemediationAncestryError) return true;
1642
1647
  if (input.resetToCanonical) return false;
1643
1648
  try {
1644
- return input.inspectCurrentHydration();
1649
+ return input.inspectRecoverableHydration();
1645
1650
  } catch {
1646
1651
  return false;
1647
1652
  }
1648
1653
  }
1654
+ const WORKSPACE_UNSAFE_SYNC_MAX_RECOVERY_ATTEMPTS = 3;
1655
+ const WORKSPACE_UNSAFE_SYNC_RECOVERY_CLOSE_CODE = 4008;
1656
+ function unsafeWorkspaceSyncRecoveryBounceDelayMs(attempt) {
1657
+ return Math.min(1e3 * 2 ** Math.max(0, attempt - 1), 3e4);
1658
+ }
1659
+ function unsafeWorkspaceSyncFailureResponse(input) {
1660
+ if (input.resetToCanonical) return "fatal_exit";
1661
+ if (!input.hasRetainableWork) return "fatal_exit";
1662
+ if (input.recoveryAttempts >= input.maxRecoveryAttempts) return "fatal_exit";
1663
+ if (input.bouncePending) return "recovery_pending";
1664
+ return "recovery_bounce";
1665
+ }
1649
1666
  function fenceUnsafeWorkspaceSyncFailure(input) {
1650
- if (input.hydrationCurrent) return false;
1667
+ if (input.hydrationRecoverable) return "none";
1651
1668
  input.invalidateExecution();
1652
- requireCredentialGenerationRestart(true, input.fatalExit, input.exitCode);
1653
- return true;
1669
+ const response = unsafeWorkspaceSyncFailureResponse(input);
1670
+ if (response === "fatal_exit") {
1671
+ requireCredentialGenerationRestart(true, input.fatalExit, input.exitCode);
1672
+ throw new Error("Unsafe workspace synchronization fence continued past its fatal exit");
1673
+ }
1674
+ if (response === "recovery_bounce") input.scheduleRecoveryBounce();
1675
+ return response;
1654
1676
  }
1655
1677
  const workerGitSecurityTestHarness = {
1656
1678
  commandArgs: workerGitCommand.commandArgs,
@@ -1678,6 +1700,11 @@ const workerGitSecurityTestHarness = {
1678
1700
  credentialReapContractStatus,
1679
1701
  verifiedCredentialReapContract,
1680
1702
  fenceUnsafeWorkspaceSyncFailure,
1703
+ unsafeWorkspaceSyncFailureResponse,
1704
+ unsafeWorkspaceSyncRecoveryBounceDelayMs,
1705
+ WORKSPACE_UNSAFE_SYNC_MAX_RECOVERY_ATTEMPTS,
1706
+ WORKSPACE_UNSAFE_SYNC_RECOVERY_CLOSE_CODE,
1707
+ WorkspaceSyncAdmissionChangedError,
1681
1708
  workspaceConfigurationIncidentDeferral,
1682
1709
  workspaceIncidentTerminalClearDisposition,
1683
1710
  workspaceIncidentTerminalRefreshFence,
@@ -4294,6 +4321,28 @@ async function startWorker(options) {
4294
4321
  );
4295
4322
  }
4296
4323
  };
4324
+ const scheduleUnsafeWorkspaceSyncRecoveryBounce = (cause) => {
4325
+ unsafeWorkspaceSyncBouncePending = true;
4326
+ const attempt = unsafeWorkspaceSyncRecoveryAttempts;
4327
+ if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
4328
+ process.stderr.write(
4329
+ `[r5d-worker] workspace synchronization failed without a recoverable hydration basis; awaiting reconnect recovery (attempt ${attempt} of ${WORKSPACE_UNSAFE_SYNC_MAX_RECOVERY_ATTEMPTS}): ${cause}
4330
+ `
4331
+ );
4332
+ return;
4333
+ }
4334
+ process.stderr.write(
4335
+ `[r5d-worker] workspace synchronization failed without a recoverable hydration basis; bouncing the worker socket for in-process recovery (attempt ${attempt} of ${WORKSPACE_UNSAFE_SYNC_MAX_RECOVERY_ATTEMPTS}): ${cause}
4336
+ `
4337
+ );
4338
+ const timer = setTimeout(() => {
4339
+ if (!unsafeWorkspaceSyncBouncePending) return;
4340
+ if (currentWorkerSocket === ws && (ws.readyState === WebSocket.OPEN || ws.readyState === WebSocket.CONNECTING)) {
4341
+ ws.close(WORKSPACE_UNSAFE_SYNC_RECOVERY_CLOSE_CODE, "Workspace synchronization failed without a recoverable hydration basis");
4342
+ }
4343
+ }, unsafeWorkspaceSyncRecoveryBounceDelayMs(attempt));
4344
+ timer.unref();
4345
+ };
4297
4346
  const runWorkspaceSync = async (input) => {
4298
4347
  const attemptId = input.attemptId ?? crypto.randomUUID();
4299
4348
  const requestedAt = Date.now();
@@ -4302,7 +4351,7 @@ async function startWorker(options) {
4302
4351
  const admittedDeferredIncidentId = deferredWorkspaceConfiguration?.incidentId ?? null;
4303
4352
  const assertStillAdmitted = () => {
4304
4353
  if (admittedGeneration !== workerAdmissionGeneration || admittedActiveIncidentId !== activeWorkspaceIncidentId || admittedDeferredIncidentId !== (deferredWorkspaceConfiguration?.incidentId ?? null) || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN || !deferredWorkspaceSyncTriggerIsAllowed(deferredWorkspaceConfiguration, activeWorkspaceIncidentId, input.trigger)) {
4305
- throw new Error("Workspace synchronization admission changed while asynchronous work was in flight");
4354
+ throw new WorkspaceSyncAdmissionChangedError("Workspace synchronization admission changed while asynchronous work was in flight");
4306
4355
  }
4307
4356
  };
4308
4357
  if (!deferredWorkspaceSyncTriggerIsAllowed(deferredWorkspaceConfiguration, activeWorkspaceIncidentId, input.trigger)) {
@@ -4354,31 +4403,62 @@ async function startWorker(options) {
4354
4403
  assertStillAdmitted
4355
4404
  });
4356
4405
  } catch (error) {
4357
- const hydrationCurrent = workspaceSyncFailureHydrationIsSafe({
4406
+ const errorMessage = error instanceof Error ? error.message : String(error);
4407
+ const hydrationRecoverable = workspaceSyncFailureHydrationIsSafe({
4358
4408
  error,
4359
4409
  resetToCanonical: input.resetToCanonical === true,
4360
- inspectCurrentHydration: () => workspaceGitHydrationIsCurrent(workspaceShadowRoot, buildWorkspaceMounts())
4410
+ inspectRecoverableHydration: () => workspaceGitHydrationIsRecoverable(workspaceShadowRoot, buildWorkspaceMounts())
4361
4411
  });
4362
- if (!hydrationCurrent) {
4363
- process.stderr.write(
4364
- `[r5d-worker] workspace synchronization failed with an incompletely hydrated visible tree; exiting for exact recovery: ${error instanceof Error ? error.message : String(error)}
4412
+ if (hydrationRecoverable) {
4413
+ if (error instanceof WorkspaceSyncAdmissionChangedError) {
4414
+ process.stderr.write(
4415
+ `[r5d-worker] workspace synchronization was cancelled with every mount at a recorded basis; the next cycle converges it: ${errorMessage}
4365
4416
  `
4366
- );
4417
+ );
4418
+ }
4419
+ return await failedWorkspaceSyncResult(attemptId, input.trigger, error);
4367
4420
  }
4368
- fenceUnsafeWorkspaceSyncFailure({
4369
- hydrationCurrent,
4421
+ const fenced = fenceUnsafeWorkspaceSyncFailure({
4422
+ hydrationRecoverable,
4423
+ resetToCanonical: input.resetToCanonical === true,
4424
+ recoveryAttempts: unsafeWorkspaceSyncRecoveryAttempts,
4425
+ maxRecoveryAttempts: WORKSPACE_UNSAFE_SYNC_MAX_RECOVERY_ATTEMPTS,
4426
+ hasRetainableWork: activeProcesses.size > 0 || credentialBearingProcessGroups.size > 0 || activePtys.size > 0 || pendingProcessTerminals.size > 0,
4427
+ bouncePending: unsafeWorkspaceSyncBouncePending,
4370
4428
  invalidateExecution: () => {
4371
4429
  workspaceConfigured = false;
4372
4430
  advanceWorkerAdmissionGeneration();
4373
4431
  },
4432
+ scheduleRecoveryBounce: () => {
4433
+ unsafeWorkspaceSyncRecoveryAttempts += 1;
4434
+ scheduleUnsafeWorkspaceSyncRecoveryBounce(errorMessage);
4435
+ },
4436
+ fatalExit: (exitCode) => {
4437
+ process.stderr.write(
4438
+ `[r5d-worker] workspace synchronization failed without a recoverable hydration basis; exiting for exact recovery after ${unsafeWorkspaceSyncRecoveryAttempts} in-process recovery attempt(s): ${errorMessage}
4439
+ `
4440
+ );
4441
+ process.exit(exitCode);
4442
+ },
4374
4443
  exitCode: 1
4375
4444
  });
4376
- return await failedWorkspaceSyncResult(attemptId, input.trigger, error);
4445
+ if (fenced === "recovery_pending") {
4446
+ process.stderr.write(
4447
+ `[r5d-worker] workspace synchronization failed without a recoverable hydration basis while in-process recovery is already pending: ${errorMessage}
4448
+ `
4449
+ );
4450
+ }
4451
+ return {
4452
+ ...await failedWorkspaceSyncResult(attemptId, input.trigger, error),
4453
+ failureReason: "unsafe_hydration",
4454
+ recoveryAttempt: unsafeWorkspaceSyncRecoveryAttempts
4455
+ };
4377
4456
  }
4378
4457
  });
4379
4458
  } finally {
4380
4459
  workspaceSyncRequestsInFlight -= 1;
4381
4460
  }
4461
+ if (result.outcome !== "failed") unsafeWorkspaceSyncRecoveryAttempts = 0;
4382
4462
  const finishedAt = Date.now();
4383
4463
  result = {
4384
4464
  ...result,
@@ -4833,7 +4913,9 @@ async function startWorker(options) {
4833
4913
  const configurationSyncAdmissionGeneration = workerAdmissionGeneration;
4834
4914
  const assertConfigurationSyncStillAdmitted = () => {
4835
4915
  if (receiptGeneration !== workspaceConfigurationReceiptGeneration || configurationSyncAdmissionGeneration !== workerAdmissionGeneration || !workspaceConfigurationIncidentSnapshotIsCurrent(incidentDeferral.incidentId, activeWorkspaceIncidentId) || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
4836
- throw new Error("Workspace configuration synchronization admission changed while asynchronous work was in flight");
4916
+ throw new WorkspaceSyncAdmissionChangedError(
4917
+ "Workspace configuration synchronization admission changed while asynchronous work was in flight"
4918
+ );
4837
4919
  }
4838
4920
  };
4839
4921
  const result = await performWorkspaceSync({
@@ -5030,6 +5112,8 @@ async function startWorker(options) {
5030
5112
  const refreshesDeferredConfiguration = deferredWorkspaceConfiguration !== null && activeWorkspaceIncidentId === null && message.deferWorkspaceSyncForIncidentId === void 0;
5031
5113
  advanceWorkerAdmissionGeneration();
5032
5114
  const configured = await configureWorkerWorkspace(message, receiptGeneration);
5115
+ unsafeWorkspaceSyncBouncePending = false;
5116
+ if (configured.result.outcome !== "failed") unsafeWorkspaceSyncRecoveryAttempts = 0;
5033
5117
  sendWorkerMessageFromCurrentSource(ws, {
5034
5118
  type: "workspace_configured",
5035
5119
  requestId: message.requestId,
@@ -1,5 +1,5 @@
1
1
  {
2
2
  "name": "@ricsam/r5d-worker",
3
- "version": "0.0.88",
3
+ "version": "0.0.89",
4
4
  "type": "module"
5
5
  }
@@ -376,18 +376,25 @@ function hydratedWorkspaceReceiptFileIsV3(workspacePath) {
376
376
  }
377
377
  }
378
378
  function workspaceGitHydrationIsCurrent(workspacePath, mounts) {
379
+ return workspaceGitHydrationHolds(workspacePath, mounts, false);
380
+ }
381
+ function workspaceGitHydrationIsRecoverable(workspacePath, mounts) {
382
+ return workspaceGitHydrationHolds(workspacePath, mounts, true);
383
+ }
384
+ function workspaceGitHydrationHolds(workspacePath, mounts, tolerateIdleRecordedBasis) {
379
385
  const resolvedWorkspacePath = path.resolve(workspacePath);
380
386
  if (!fs.existsSync(path.join(resolvedWorkspacePath, ".git"))) return true;
381
387
  if (mounts) validateMounts(resolvedWorkspacePath, mounts);
382
388
  const head = revParse(resolvedWorkspacePath, "HEAD");
383
389
  if (!workspaceCheckoutDurabilityIsCurrent(resolvedWorkspacePath)) return false;
390
+ if (tolerateIdleRecordedBasis && lstatIfExists(hydrationTransactionPath(resolvedWorkspacePath)) !== null) return false;
384
391
  const receipt = readHydratedWorkspaceReceipt(resolvedWorkspacePath);
385
392
  if (!workspaceBasisRefsAreCurrent(resolvedWorkspacePath, receipt ?? { version: 3, mounts: [] })) return false;
386
393
  if (head === null) return true;
387
394
  if (!receipt) return false;
388
395
  return mounts ? configuredWorkspaceHydrationBasisMounts(resolvedWorkspacePath, mounts).every(
389
- (mount) => workspaceHydrationReceiptCoversMount(receipt, head, mount) || Boolean(mount.busy?.() && workspaceHydrationReceiptMountBasisHead(receipt, mount))
390
- ) : Boolean(receipt && receipt.mounts.every((mount) => mount.head === head));
396
+ (mount) => workspaceHydrationReceiptCoversMount(receipt, head, mount) || Boolean((tolerateIdleRecordedBasis || mount.busy?.()) && workspaceHydrationReceiptMountBasisHead(receipt, mount))
397
+ ) : Boolean(receipt && receipt.mounts.every((mount) => tolerateIdleRecordedBasis || mount.head === head));
391
398
  }
392
399
  const WORKSPACE_GIT_BASIS_REF_PREFIX = "refs/r5d/workspace-basis/";
393
400
  const WORKSPACE_GIT_BASIS_STAGING_REF_PREFIX = "refs/r5d/workspace-basis-staging/";
@@ -2145,6 +2152,7 @@ export {
2145
2152
  resetWorkspaceGit,
2146
2153
  synchronizeWorkspaceGit,
2147
2154
  workspaceGitHydrationIsCurrent,
2155
+ workspaceGitHydrationIsRecoverable,
2148
2156
  workspaceGitMountsForRemediation,
2149
2157
  workspaceGitSyncTestHarness
2150
2158
  };
@@ -69,6 +69,10 @@ type WorkspaceSyncResult = {
69
69
  synchronizeMs: number;
70
70
  };
71
71
  error?: string;
72
+ /** Typed reason for a failed outcome the server should surface loudly. */
73
+ failureReason?: "unsafe_hydration";
74
+ /** Consecutive in-process recovery bounces taken for unsafe hydration failures. */
75
+ recoveryAttempt?: number;
72
76
  };
73
77
  export type WorkerSessionTarget = {
74
78
  type: "project";
@@ -356,6 +360,9 @@ type WorkerCodeReadResult = {
356
360
  mtime: string;
357
361
  base64: string;
358
362
  };
363
+ declare class WorkspaceSyncAdmissionChangedError extends Error {
364
+ readonly name = "WorkspaceSyncAdmissionChangedError";
365
+ }
359
366
  declare function assertWorkerChildAdmission(input: {
360
367
  capturedGeneration: number;
361
368
  currentGeneration: number;
@@ -609,14 +616,29 @@ declare function deferredIncidentWorkspaceConfigurationResult(input: {
609
616
  declare function workspaceSyncFailureHydrationIsSafe(input: {
610
617
  error: unknown;
611
618
  resetToCanonical: boolean;
612
- inspectCurrentHydration: () => boolean;
619
+ inspectRecoverableHydration: () => boolean;
613
620
  }): boolean;
621
+ declare function unsafeWorkspaceSyncRecoveryBounceDelayMs(attempt: number): number;
622
+ type UnsafeWorkspaceSyncFailureResponse = "fatal_exit" | "recovery_bounce" | "recovery_pending";
623
+ declare function unsafeWorkspaceSyncFailureResponse(input: {
624
+ resetToCanonical: boolean;
625
+ recoveryAttempts: number;
626
+ maxRecoveryAttempts: number;
627
+ hasRetainableWork: boolean;
628
+ bouncePending: boolean;
629
+ }): UnsafeWorkspaceSyncFailureResponse;
614
630
  declare function fenceUnsafeWorkspaceSyncFailure(input: {
615
- hydrationCurrent: boolean;
631
+ hydrationRecoverable: boolean;
632
+ resetToCanonical: boolean;
633
+ recoveryAttempts: number;
634
+ maxRecoveryAttempts: number;
635
+ hasRetainableWork: boolean;
636
+ bouncePending: boolean;
616
637
  invalidateExecution: () => void;
638
+ scheduleRecoveryBounce: () => void;
617
639
  fatalExit?: (exitCode: number) => void;
618
640
  exitCode?: number;
619
- }): boolean;
641
+ }): "none" | "recovery_bounce" | "recovery_pending";
620
642
  export declare const workerGitSecurityTestHarness: {
621
643
  commandArgs: (args: string[]) => string[];
622
644
  credentialHelperForRemotes: typeof credentialHelperForRemotes;
@@ -641,6 +663,11 @@ export declare const workerGitSecurityTestHarness: {
641
663
  credentialReapContractStatus: typeof credentialReapContractStatus;
642
664
  verifiedCredentialReapContract: typeof verifiedCredentialReapContract;
643
665
  fenceUnsafeWorkspaceSyncFailure: typeof fenceUnsafeWorkspaceSyncFailure;
666
+ unsafeWorkspaceSyncFailureResponse: typeof unsafeWorkspaceSyncFailureResponse;
667
+ unsafeWorkspaceSyncRecoveryBounceDelayMs: typeof unsafeWorkspaceSyncRecoveryBounceDelayMs;
668
+ WORKSPACE_UNSAFE_SYNC_MAX_RECOVERY_ATTEMPTS: number;
669
+ WORKSPACE_UNSAFE_SYNC_RECOVERY_CLOSE_CODE: number;
670
+ WorkspaceSyncAdmissionChangedError: typeof WorkspaceSyncAdmissionChangedError;
644
671
  workspaceConfigurationIncidentDeferral: typeof workspaceConfigurationIncidentDeferral;
645
672
  workspaceIncidentTerminalClearDisposition: typeof workspaceIncidentTerminalClearDisposition;
646
673
  workspaceIncidentTerminalRefreshFence: typeof workspaceIncidentTerminalRefreshFence;
@@ -71,6 +71,16 @@ export declare class WorkspaceRemediationAncestryError extends Error {
71
71
  declare function gitCommandArgs(args: string[]): string[];
72
72
  declare function workspaceCloneCommandArgs(args: string[], remoteUrl: string, credentialHelper?: string | null, credentialUsername?: string | null): string[];
73
73
  export declare function workspaceGitHydrationIsCurrent(workspacePath: string, mounts?: readonly WorkspaceGitMount[]): boolean;
74
+ /**
75
+ * Failure-time variant of the currency check. A mount pinned at a valid
76
+ * recorded basis is convergent whether or not it is busy right now: the next
77
+ * cycle merges it forward from that exact basis and a genuine collision
78
+ * surfaces as a projection_merge conflict, never a silent overwrite. Live
79
+ * busy-ness must not decide safety here because the same disconnect that
80
+ * aborts a sync also clears the process/PTY maps busy-ness is computed from.
81
+ * Configuration-time recovery keeps the strict predicate.
82
+ */
83
+ export declare function workspaceGitHydrationIsRecoverable(workspacePath: string, mounts?: readonly WorkspaceGitMount[]): boolean;
74
84
  declare function configureWorkspaceRepository(input: {
75
85
  workspacePath: string;
76
86
  remoteUrl: string;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@ricsam/r5d-worker",
3
- "version": "0.0.88",
3
+ "version": "0.0.89",
4
4
  "type": "module",
5
5
  "main": "./dist/cjs/main.cjs",
6
6
  "module": "./dist/mjs/main.mjs",