@ricsam/r5d-worker 0.0.88 → 0.0.89
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/main.cjs +99 -15
- package/dist/cjs/package.json +1 -1
- package/dist/cjs/workspace-git-sync.cjs +11 -2
- package/dist/mjs/main.mjs +100 -16
- package/dist/mjs/package.json +1 -1
- package/dist/mjs/workspace-git-sync.mjs +10 -2
- package/dist/types/main.d.ts +30 -3
- package/dist/types/workspace-git-sync.d.ts +10 -0
- package/package.json +1 -1
package/dist/cjs/main.cjs
CHANGED
|
@@ -150,6 +150,11 @@ class StaleWorkerAdmissionError extends Error {
|
|
|
150
150
|
this.name = "StaleWorkerAdmissionError";
|
|
151
151
|
}
|
|
152
152
|
}
|
|
153
|
+
class WorkspaceSyncAdmissionChangedError extends Error {
|
|
154
|
+
name = "WorkspaceSyncAdmissionChangedError";
|
|
155
|
+
}
|
|
156
|
+
let unsafeWorkspaceSyncRecoveryAttempts = 0;
|
|
157
|
+
let unsafeWorkspaceSyncBouncePending = false;
|
|
153
158
|
function advanceWorkerAdmissionGeneration() {
|
|
154
159
|
workerAdmissionGeneration += 1;
|
|
155
160
|
return workerAdmissionGeneration;
|
|
@@ -1633,16 +1638,33 @@ function workspaceSyncFailureHydrationIsSafe(input) {
|
|
|
1633
1638
|
if (input.error instanceof import_workspace_git_sync.WorkspaceRemediationAncestryError) return true;
|
|
1634
1639
|
if (input.resetToCanonical) return false;
|
|
1635
1640
|
try {
|
|
1636
|
-
return input.
|
|
1641
|
+
return input.inspectRecoverableHydration();
|
|
1637
1642
|
} catch {
|
|
1638
1643
|
return false;
|
|
1639
1644
|
}
|
|
1640
1645
|
}
|
|
1646
|
+
const WORKSPACE_UNSAFE_SYNC_MAX_RECOVERY_ATTEMPTS = 3;
|
|
1647
|
+
const WORKSPACE_UNSAFE_SYNC_RECOVERY_CLOSE_CODE = 4008;
|
|
1648
|
+
function unsafeWorkspaceSyncRecoveryBounceDelayMs(attempt) {
|
|
1649
|
+
return Math.min(1e3 * 2 ** Math.max(0, attempt - 1), 3e4);
|
|
1650
|
+
}
|
|
1651
|
+
function unsafeWorkspaceSyncFailureResponse(input) {
|
|
1652
|
+
if (input.resetToCanonical) return "fatal_exit";
|
|
1653
|
+
if (!input.hasRetainableWork) return "fatal_exit";
|
|
1654
|
+
if (input.recoveryAttempts >= input.maxRecoveryAttempts) return "fatal_exit";
|
|
1655
|
+
if (input.bouncePending) return "recovery_pending";
|
|
1656
|
+
return "recovery_bounce";
|
|
1657
|
+
}
|
|
1641
1658
|
function fenceUnsafeWorkspaceSyncFailure(input) {
|
|
1642
|
-
if (input.
|
|
1659
|
+
if (input.hydrationRecoverable) return "none";
|
|
1643
1660
|
input.invalidateExecution();
|
|
1644
|
-
|
|
1645
|
-
|
|
1661
|
+
const response = unsafeWorkspaceSyncFailureResponse(input);
|
|
1662
|
+
if (response === "fatal_exit") {
|
|
1663
|
+
requireCredentialGenerationRestart(true, input.fatalExit, input.exitCode);
|
|
1664
|
+
throw new Error("Unsafe workspace synchronization fence continued past its fatal exit");
|
|
1665
|
+
}
|
|
1666
|
+
if (response === "recovery_bounce") input.scheduleRecoveryBounce();
|
|
1667
|
+
return response;
|
|
1646
1668
|
}
|
|
1647
1669
|
const workerGitSecurityTestHarness = {
|
|
1648
1670
|
commandArgs: workerGitCommand.commandArgs,
|
|
@@ -1670,6 +1692,11 @@ const workerGitSecurityTestHarness = {
|
|
|
1670
1692
|
credentialReapContractStatus,
|
|
1671
1693
|
verifiedCredentialReapContract,
|
|
1672
1694
|
fenceUnsafeWorkspaceSyncFailure,
|
|
1695
|
+
unsafeWorkspaceSyncFailureResponse,
|
|
1696
|
+
unsafeWorkspaceSyncRecoveryBounceDelayMs,
|
|
1697
|
+
WORKSPACE_UNSAFE_SYNC_MAX_RECOVERY_ATTEMPTS,
|
|
1698
|
+
WORKSPACE_UNSAFE_SYNC_RECOVERY_CLOSE_CODE,
|
|
1699
|
+
WorkspaceSyncAdmissionChangedError,
|
|
1673
1700
|
workspaceConfigurationIncidentDeferral,
|
|
1674
1701
|
workspaceIncidentTerminalClearDisposition,
|
|
1675
1702
|
workspaceIncidentTerminalRefreshFence,
|
|
@@ -4286,6 +4313,28 @@ async function startWorker(options) {
|
|
|
4286
4313
|
);
|
|
4287
4314
|
}
|
|
4288
4315
|
};
|
|
4316
|
+
const scheduleUnsafeWorkspaceSyncRecoveryBounce = (cause) => {
|
|
4317
|
+
unsafeWorkspaceSyncBouncePending = true;
|
|
4318
|
+
const attempt = unsafeWorkspaceSyncRecoveryAttempts;
|
|
4319
|
+
if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
|
|
4320
|
+
process.stderr.write(
|
|
4321
|
+
`[r5d-worker] workspace synchronization failed without a recoverable hydration basis; awaiting reconnect recovery (attempt ${attempt} of ${WORKSPACE_UNSAFE_SYNC_MAX_RECOVERY_ATTEMPTS}): ${cause}
|
|
4322
|
+
`
|
|
4323
|
+
);
|
|
4324
|
+
return;
|
|
4325
|
+
}
|
|
4326
|
+
process.stderr.write(
|
|
4327
|
+
`[r5d-worker] workspace synchronization failed without a recoverable hydration basis; bouncing the worker socket for in-process recovery (attempt ${attempt} of ${WORKSPACE_UNSAFE_SYNC_MAX_RECOVERY_ATTEMPTS}): ${cause}
|
|
4328
|
+
`
|
|
4329
|
+
);
|
|
4330
|
+
const timer = setTimeout(() => {
|
|
4331
|
+
if (!unsafeWorkspaceSyncBouncePending) return;
|
|
4332
|
+
if (currentWorkerSocket === ws && (ws.readyState === WebSocket.OPEN || ws.readyState === WebSocket.CONNECTING)) {
|
|
4333
|
+
ws.close(WORKSPACE_UNSAFE_SYNC_RECOVERY_CLOSE_CODE, "Workspace synchronization failed without a recoverable hydration basis");
|
|
4334
|
+
}
|
|
4335
|
+
}, unsafeWorkspaceSyncRecoveryBounceDelayMs(attempt));
|
|
4336
|
+
timer.unref();
|
|
4337
|
+
};
|
|
4289
4338
|
const runWorkspaceSync = async (input) => {
|
|
4290
4339
|
const attemptId = input.attemptId ?? crypto.randomUUID();
|
|
4291
4340
|
const requestedAt = Date.now();
|
|
@@ -4294,7 +4343,7 @@ async function startWorker(options) {
|
|
|
4294
4343
|
const admittedDeferredIncidentId = deferredWorkspaceConfiguration?.incidentId ?? null;
|
|
4295
4344
|
const assertStillAdmitted = () => {
|
|
4296
4345
|
if (admittedGeneration !== workerAdmissionGeneration || admittedActiveIncidentId !== activeWorkspaceIncidentId || admittedDeferredIncidentId !== (deferredWorkspaceConfiguration?.incidentId ?? null) || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN || !deferredWorkspaceSyncTriggerIsAllowed(deferredWorkspaceConfiguration, activeWorkspaceIncidentId, input.trigger)) {
|
|
4297
|
-
throw new
|
|
4346
|
+
throw new WorkspaceSyncAdmissionChangedError("Workspace synchronization admission changed while asynchronous work was in flight");
|
|
4298
4347
|
}
|
|
4299
4348
|
};
|
|
4300
4349
|
if (!deferredWorkspaceSyncTriggerIsAllowed(deferredWorkspaceConfiguration, activeWorkspaceIncidentId, input.trigger)) {
|
|
@@ -4346,31 +4395,62 @@ async function startWorker(options) {
|
|
|
4346
4395
|
assertStillAdmitted
|
|
4347
4396
|
});
|
|
4348
4397
|
} catch (error) {
|
|
4349
|
-
const
|
|
4398
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
4399
|
+
const hydrationRecoverable = workspaceSyncFailureHydrationIsSafe({
|
|
4350
4400
|
error,
|
|
4351
4401
|
resetToCanonical: input.resetToCanonical === true,
|
|
4352
|
-
|
|
4402
|
+
inspectRecoverableHydration: () => (0, import_workspace_git_sync.workspaceGitHydrationIsRecoverable)(workspaceShadowRoot, buildWorkspaceMounts())
|
|
4353
4403
|
});
|
|
4354
|
-
if (
|
|
4355
|
-
|
|
4356
|
-
|
|
4404
|
+
if (hydrationRecoverable) {
|
|
4405
|
+
if (error instanceof WorkspaceSyncAdmissionChangedError) {
|
|
4406
|
+
process.stderr.write(
|
|
4407
|
+
`[r5d-worker] workspace synchronization was cancelled with every mount at a recorded basis; the next cycle converges it: ${errorMessage}
|
|
4357
4408
|
`
|
|
4358
|
-
|
|
4409
|
+
);
|
|
4410
|
+
}
|
|
4411
|
+
return await failedWorkspaceSyncResult(attemptId, input.trigger, error);
|
|
4359
4412
|
}
|
|
4360
|
-
fenceUnsafeWorkspaceSyncFailure({
|
|
4361
|
-
|
|
4413
|
+
const fenced = fenceUnsafeWorkspaceSyncFailure({
|
|
4414
|
+
hydrationRecoverable,
|
|
4415
|
+
resetToCanonical: input.resetToCanonical === true,
|
|
4416
|
+
recoveryAttempts: unsafeWorkspaceSyncRecoveryAttempts,
|
|
4417
|
+
maxRecoveryAttempts: WORKSPACE_UNSAFE_SYNC_MAX_RECOVERY_ATTEMPTS,
|
|
4418
|
+
hasRetainableWork: activeProcesses.size > 0 || credentialBearingProcessGroups.size > 0 || activePtys.size > 0 || pendingProcessTerminals.size > 0,
|
|
4419
|
+
bouncePending: unsafeWorkspaceSyncBouncePending,
|
|
4362
4420
|
invalidateExecution: () => {
|
|
4363
4421
|
workspaceConfigured = false;
|
|
4364
4422
|
advanceWorkerAdmissionGeneration();
|
|
4365
4423
|
},
|
|
4424
|
+
scheduleRecoveryBounce: () => {
|
|
4425
|
+
unsafeWorkspaceSyncRecoveryAttempts += 1;
|
|
4426
|
+
scheduleUnsafeWorkspaceSyncRecoveryBounce(errorMessage);
|
|
4427
|
+
},
|
|
4428
|
+
fatalExit: (exitCode) => {
|
|
4429
|
+
process.stderr.write(
|
|
4430
|
+
`[r5d-worker] workspace synchronization failed without a recoverable hydration basis; exiting for exact recovery after ${unsafeWorkspaceSyncRecoveryAttempts} in-process recovery attempt(s): ${errorMessage}
|
|
4431
|
+
`
|
|
4432
|
+
);
|
|
4433
|
+
process.exit(exitCode);
|
|
4434
|
+
},
|
|
4366
4435
|
exitCode: 1
|
|
4367
4436
|
});
|
|
4368
|
-
|
|
4437
|
+
if (fenced === "recovery_pending") {
|
|
4438
|
+
process.stderr.write(
|
|
4439
|
+
`[r5d-worker] workspace synchronization failed without a recoverable hydration basis while in-process recovery is already pending: ${errorMessage}
|
|
4440
|
+
`
|
|
4441
|
+
);
|
|
4442
|
+
}
|
|
4443
|
+
return {
|
|
4444
|
+
...await failedWorkspaceSyncResult(attemptId, input.trigger, error),
|
|
4445
|
+
failureReason: "unsafe_hydration",
|
|
4446
|
+
recoveryAttempt: unsafeWorkspaceSyncRecoveryAttempts
|
|
4447
|
+
};
|
|
4369
4448
|
}
|
|
4370
4449
|
});
|
|
4371
4450
|
} finally {
|
|
4372
4451
|
workspaceSyncRequestsInFlight -= 1;
|
|
4373
4452
|
}
|
|
4453
|
+
if (result.outcome !== "failed") unsafeWorkspaceSyncRecoveryAttempts = 0;
|
|
4374
4454
|
const finishedAt = Date.now();
|
|
4375
4455
|
result = {
|
|
4376
4456
|
...result,
|
|
@@ -4825,7 +4905,9 @@ async function startWorker(options) {
|
|
|
4825
4905
|
const configurationSyncAdmissionGeneration = workerAdmissionGeneration;
|
|
4826
4906
|
const assertConfigurationSyncStillAdmitted = () => {
|
|
4827
4907
|
if (receiptGeneration !== workspaceConfigurationReceiptGeneration || configurationSyncAdmissionGeneration !== workerAdmissionGeneration || !workspaceConfigurationIncidentSnapshotIsCurrent(incidentDeferral.incidentId, activeWorkspaceIncidentId) || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
|
|
4828
|
-
throw new
|
|
4908
|
+
throw new WorkspaceSyncAdmissionChangedError(
|
|
4909
|
+
"Workspace configuration synchronization admission changed while asynchronous work was in flight"
|
|
4910
|
+
);
|
|
4829
4911
|
}
|
|
4830
4912
|
};
|
|
4831
4913
|
const result = await performWorkspaceSync({
|
|
@@ -5022,6 +5104,8 @@ async function startWorker(options) {
|
|
|
5022
5104
|
const refreshesDeferredConfiguration = deferredWorkspaceConfiguration !== null && activeWorkspaceIncidentId === null && message.deferWorkspaceSyncForIncidentId === void 0;
|
|
5023
5105
|
advanceWorkerAdmissionGeneration();
|
|
5024
5106
|
const configured = await configureWorkerWorkspace(message, receiptGeneration);
|
|
5107
|
+
unsafeWorkspaceSyncBouncePending = false;
|
|
5108
|
+
if (configured.result.outcome !== "failed") unsafeWorkspaceSyncRecoveryAttempts = 0;
|
|
5025
5109
|
sendWorkerMessageFromCurrentSource(ws, {
|
|
5026
5110
|
type: "workspace_configured",
|
|
5027
5111
|
requestId: message.requestId,
|
package/dist/cjs/package.json
CHANGED
|
@@ -42,6 +42,7 @@ __export(workspace_git_sync_exports, {
|
|
|
42
42
|
resetWorkspaceGit: () => resetWorkspaceGit,
|
|
43
43
|
synchronizeWorkspaceGit: () => synchronizeWorkspaceGit,
|
|
44
44
|
workspaceGitHydrationIsCurrent: () => workspaceGitHydrationIsCurrent,
|
|
45
|
+
workspaceGitHydrationIsRecoverable: () => workspaceGitHydrationIsRecoverable,
|
|
45
46
|
workspaceGitMountsForRemediation: () => workspaceGitMountsForRemediation,
|
|
46
47
|
workspaceGitSyncTestHarness: () => workspaceGitSyncTestHarness
|
|
47
48
|
});
|
|
@@ -420,18 +421,25 @@ function hydratedWorkspaceReceiptFileIsV3(workspacePath) {
|
|
|
420
421
|
}
|
|
421
422
|
}
|
|
422
423
|
function workspaceGitHydrationIsCurrent(workspacePath, mounts) {
|
|
424
|
+
return workspaceGitHydrationHolds(workspacePath, mounts, false);
|
|
425
|
+
}
|
|
426
|
+
function workspaceGitHydrationIsRecoverable(workspacePath, mounts) {
|
|
427
|
+
return workspaceGitHydrationHolds(workspacePath, mounts, true);
|
|
428
|
+
}
|
|
429
|
+
function workspaceGitHydrationHolds(workspacePath, mounts, tolerateIdleRecordedBasis) {
|
|
423
430
|
const resolvedWorkspacePath = import_node_path.default.resolve(workspacePath);
|
|
424
431
|
if (!import_node_fs.default.existsSync(import_node_path.default.join(resolvedWorkspacePath, ".git"))) return true;
|
|
425
432
|
if (mounts) validateMounts(resolvedWorkspacePath, mounts);
|
|
426
433
|
const head = revParse(resolvedWorkspacePath, "HEAD");
|
|
427
434
|
if (!workspaceCheckoutDurabilityIsCurrent(resolvedWorkspacePath)) return false;
|
|
435
|
+
if (tolerateIdleRecordedBasis && lstatIfExists(hydrationTransactionPath(resolvedWorkspacePath)) !== null) return false;
|
|
428
436
|
const receipt = readHydratedWorkspaceReceipt(resolvedWorkspacePath);
|
|
429
437
|
if (!workspaceBasisRefsAreCurrent(resolvedWorkspacePath, receipt ?? { version: 3, mounts: [] })) return false;
|
|
430
438
|
if (head === null) return true;
|
|
431
439
|
if (!receipt) return false;
|
|
432
440
|
return mounts ? configuredWorkspaceHydrationBasisMounts(resolvedWorkspacePath, mounts).every(
|
|
433
|
-
(mount) => workspaceHydrationReceiptCoversMount(receipt, head, mount) || Boolean(mount.busy?.() && workspaceHydrationReceiptMountBasisHead(receipt, mount))
|
|
434
|
-
) : Boolean(receipt && receipt.mounts.every((mount) => mount.head === head));
|
|
441
|
+
(mount) => workspaceHydrationReceiptCoversMount(receipt, head, mount) || Boolean((tolerateIdleRecordedBasis || mount.busy?.()) && workspaceHydrationReceiptMountBasisHead(receipt, mount))
|
|
442
|
+
) : Boolean(receipt && receipt.mounts.every((mount) => tolerateIdleRecordedBasis || mount.head === head));
|
|
435
443
|
}
|
|
436
444
|
const WORKSPACE_GIT_BASIS_REF_PREFIX = "refs/r5d/workspace-basis/";
|
|
437
445
|
const WORKSPACE_GIT_BASIS_STAGING_REF_PREFIX = "refs/r5d/workspace-basis-staging/";
|
|
@@ -2190,6 +2198,7 @@ const workspaceGitSyncTestHarness = {
|
|
|
2190
2198
|
resetWorkspaceGit,
|
|
2191
2199
|
synchronizeWorkspaceGit,
|
|
2192
2200
|
workspaceGitHydrationIsCurrent,
|
|
2201
|
+
workspaceGitHydrationIsRecoverable,
|
|
2193
2202
|
workspaceGitMountsForRemediation,
|
|
2194
2203
|
workspaceGitSyncTestHarness
|
|
2195
2204
|
});
|
package/dist/mjs/main.mjs
CHANGED
|
@@ -88,7 +88,7 @@ import {
|
|
|
88
88
|
recoverWorkspaceGitHydration,
|
|
89
89
|
resetWorkspaceGit,
|
|
90
90
|
synchronizeWorkspaceGit,
|
|
91
|
-
|
|
91
|
+
workspaceGitHydrationIsRecoverable,
|
|
92
92
|
workspaceGitMountsForRemediation,
|
|
93
93
|
WorkspaceRemediationAncestryError
|
|
94
94
|
} from "./workspace-git-sync.mjs";
|
|
@@ -158,6 +158,11 @@ class StaleWorkerAdmissionError extends Error {
|
|
|
158
158
|
this.name = "StaleWorkerAdmissionError";
|
|
159
159
|
}
|
|
160
160
|
}
|
|
161
|
+
class WorkspaceSyncAdmissionChangedError extends Error {
|
|
162
|
+
name = "WorkspaceSyncAdmissionChangedError";
|
|
163
|
+
}
|
|
164
|
+
let unsafeWorkspaceSyncRecoveryAttempts = 0;
|
|
165
|
+
let unsafeWorkspaceSyncBouncePending = false;
|
|
161
166
|
function advanceWorkerAdmissionGeneration() {
|
|
162
167
|
workerAdmissionGeneration += 1;
|
|
163
168
|
return workerAdmissionGeneration;
|
|
@@ -1641,16 +1646,33 @@ function workspaceSyncFailureHydrationIsSafe(input) {
|
|
|
1641
1646
|
if (input.error instanceof WorkspaceRemediationAncestryError) return true;
|
|
1642
1647
|
if (input.resetToCanonical) return false;
|
|
1643
1648
|
try {
|
|
1644
|
-
return input.
|
|
1649
|
+
return input.inspectRecoverableHydration();
|
|
1645
1650
|
} catch {
|
|
1646
1651
|
return false;
|
|
1647
1652
|
}
|
|
1648
1653
|
}
|
|
1654
|
+
const WORKSPACE_UNSAFE_SYNC_MAX_RECOVERY_ATTEMPTS = 3;
|
|
1655
|
+
const WORKSPACE_UNSAFE_SYNC_RECOVERY_CLOSE_CODE = 4008;
|
|
1656
|
+
function unsafeWorkspaceSyncRecoveryBounceDelayMs(attempt) {
|
|
1657
|
+
return Math.min(1e3 * 2 ** Math.max(0, attempt - 1), 3e4);
|
|
1658
|
+
}
|
|
1659
|
+
function unsafeWorkspaceSyncFailureResponse(input) {
|
|
1660
|
+
if (input.resetToCanonical) return "fatal_exit";
|
|
1661
|
+
if (!input.hasRetainableWork) return "fatal_exit";
|
|
1662
|
+
if (input.recoveryAttempts >= input.maxRecoveryAttempts) return "fatal_exit";
|
|
1663
|
+
if (input.bouncePending) return "recovery_pending";
|
|
1664
|
+
return "recovery_bounce";
|
|
1665
|
+
}
|
|
1649
1666
|
function fenceUnsafeWorkspaceSyncFailure(input) {
|
|
1650
|
-
if (input.
|
|
1667
|
+
if (input.hydrationRecoverable) return "none";
|
|
1651
1668
|
input.invalidateExecution();
|
|
1652
|
-
|
|
1653
|
-
|
|
1669
|
+
const response = unsafeWorkspaceSyncFailureResponse(input);
|
|
1670
|
+
if (response === "fatal_exit") {
|
|
1671
|
+
requireCredentialGenerationRestart(true, input.fatalExit, input.exitCode);
|
|
1672
|
+
throw new Error("Unsafe workspace synchronization fence continued past its fatal exit");
|
|
1673
|
+
}
|
|
1674
|
+
if (response === "recovery_bounce") input.scheduleRecoveryBounce();
|
|
1675
|
+
return response;
|
|
1654
1676
|
}
|
|
1655
1677
|
const workerGitSecurityTestHarness = {
|
|
1656
1678
|
commandArgs: workerGitCommand.commandArgs,
|
|
@@ -1678,6 +1700,11 @@ const workerGitSecurityTestHarness = {
|
|
|
1678
1700
|
credentialReapContractStatus,
|
|
1679
1701
|
verifiedCredentialReapContract,
|
|
1680
1702
|
fenceUnsafeWorkspaceSyncFailure,
|
|
1703
|
+
unsafeWorkspaceSyncFailureResponse,
|
|
1704
|
+
unsafeWorkspaceSyncRecoveryBounceDelayMs,
|
|
1705
|
+
WORKSPACE_UNSAFE_SYNC_MAX_RECOVERY_ATTEMPTS,
|
|
1706
|
+
WORKSPACE_UNSAFE_SYNC_RECOVERY_CLOSE_CODE,
|
|
1707
|
+
WorkspaceSyncAdmissionChangedError,
|
|
1681
1708
|
workspaceConfigurationIncidentDeferral,
|
|
1682
1709
|
workspaceIncidentTerminalClearDisposition,
|
|
1683
1710
|
workspaceIncidentTerminalRefreshFence,
|
|
@@ -4294,6 +4321,28 @@ async function startWorker(options) {
|
|
|
4294
4321
|
);
|
|
4295
4322
|
}
|
|
4296
4323
|
};
|
|
4324
|
+
const scheduleUnsafeWorkspaceSyncRecoveryBounce = (cause) => {
|
|
4325
|
+
unsafeWorkspaceSyncBouncePending = true;
|
|
4326
|
+
const attempt = unsafeWorkspaceSyncRecoveryAttempts;
|
|
4327
|
+
if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
|
|
4328
|
+
process.stderr.write(
|
|
4329
|
+
`[r5d-worker] workspace synchronization failed without a recoverable hydration basis; awaiting reconnect recovery (attempt ${attempt} of ${WORKSPACE_UNSAFE_SYNC_MAX_RECOVERY_ATTEMPTS}): ${cause}
|
|
4330
|
+
`
|
|
4331
|
+
);
|
|
4332
|
+
return;
|
|
4333
|
+
}
|
|
4334
|
+
process.stderr.write(
|
|
4335
|
+
`[r5d-worker] workspace synchronization failed without a recoverable hydration basis; bouncing the worker socket for in-process recovery (attempt ${attempt} of ${WORKSPACE_UNSAFE_SYNC_MAX_RECOVERY_ATTEMPTS}): ${cause}
|
|
4336
|
+
`
|
|
4337
|
+
);
|
|
4338
|
+
const timer = setTimeout(() => {
|
|
4339
|
+
if (!unsafeWorkspaceSyncBouncePending) return;
|
|
4340
|
+
if (currentWorkerSocket === ws && (ws.readyState === WebSocket.OPEN || ws.readyState === WebSocket.CONNECTING)) {
|
|
4341
|
+
ws.close(WORKSPACE_UNSAFE_SYNC_RECOVERY_CLOSE_CODE, "Workspace synchronization failed without a recoverable hydration basis");
|
|
4342
|
+
}
|
|
4343
|
+
}, unsafeWorkspaceSyncRecoveryBounceDelayMs(attempt));
|
|
4344
|
+
timer.unref();
|
|
4345
|
+
};
|
|
4297
4346
|
const runWorkspaceSync = async (input) => {
|
|
4298
4347
|
const attemptId = input.attemptId ?? crypto.randomUUID();
|
|
4299
4348
|
const requestedAt = Date.now();
|
|
@@ -4302,7 +4351,7 @@ async function startWorker(options) {
|
|
|
4302
4351
|
const admittedDeferredIncidentId = deferredWorkspaceConfiguration?.incidentId ?? null;
|
|
4303
4352
|
const assertStillAdmitted = () => {
|
|
4304
4353
|
if (admittedGeneration !== workerAdmissionGeneration || admittedActiveIncidentId !== activeWorkspaceIncidentId || admittedDeferredIncidentId !== (deferredWorkspaceConfiguration?.incidentId ?? null) || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN || !deferredWorkspaceSyncTriggerIsAllowed(deferredWorkspaceConfiguration, activeWorkspaceIncidentId, input.trigger)) {
|
|
4305
|
-
throw new
|
|
4354
|
+
throw new WorkspaceSyncAdmissionChangedError("Workspace synchronization admission changed while asynchronous work was in flight");
|
|
4306
4355
|
}
|
|
4307
4356
|
};
|
|
4308
4357
|
if (!deferredWorkspaceSyncTriggerIsAllowed(deferredWorkspaceConfiguration, activeWorkspaceIncidentId, input.trigger)) {
|
|
@@ -4354,31 +4403,62 @@ async function startWorker(options) {
|
|
|
4354
4403
|
assertStillAdmitted
|
|
4355
4404
|
});
|
|
4356
4405
|
} catch (error) {
|
|
4357
|
-
const
|
|
4406
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
4407
|
+
const hydrationRecoverable = workspaceSyncFailureHydrationIsSafe({
|
|
4358
4408
|
error,
|
|
4359
4409
|
resetToCanonical: input.resetToCanonical === true,
|
|
4360
|
-
|
|
4410
|
+
inspectRecoverableHydration: () => workspaceGitHydrationIsRecoverable(workspaceShadowRoot, buildWorkspaceMounts())
|
|
4361
4411
|
});
|
|
4362
|
-
if (
|
|
4363
|
-
|
|
4364
|
-
|
|
4412
|
+
if (hydrationRecoverable) {
|
|
4413
|
+
if (error instanceof WorkspaceSyncAdmissionChangedError) {
|
|
4414
|
+
process.stderr.write(
|
|
4415
|
+
`[r5d-worker] workspace synchronization was cancelled with every mount at a recorded basis; the next cycle converges it: ${errorMessage}
|
|
4365
4416
|
`
|
|
4366
|
-
|
|
4417
|
+
);
|
|
4418
|
+
}
|
|
4419
|
+
return await failedWorkspaceSyncResult(attemptId, input.trigger, error);
|
|
4367
4420
|
}
|
|
4368
|
-
fenceUnsafeWorkspaceSyncFailure({
|
|
4369
|
-
|
|
4421
|
+
const fenced = fenceUnsafeWorkspaceSyncFailure({
|
|
4422
|
+
hydrationRecoverable,
|
|
4423
|
+
resetToCanonical: input.resetToCanonical === true,
|
|
4424
|
+
recoveryAttempts: unsafeWorkspaceSyncRecoveryAttempts,
|
|
4425
|
+
maxRecoveryAttempts: WORKSPACE_UNSAFE_SYNC_MAX_RECOVERY_ATTEMPTS,
|
|
4426
|
+
hasRetainableWork: activeProcesses.size > 0 || credentialBearingProcessGroups.size > 0 || activePtys.size > 0 || pendingProcessTerminals.size > 0,
|
|
4427
|
+
bouncePending: unsafeWorkspaceSyncBouncePending,
|
|
4370
4428
|
invalidateExecution: () => {
|
|
4371
4429
|
workspaceConfigured = false;
|
|
4372
4430
|
advanceWorkerAdmissionGeneration();
|
|
4373
4431
|
},
|
|
4432
|
+
scheduleRecoveryBounce: () => {
|
|
4433
|
+
unsafeWorkspaceSyncRecoveryAttempts += 1;
|
|
4434
|
+
scheduleUnsafeWorkspaceSyncRecoveryBounce(errorMessage);
|
|
4435
|
+
},
|
|
4436
|
+
fatalExit: (exitCode) => {
|
|
4437
|
+
process.stderr.write(
|
|
4438
|
+
`[r5d-worker] workspace synchronization failed without a recoverable hydration basis; exiting for exact recovery after ${unsafeWorkspaceSyncRecoveryAttempts} in-process recovery attempt(s): ${errorMessage}
|
|
4439
|
+
`
|
|
4440
|
+
);
|
|
4441
|
+
process.exit(exitCode);
|
|
4442
|
+
},
|
|
4374
4443
|
exitCode: 1
|
|
4375
4444
|
});
|
|
4376
|
-
|
|
4445
|
+
if (fenced === "recovery_pending") {
|
|
4446
|
+
process.stderr.write(
|
|
4447
|
+
`[r5d-worker] workspace synchronization failed without a recoverable hydration basis while in-process recovery is already pending: ${errorMessage}
|
|
4448
|
+
`
|
|
4449
|
+
);
|
|
4450
|
+
}
|
|
4451
|
+
return {
|
|
4452
|
+
...await failedWorkspaceSyncResult(attemptId, input.trigger, error),
|
|
4453
|
+
failureReason: "unsafe_hydration",
|
|
4454
|
+
recoveryAttempt: unsafeWorkspaceSyncRecoveryAttempts
|
|
4455
|
+
};
|
|
4377
4456
|
}
|
|
4378
4457
|
});
|
|
4379
4458
|
} finally {
|
|
4380
4459
|
workspaceSyncRequestsInFlight -= 1;
|
|
4381
4460
|
}
|
|
4461
|
+
if (result.outcome !== "failed") unsafeWorkspaceSyncRecoveryAttempts = 0;
|
|
4382
4462
|
const finishedAt = Date.now();
|
|
4383
4463
|
result = {
|
|
4384
4464
|
...result,
|
|
@@ -4833,7 +4913,9 @@ async function startWorker(options) {
|
|
|
4833
4913
|
const configurationSyncAdmissionGeneration = workerAdmissionGeneration;
|
|
4834
4914
|
const assertConfigurationSyncStillAdmitted = () => {
|
|
4835
4915
|
if (receiptGeneration !== workspaceConfigurationReceiptGeneration || configurationSyncAdmissionGeneration !== workerAdmissionGeneration || !workspaceConfigurationIncidentSnapshotIsCurrent(incidentDeferral.incidentId, activeWorkspaceIncidentId) || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
|
|
4836
|
-
throw new
|
|
4916
|
+
throw new WorkspaceSyncAdmissionChangedError(
|
|
4917
|
+
"Workspace configuration synchronization admission changed while asynchronous work was in flight"
|
|
4918
|
+
);
|
|
4837
4919
|
}
|
|
4838
4920
|
};
|
|
4839
4921
|
const result = await performWorkspaceSync({
|
|
@@ -5030,6 +5112,8 @@ async function startWorker(options) {
|
|
|
5030
5112
|
const refreshesDeferredConfiguration = deferredWorkspaceConfiguration !== null && activeWorkspaceIncidentId === null && message.deferWorkspaceSyncForIncidentId === void 0;
|
|
5031
5113
|
advanceWorkerAdmissionGeneration();
|
|
5032
5114
|
const configured = await configureWorkerWorkspace(message, receiptGeneration);
|
|
5115
|
+
unsafeWorkspaceSyncBouncePending = false;
|
|
5116
|
+
if (configured.result.outcome !== "failed") unsafeWorkspaceSyncRecoveryAttempts = 0;
|
|
5033
5117
|
sendWorkerMessageFromCurrentSource(ws, {
|
|
5034
5118
|
type: "workspace_configured",
|
|
5035
5119
|
requestId: message.requestId,
|
package/dist/mjs/package.json
CHANGED
|
@@ -376,18 +376,25 @@ function hydratedWorkspaceReceiptFileIsV3(workspacePath) {
|
|
|
376
376
|
}
|
|
377
377
|
}
|
|
378
378
|
function workspaceGitHydrationIsCurrent(workspacePath, mounts) {
|
|
379
|
+
return workspaceGitHydrationHolds(workspacePath, mounts, false);
|
|
380
|
+
}
|
|
381
|
+
function workspaceGitHydrationIsRecoverable(workspacePath, mounts) {
|
|
382
|
+
return workspaceGitHydrationHolds(workspacePath, mounts, true);
|
|
383
|
+
}
|
|
384
|
+
function workspaceGitHydrationHolds(workspacePath, mounts, tolerateIdleRecordedBasis) {
|
|
379
385
|
const resolvedWorkspacePath = path.resolve(workspacePath);
|
|
380
386
|
if (!fs.existsSync(path.join(resolvedWorkspacePath, ".git"))) return true;
|
|
381
387
|
if (mounts) validateMounts(resolvedWorkspacePath, mounts);
|
|
382
388
|
const head = revParse(resolvedWorkspacePath, "HEAD");
|
|
383
389
|
if (!workspaceCheckoutDurabilityIsCurrent(resolvedWorkspacePath)) return false;
|
|
390
|
+
if (tolerateIdleRecordedBasis && lstatIfExists(hydrationTransactionPath(resolvedWorkspacePath)) !== null) return false;
|
|
384
391
|
const receipt = readHydratedWorkspaceReceipt(resolvedWorkspacePath);
|
|
385
392
|
if (!workspaceBasisRefsAreCurrent(resolvedWorkspacePath, receipt ?? { version: 3, mounts: [] })) return false;
|
|
386
393
|
if (head === null) return true;
|
|
387
394
|
if (!receipt) return false;
|
|
388
395
|
return mounts ? configuredWorkspaceHydrationBasisMounts(resolvedWorkspacePath, mounts).every(
|
|
389
|
-
(mount) => workspaceHydrationReceiptCoversMount(receipt, head, mount) || Boolean(mount.busy?.() && workspaceHydrationReceiptMountBasisHead(receipt, mount))
|
|
390
|
-
) : Boolean(receipt && receipt.mounts.every((mount) => mount.head === head));
|
|
396
|
+
(mount) => workspaceHydrationReceiptCoversMount(receipt, head, mount) || Boolean((tolerateIdleRecordedBasis || mount.busy?.()) && workspaceHydrationReceiptMountBasisHead(receipt, mount))
|
|
397
|
+
) : Boolean(receipt && receipt.mounts.every((mount) => tolerateIdleRecordedBasis || mount.head === head));
|
|
391
398
|
}
|
|
392
399
|
const WORKSPACE_GIT_BASIS_REF_PREFIX = "refs/r5d/workspace-basis/";
|
|
393
400
|
const WORKSPACE_GIT_BASIS_STAGING_REF_PREFIX = "refs/r5d/workspace-basis-staging/";
|
|
@@ -2145,6 +2152,7 @@ export {
|
|
|
2145
2152
|
resetWorkspaceGit,
|
|
2146
2153
|
synchronizeWorkspaceGit,
|
|
2147
2154
|
workspaceGitHydrationIsCurrent,
|
|
2155
|
+
workspaceGitHydrationIsRecoverable,
|
|
2148
2156
|
workspaceGitMountsForRemediation,
|
|
2149
2157
|
workspaceGitSyncTestHarness
|
|
2150
2158
|
};
|
package/dist/types/main.d.ts
CHANGED
|
@@ -69,6 +69,10 @@ type WorkspaceSyncResult = {
|
|
|
69
69
|
synchronizeMs: number;
|
|
70
70
|
};
|
|
71
71
|
error?: string;
|
|
72
|
+
/** Typed reason for a failed outcome the server should surface loudly. */
|
|
73
|
+
failureReason?: "unsafe_hydration";
|
|
74
|
+
/** Consecutive in-process recovery bounces taken for unsafe hydration failures. */
|
|
75
|
+
recoveryAttempt?: number;
|
|
72
76
|
};
|
|
73
77
|
export type WorkerSessionTarget = {
|
|
74
78
|
type: "project";
|
|
@@ -356,6 +360,9 @@ type WorkerCodeReadResult = {
|
|
|
356
360
|
mtime: string;
|
|
357
361
|
base64: string;
|
|
358
362
|
};
|
|
363
|
+
declare class WorkspaceSyncAdmissionChangedError extends Error {
|
|
364
|
+
readonly name = "WorkspaceSyncAdmissionChangedError";
|
|
365
|
+
}
|
|
359
366
|
declare function assertWorkerChildAdmission(input: {
|
|
360
367
|
capturedGeneration: number;
|
|
361
368
|
currentGeneration: number;
|
|
@@ -609,14 +616,29 @@ declare function deferredIncidentWorkspaceConfigurationResult(input: {
|
|
|
609
616
|
declare function workspaceSyncFailureHydrationIsSafe(input: {
|
|
610
617
|
error: unknown;
|
|
611
618
|
resetToCanonical: boolean;
|
|
612
|
-
|
|
619
|
+
inspectRecoverableHydration: () => boolean;
|
|
613
620
|
}): boolean;
|
|
621
|
+
declare function unsafeWorkspaceSyncRecoveryBounceDelayMs(attempt: number): number;
|
|
622
|
+
type UnsafeWorkspaceSyncFailureResponse = "fatal_exit" | "recovery_bounce" | "recovery_pending";
|
|
623
|
+
declare function unsafeWorkspaceSyncFailureResponse(input: {
|
|
624
|
+
resetToCanonical: boolean;
|
|
625
|
+
recoveryAttempts: number;
|
|
626
|
+
maxRecoveryAttempts: number;
|
|
627
|
+
hasRetainableWork: boolean;
|
|
628
|
+
bouncePending: boolean;
|
|
629
|
+
}): UnsafeWorkspaceSyncFailureResponse;
|
|
614
630
|
declare function fenceUnsafeWorkspaceSyncFailure(input: {
|
|
615
|
-
|
|
631
|
+
hydrationRecoverable: boolean;
|
|
632
|
+
resetToCanonical: boolean;
|
|
633
|
+
recoveryAttempts: number;
|
|
634
|
+
maxRecoveryAttempts: number;
|
|
635
|
+
hasRetainableWork: boolean;
|
|
636
|
+
bouncePending: boolean;
|
|
616
637
|
invalidateExecution: () => void;
|
|
638
|
+
scheduleRecoveryBounce: () => void;
|
|
617
639
|
fatalExit?: (exitCode: number) => void;
|
|
618
640
|
exitCode?: number;
|
|
619
|
-
}):
|
|
641
|
+
}): "none" | "recovery_bounce" | "recovery_pending";
|
|
620
642
|
export declare const workerGitSecurityTestHarness: {
|
|
621
643
|
commandArgs: (args: string[]) => string[];
|
|
622
644
|
credentialHelperForRemotes: typeof credentialHelperForRemotes;
|
|
@@ -641,6 +663,11 @@ export declare const workerGitSecurityTestHarness: {
|
|
|
641
663
|
credentialReapContractStatus: typeof credentialReapContractStatus;
|
|
642
664
|
verifiedCredentialReapContract: typeof verifiedCredentialReapContract;
|
|
643
665
|
fenceUnsafeWorkspaceSyncFailure: typeof fenceUnsafeWorkspaceSyncFailure;
|
|
666
|
+
unsafeWorkspaceSyncFailureResponse: typeof unsafeWorkspaceSyncFailureResponse;
|
|
667
|
+
unsafeWorkspaceSyncRecoveryBounceDelayMs: typeof unsafeWorkspaceSyncRecoveryBounceDelayMs;
|
|
668
|
+
WORKSPACE_UNSAFE_SYNC_MAX_RECOVERY_ATTEMPTS: number;
|
|
669
|
+
WORKSPACE_UNSAFE_SYNC_RECOVERY_CLOSE_CODE: number;
|
|
670
|
+
WorkspaceSyncAdmissionChangedError: typeof WorkspaceSyncAdmissionChangedError;
|
|
644
671
|
workspaceConfigurationIncidentDeferral: typeof workspaceConfigurationIncidentDeferral;
|
|
645
672
|
workspaceIncidentTerminalClearDisposition: typeof workspaceIncidentTerminalClearDisposition;
|
|
646
673
|
workspaceIncidentTerminalRefreshFence: typeof workspaceIncidentTerminalRefreshFence;
|
|
@@ -71,6 +71,16 @@ export declare class WorkspaceRemediationAncestryError extends Error {
|
|
|
71
71
|
declare function gitCommandArgs(args: string[]): string[];
|
|
72
72
|
declare function workspaceCloneCommandArgs(args: string[], remoteUrl: string, credentialHelper?: string | null, credentialUsername?: string | null): string[];
|
|
73
73
|
export declare function workspaceGitHydrationIsCurrent(workspacePath: string, mounts?: readonly WorkspaceGitMount[]): boolean;
|
|
74
|
+
/**
|
|
75
|
+
* Failure-time variant of the currency check. A mount pinned at a valid
|
|
76
|
+
* recorded basis is convergent whether or not it is busy right now: the next
|
|
77
|
+
* cycle merges it forward from that exact basis and a genuine collision
|
|
78
|
+
* surfaces as a projection_merge conflict, never a silent overwrite. Live
|
|
79
|
+
* busy-ness must not decide safety here because the same disconnect that
|
|
80
|
+
* aborts a sync also clears the process/PTY maps busy-ness is computed from.
|
|
81
|
+
* Configuration-time recovery keeps the strict predicate.
|
|
82
|
+
*/
|
|
83
|
+
export declare function workspaceGitHydrationIsRecoverable(workspacePath: string, mounts?: readonly WorkspaceGitMount[]): boolean;
|
|
74
84
|
declare function configureWorkspaceRepository(input: {
|
|
75
85
|
workspacePath: string;
|
|
76
86
|
remoteUrl: string;
|