@ricsam/r5d-worker 0.0.82 → 0.0.83
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/main.cjs +422 -30
- package/dist/cjs/package.json +1 -1
- package/dist/cjs/workspace-git-sync.cjs +69 -1
- package/dist/mjs/main.mjs +425 -31
- package/dist/mjs/package.json +1 -1
- package/dist/mjs/workspace-git-sync.mjs +67 -1
- package/dist/types/main.d.ts +332 -0
- package/dist/types/workspace-git-sync.d.ts +28 -0
- package/package.json +1 -1
package/dist/mjs/main.mjs
CHANGED
|
@@ -79,12 +79,14 @@ import {
|
|
|
79
79
|
pruneAuthoritativelyDesiredBranchDeletions
|
|
80
80
|
} from "./project-workspace-state.mjs";
|
|
81
81
|
import {
|
|
82
|
+
configureExistingWorkspaceGitForRemediation,
|
|
82
83
|
ensureWorkspaceGitClone,
|
|
83
84
|
hydrateWorkspaceGitMounts,
|
|
84
85
|
recoverWorkspaceGitHydration,
|
|
85
86
|
resetWorkspaceGit,
|
|
86
87
|
synchronizeWorkspaceGit,
|
|
87
|
-
workspaceGitHydrationIsCurrent
|
|
88
|
+
workspaceGitHydrationIsCurrent,
|
|
89
|
+
WorkspaceRemediationAncestryError
|
|
88
90
|
} from "./workspace-git-sync.mjs";
|
|
89
91
|
class ProjectWorkspaceConfigurationDeferredError extends Error {
|
|
90
92
|
}
|
|
@@ -99,6 +101,7 @@ const DEFAULT_READ_MAX_BYTES = 5e4;
|
|
|
99
101
|
const MAX_LINE_LENGTH = 2e3;
|
|
100
102
|
const WORKSPACE_GIT_QUIET_MS = 5e3;
|
|
101
103
|
const WORKSPACE_GIT_PERIODIC_MS = 6e4;
|
|
104
|
+
const WORKSPACE_INCIDENT_CONFIG_REFRESH_TIMEOUT_MS = 6e4;
|
|
102
105
|
const PTY_INPUT_BUSY_GRACE_MS = 3e3;
|
|
103
106
|
const PTY_FOREGROUND_POLL_MS = 1e3;
|
|
104
107
|
const PTY_FOREGROUND_IDLE_ENABLED = process.env.R5D_PTY_FOREGROUND_IDLE !== "0";
|
|
@@ -257,6 +260,7 @@ const workerPtyTestHarness = {
|
|
|
257
260
|
removeEnvFiles: removePtyEnvFiles,
|
|
258
261
|
resolveEnvFileReferences: resolvePtyEnvFileReferences,
|
|
259
262
|
commandHasWorkspaceEffect: workerCommandHasWorkspaceEffect,
|
|
263
|
+
canonicalSyncTerminalHead,
|
|
260
264
|
ptyIsWorkspaceBusy: workerPtyIsWorkspaceBusy,
|
|
261
265
|
parseLinuxForegroundBusy: parseLinuxPtyForegroundBusy
|
|
262
266
|
};
|
|
@@ -1518,6 +1522,86 @@ function credentialReapContractStatus(probe) {
|
|
|
1518
1522
|
function verifiedCredentialReapContract(probe) {
|
|
1519
1523
|
return credentialReapContractStatus(probe) === "verified_systemd";
|
|
1520
1524
|
}
|
|
1525
|
+
function workspaceConfigurationIncidentDeferral(input) {
|
|
1526
|
+
const incidentId = input.requestedIncidentId ?? input.activeIncidentId;
|
|
1527
|
+
if (incidentId === null) return { incidentId: null };
|
|
1528
|
+
if (!/^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$/.test(incidentId)) {
|
|
1529
|
+
return { incidentId: null, error: new Error("Workspace configuration incident deferral requires a canonical incident UUID") };
|
|
1530
|
+
}
|
|
1531
|
+
if (input.requestedIncidentId !== void 0 && input.requestedIncidentId !== input.activeIncidentId) {
|
|
1532
|
+
return {
|
|
1533
|
+
incidentId: null,
|
|
1534
|
+
error: new Error(
|
|
1535
|
+
`Workspace configuration incident deferral ${input.requestedIncidentId} does not match active incident ${input.activeIncidentId ?? "none"}`
|
|
1536
|
+
)
|
|
1537
|
+
};
|
|
1538
|
+
}
|
|
1539
|
+
return { incidentId };
|
|
1540
|
+
}
|
|
1541
|
+
function workspaceIncidentTerminalClearDisposition(input) {
|
|
1542
|
+
if (!input.deferredConfiguration) return "ordinary_resume";
|
|
1543
|
+
void input.clearedIncidentId;
|
|
1544
|
+
return input.deferredConfiguration.serverRefreshExpected ? "await_server_refresh" : "reconnect_for_refresh";
|
|
1545
|
+
}
|
|
1546
|
+
function deferredWorkspaceTargetIsAllowed(deferredConfiguration, activeIncidentId, target) {
|
|
1547
|
+
const canonicalTarget = target.type === "workspace" && target.rootProfile === "canonical_sync";
|
|
1548
|
+
if (deferredConfiguration) return activeIncidentId === deferredConfiguration.incidentId && canonicalTarget;
|
|
1549
|
+
return activeIncidentId === null || canonicalTarget;
|
|
1550
|
+
}
|
|
1551
|
+
function deferredWorkspaceSyncTriggerIsAllowed(deferredConfiguration, activeIncidentId, trigger) {
|
|
1552
|
+
const incidentId = deferredConfiguration?.incidentId ?? activeIncidentId;
|
|
1553
|
+
return incidentId === null || (deferredConfiguration === null || activeIncidentId === deferredConfiguration.incidentId) && (trigger.type === "remediation" || trigger.type === "remediation_confirm" || trigger.type === "remediation_reset");
|
|
1554
|
+
}
|
|
1555
|
+
function workspaceOperationsAreFenced(deferredConfiguration, activeIncidentId) {
|
|
1556
|
+
return deferredConfiguration !== null || activeIncidentId !== null;
|
|
1557
|
+
}
|
|
1558
|
+
function workspaceCommandTransportIsAllowed(deferredConfiguration, activeIncidentId, transport) {
|
|
1559
|
+
return !workspaceOperationsAreFenced(deferredConfiguration, activeIncidentId) || transport === "exec_start";
|
|
1560
|
+
}
|
|
1561
|
+
function deferredCredentialTransitionMustWait(input) {
|
|
1562
|
+
return input.incidentId !== null && input.transitionPhase !== "current" && input.canonicalRemediationActive;
|
|
1563
|
+
}
|
|
1564
|
+
function workspaceConfigurationIncidentSnapshotIsCurrent(capturedIncidentId, activeIncidentId) {
|
|
1565
|
+
return capturedIncidentId === null ? activeIncidentId === null : activeIncidentId === null || activeIncidentId === capturedIncidentId;
|
|
1566
|
+
}
|
|
1567
|
+
function deferredWorkspaceRefreshWatchdogIsCurrent(input) {
|
|
1568
|
+
return input.deferredConfiguration?.incidentId === input.capturedIncidentId && input.activeIncidentId === null;
|
|
1569
|
+
}
|
|
1570
|
+
function incidentDeferredProjectCheckouts(projects) {
|
|
1571
|
+
return projects.flatMap(
|
|
1572
|
+
(project) => project.executionDisabled ? [] : project.branches.map(({ branchName }) => ({ projectId: project.projectId, branchName }))
|
|
1573
|
+
).sort((left, right) => left.projectId.localeCompare(right.projectId) || left.branchName.localeCompare(right.branchName));
|
|
1574
|
+
}
|
|
1575
|
+
function deferredIncidentWorkspaceConfigurationResult(input) {
|
|
1576
|
+
return {
|
|
1577
|
+
type: "workspace_sync",
|
|
1578
|
+
attemptId: input.attemptId,
|
|
1579
|
+
workerLabel: input.workerLabel,
|
|
1580
|
+
trigger: { type: "connect", detail: "workspace synchronization deferred for active remediation" },
|
|
1581
|
+
outcome: "no_change",
|
|
1582
|
+
startingHead: input.head,
|
|
1583
|
+
...input.head ? { localHead: input.head } : {},
|
|
1584
|
+
rebaseCount: 0,
|
|
1585
|
+
diffSizeBytes: 0,
|
|
1586
|
+
gitStatus: "",
|
|
1587
|
+
affectedProjects: [],
|
|
1588
|
+
affectedPaths: [],
|
|
1589
|
+
activeMountIds: [],
|
|
1590
|
+
skippedMountIds: [...input.skippedMountIds].sort(),
|
|
1591
|
+
activeProjectBranchPublications: [],
|
|
1592
|
+
discardedPaths: [],
|
|
1593
|
+
localChangesDiscarded: false
|
|
1594
|
+
};
|
|
1595
|
+
}
|
|
1596
|
+
function workspaceSyncFailureHydrationIsSafe(input) {
|
|
1597
|
+
if (input.error instanceof WorkspaceRemediationAncestryError) return true;
|
|
1598
|
+
if (input.resetToCanonical) return false;
|
|
1599
|
+
try {
|
|
1600
|
+
return input.inspectCurrentHydration();
|
|
1601
|
+
} catch {
|
|
1602
|
+
return false;
|
|
1603
|
+
}
|
|
1604
|
+
}
|
|
1521
1605
|
function fenceUnsafeWorkspaceSyncFailure(input) {
|
|
1522
1606
|
if (input.hydrationCurrent) return false;
|
|
1523
1607
|
input.invalidateExecution();
|
|
@@ -1550,7 +1634,21 @@ const workerGitSecurityTestHarness = {
|
|
|
1550
1634
|
credentialReapContractStatus,
|
|
1551
1635
|
verifiedCredentialReapContract,
|
|
1552
1636
|
fenceUnsafeWorkspaceSyncFailure,
|
|
1637
|
+
workspaceConfigurationIncidentDeferral,
|
|
1638
|
+
workspaceIncidentTerminalClearDisposition,
|
|
1639
|
+
deferredWorkspaceTargetIsAllowed,
|
|
1640
|
+
deferredWorkspaceSyncTriggerIsAllowed,
|
|
1641
|
+
workspaceOperationsAreFenced,
|
|
1642
|
+
workspaceCommandTransportIsAllowed,
|
|
1643
|
+
deferredCredentialTransitionMustWait,
|
|
1644
|
+
workspaceConfigurationIncidentSnapshotIsCurrent,
|
|
1645
|
+
deferredWorkspaceRefreshWatchdogIsCurrent,
|
|
1646
|
+
incidentDeferredProjectCheckouts,
|
|
1647
|
+
deferredIncidentWorkspaceConfigurationResult,
|
|
1648
|
+
workspaceSyncFailureHydrationIsSafe,
|
|
1553
1649
|
assertWorkerChildAdmission,
|
|
1650
|
+
executeWriteFileOperation,
|
|
1651
|
+
executeEditFileOperation,
|
|
1554
1652
|
terminateCredentialBearingChildren,
|
|
1555
1653
|
terminateCredentialBearingChildrenWithRetention,
|
|
1556
1654
|
async commitCredentialGeneration(prepared, credential, children, beforeMutation, previousGenerationFenced = false) {
|
|
@@ -1701,6 +1799,15 @@ function hasProjectWorktree(checkoutPath) {
|
|
|
1701
1799
|
return false;
|
|
1702
1800
|
}
|
|
1703
1801
|
}
|
|
1802
|
+
function canonicalSyncTerminalHead(resolvedTarget, readHead = (rootPath) => runGit(["rev-parse", "HEAD"], { cwd: rootPath })) {
|
|
1803
|
+
if (resolvedTarget.target.type !== "workspace" || resolvedTarget.target.rootProfile !== "canonical_sync") return void 0;
|
|
1804
|
+
try {
|
|
1805
|
+
const head = readHead(resolvedTarget.rootPath);
|
|
1806
|
+
return /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(head) ? head : void 0;
|
|
1807
|
+
} catch {
|
|
1808
|
+
return void 0;
|
|
1809
|
+
}
|
|
1810
|
+
}
|
|
1704
1811
|
function describeWorkerSessionTarget(target) {
|
|
1705
1812
|
return target.type === "project" ? `${target.projectId}/${target.branchName}` : `${target.ownerUserId}/${target.rootProfile}`;
|
|
1706
1813
|
}
|
|
@@ -2005,8 +2112,10 @@ async function executeWriteFileOperation(input) {
|
|
|
2005
2112
|
planRoot: input.planRoot,
|
|
2006
2113
|
access: "write"
|
|
2007
2114
|
});
|
|
2115
|
+
input.assertAdmission();
|
|
2008
2116
|
const resolved = resolveWorkerFilePath(input.resolvedTarget.rootPath, input.message.filePath, builtInPaths);
|
|
2009
2117
|
return withFileMutationQueue(mutationQueueKey(input.resolvedTarget.target, resolved), async () => {
|
|
2118
|
+
input.assertAdmission();
|
|
2010
2119
|
return writeWorkerTextFile(input.resolvedTarget.rootPath, input.message.filePath, input.message.content, builtInPaths);
|
|
2011
2120
|
});
|
|
2012
2121
|
}
|
|
@@ -2022,8 +2131,10 @@ async function executeEditFileOperation(input) {
|
|
|
2022
2131
|
planRoot: input.planRoot,
|
|
2023
2132
|
access: "write"
|
|
2024
2133
|
});
|
|
2134
|
+
input.assertAdmission();
|
|
2025
2135
|
const resolved = resolveWorkerFilePath(input.resolvedTarget.rootPath, input.message.filePath, builtInPaths);
|
|
2026
2136
|
return withFileMutationQueue(mutationQueueKey(input.resolvedTarget.target, resolved), async () => {
|
|
2137
|
+
input.assertAdmission();
|
|
2027
2138
|
return editWorkerTextFile(input.resolvedTarget.rootPath, input.message.filePath, input.message.edits, builtInPaths);
|
|
2028
2139
|
});
|
|
2029
2140
|
}
|
|
@@ -2599,13 +2710,15 @@ async function executeStreamingCommand(input) {
|
|
|
2599
2710
|
});
|
|
2600
2711
|
})
|
|
2601
2712
|
]);
|
|
2713
|
+
const canonicalWorkspaceHead = canonicalSyncTerminalHead(input.resolvedTarget);
|
|
2602
2714
|
const terminal = {
|
|
2603
2715
|
type: "exec_exit",
|
|
2604
2716
|
runId: input.message.runId,
|
|
2605
2717
|
exitCode,
|
|
2606
2718
|
durationMs: Date.now() - startedAt,
|
|
2607
2719
|
...timedOut ? { timedOut: true } : {},
|
|
2608
|
-
...input.message.workspaceEffect === "none" ? { workspaceEffect: "none" } : {}
|
|
2720
|
+
...input.message.workspaceEffect === "none" ? { workspaceEffect: "none" } : {},
|
|
2721
|
+
...canonicalWorkspaceHead ? { canonicalWorkspaceHead } : {}
|
|
2609
2722
|
};
|
|
2610
2723
|
pendingProcessTerminals.set(input.message.runId, terminal);
|
|
2611
2724
|
sendWorkerMessage(input.ws, terminal);
|
|
@@ -2613,12 +2726,14 @@ async function executeStreamingCommand(input) {
|
|
|
2613
2726
|
if (!started && error instanceof StaleWorkerAdmissionError) throw error;
|
|
2614
2727
|
const message = error instanceof Error ? error.message : String(error);
|
|
2615
2728
|
if (started) {
|
|
2729
|
+
const canonicalWorkspaceHead = canonicalSyncTerminalHead(input.resolvedTarget);
|
|
2616
2730
|
const terminal = {
|
|
2617
2731
|
type: "exec_error",
|
|
2618
2732
|
runId: input.message.runId,
|
|
2619
2733
|
error: message,
|
|
2620
2734
|
durationMs: Date.now() - startedAt,
|
|
2621
|
-
...input.message.workspaceEffect === "none" ? { workspaceEffect: "none" } : {}
|
|
2735
|
+
...input.message.workspaceEffect === "none" ? { workspaceEffect: "none" } : {},
|
|
2736
|
+
...canonicalWorkspaceHead ? { canonicalWorkspaceHead } : {}
|
|
2622
2737
|
};
|
|
2623
2738
|
pendingProcessTerminals.set(input.message.runId, terminal);
|
|
2624
2739
|
sendWorkerMessage(input.ws, terminal);
|
|
@@ -3273,6 +3388,9 @@ async function startWorker(options) {
|
|
|
3273
3388
|
let workspaceGitIdentity = null;
|
|
3274
3389
|
let workspaceConfigured = false;
|
|
3275
3390
|
let activeWorkspaceIncidentId = null;
|
|
3391
|
+
let deferredWorkspaceConfiguration = null;
|
|
3392
|
+
let deferredWorkspaceConfigurationRefreshTimer;
|
|
3393
|
+
let workspaceConfigurationReceiptGeneration = 0;
|
|
3276
3394
|
let workspaceAutomaticTimer;
|
|
3277
3395
|
let workspacePeriodicTimer;
|
|
3278
3396
|
let pendingAutomaticTrigger;
|
|
@@ -3990,6 +4108,7 @@ async function startWorker(options) {
|
|
|
3990
4108
|
...result.conflictPaths ? { conflictPaths: result.conflictPaths } : {},
|
|
3991
4109
|
...result.conflictSnapshotRefs ? { conflictSnapshotRefs: result.conflictSnapshotRefs } : {},
|
|
3992
4110
|
...result.conflictKind ? { conflictKind: result.conflictKind } : {},
|
|
4111
|
+
...result.verifiedAncestorHeads ? { verifiedAncestorHeads: result.verifiedAncestorHeads } : {},
|
|
3993
4112
|
...result.error ? { error: result.error } : {}
|
|
3994
4113
|
};
|
|
3995
4114
|
};
|
|
@@ -4075,10 +4194,13 @@ async function startWorker(options) {
|
|
|
4075
4194
|
commitDetail: input.confirmationReason ?? input.trigger.detail,
|
|
4076
4195
|
allowLargeDiff: input.confirmedLargeDiff,
|
|
4077
4196
|
skipMountMirror: outerRemediation,
|
|
4197
|
+
requiredAncestorHeads: input.requiredAncestorHeads,
|
|
4198
|
+
assertStillAdmitted: input.assertStillAdmitted,
|
|
4078
4199
|
afterWorkspacePublished: outerRemediation ? void 0 : ({ activeMountIds, publishedHead }) => {
|
|
4079
4200
|
pushChangedProjectHeads(activeMountIds, publishedHead, publicationHeads, inboundMoveMountIds);
|
|
4080
4201
|
}
|
|
4081
4202
|
});
|
|
4203
|
+
input.assertStillAdmitted?.();
|
|
4082
4204
|
if (["no_change", "updated", "pushed"].includes(result.outcome) && !outerRemediation) {
|
|
4083
4205
|
applyObservedProjectHeadsAfterInboundWorkspace(observedProjectHeads, result.activeMountIds, false);
|
|
4084
4206
|
}
|
|
@@ -4101,6 +4223,28 @@ async function startWorker(options) {
|
|
|
4101
4223
|
const runWorkspaceSync = async (input) => {
|
|
4102
4224
|
const attemptId = input.attemptId ?? crypto.randomUUID();
|
|
4103
4225
|
const requestedAt = Date.now();
|
|
4226
|
+
const admittedGeneration = workerAdmissionGeneration;
|
|
4227
|
+
const admittedActiveIncidentId = activeWorkspaceIncidentId;
|
|
4228
|
+
const admittedDeferredIncidentId = deferredWorkspaceConfiguration?.incidentId ?? null;
|
|
4229
|
+
const assertStillAdmitted = () => {
|
|
4230
|
+
if (admittedGeneration !== workerAdmissionGeneration || admittedActiveIncidentId !== activeWorkspaceIncidentId || admittedDeferredIncidentId !== (deferredWorkspaceConfiguration?.incidentId ?? null) || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN || !deferredWorkspaceSyncTriggerIsAllowed(deferredWorkspaceConfiguration, activeWorkspaceIncidentId, input.trigger)) {
|
|
4231
|
+
throw new Error("Workspace synchronization admission changed while asynchronous work was in flight");
|
|
4232
|
+
}
|
|
4233
|
+
};
|
|
4234
|
+
if (!deferredWorkspaceSyncTriggerIsAllowed(deferredWorkspaceConfiguration, activeWorkspaceIncidentId, input.trigger)) {
|
|
4235
|
+
const result2 = {
|
|
4236
|
+
...failedWorkspaceSyncResult(
|
|
4237
|
+
attemptId,
|
|
4238
|
+
input.trigger,
|
|
4239
|
+
new Error(
|
|
4240
|
+
`Workspace synchronization is deferred for incident ${deferredWorkspaceConfiguration?.incidentId ?? activeWorkspaceIncidentId ?? "unknown"}; only remediation synchronization is allowed`
|
|
4241
|
+
)
|
|
4242
|
+
),
|
|
4243
|
+
telemetry: { totalMs: 0, queueMs: 0, prepareMs: 0, synchronizeMs: 0 }
|
|
4244
|
+
};
|
|
4245
|
+
if (input.sendResult !== false) sendWorkspaceSyncResult(input.requestId, result2);
|
|
4246
|
+
return result2;
|
|
4247
|
+
}
|
|
4104
4248
|
if (!input.requestId && input.sendResult !== false) {
|
|
4105
4249
|
if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
|
|
4106
4250
|
throw new Error("Cannot start autonomous workspace synchronization without an open worker control socket");
|
|
@@ -4115,23 +4259,32 @@ async function startWorker(options) {
|
|
|
4115
4259
|
result = await workspaceSyncSingleFlight.runExclusive(async () => {
|
|
4116
4260
|
queueEnteredAt = Date.now();
|
|
4117
4261
|
syncStartedAt = Date.now();
|
|
4262
|
+
if (!deferredWorkspaceSyncTriggerIsAllowed(deferredWorkspaceConfiguration, activeWorkspaceIncidentId, input.trigger)) {
|
|
4263
|
+
return failedWorkspaceSyncResult(
|
|
4264
|
+
attemptId,
|
|
4265
|
+
input.trigger,
|
|
4266
|
+
new Error(
|
|
4267
|
+
`Workspace synchronization was fenced while queued by incident ${deferredWorkspaceConfiguration?.incidentId ?? activeWorkspaceIncidentId ?? "unknown"}`
|
|
4268
|
+
)
|
|
4269
|
+
);
|
|
4270
|
+
}
|
|
4118
4271
|
try {
|
|
4272
|
+
assertStillAdmitted();
|
|
4119
4273
|
return await performWorkspaceSync({
|
|
4120
4274
|
attemptId,
|
|
4121
4275
|
trigger: input.trigger,
|
|
4122
4276
|
confirmedLargeDiff: input.confirmedLargeDiff,
|
|
4123
4277
|
confirmationReason: input.confirmationReason,
|
|
4124
|
-
resetToCanonical: input.resetToCanonical
|
|
4278
|
+
resetToCanonical: input.resetToCanonical,
|
|
4279
|
+
requiredAncestorHeads: input.requiredAncestorHeads,
|
|
4280
|
+
assertStillAdmitted
|
|
4125
4281
|
});
|
|
4126
4282
|
} catch (error) {
|
|
4127
|
-
|
|
4128
|
-
|
|
4129
|
-
|
|
4130
|
-
|
|
4131
|
-
|
|
4132
|
-
hydrationCurrent = false;
|
|
4133
|
-
}
|
|
4134
|
-
}
|
|
4283
|
+
const hydrationCurrent = workspaceSyncFailureHydrationIsSafe({
|
|
4284
|
+
error,
|
|
4285
|
+
resetToCanonical: input.resetToCanonical === true,
|
|
4286
|
+
inspectCurrentHydration: () => workspaceGitHydrationIsCurrent(workspaceShadowRoot, buildWorkspaceMounts())
|
|
4287
|
+
});
|
|
4135
4288
|
if (!hydrationCurrent) {
|
|
4136
4289
|
process.stderr.write(
|
|
4137
4290
|
`[r5d-worker] workspace synchronization failed with an incompletely hydrated visible tree; exiting for exact recovery: ${error instanceof Error ? error.message : String(error)}
|
|
@@ -4173,7 +4326,7 @@ async function startWorker(options) {
|
|
|
4173
4326
|
pendingCreatedBranchPublicationNotBefore
|
|
4174
4327
|
);
|
|
4175
4328
|
const initialDeferral = pendingBranchDeferral();
|
|
4176
|
-
if (!workspaceConfigured || activeWorkspaceIncidentId || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN || automaticSyncInFlight || initialDeferral?.kind === "active_target") {
|
|
4329
|
+
if (!workspaceConfigured || activeWorkspaceIncidentId || deferredWorkspaceConfiguration || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN || automaticSyncInFlight || initialDeferral?.kind === "active_target") {
|
|
4177
4330
|
return;
|
|
4178
4331
|
}
|
|
4179
4332
|
if (workspaceAutomaticTimer) clearTimeout(workspaceAutomaticTimer);
|
|
@@ -4181,7 +4334,8 @@ async function startWorker(options) {
|
|
|
4181
4334
|
() => {
|
|
4182
4335
|
workspaceAutomaticTimer = void 0;
|
|
4183
4336
|
const scheduledTrigger = pendingAutomaticTrigger;
|
|
4184
|
-
if (!scheduledTrigger || !workspaceConfigured || activeWorkspaceIncidentId || currentWorkerSocket !== ws)
|
|
4337
|
+
if (!scheduledTrigger || !workspaceConfigured || activeWorkspaceIncidentId || deferredWorkspaceConfiguration || currentWorkerSocket !== ws)
|
|
4338
|
+
return;
|
|
4185
4339
|
const currentDeferral = pendingBranchDeferral();
|
|
4186
4340
|
if (currentDeferral?.kind === "active_target") return;
|
|
4187
4341
|
if (currentDeferral?.kind === "creation_grace") {
|
|
@@ -4203,7 +4357,7 @@ async function startWorker(options) {
|
|
|
4203
4357
|
}).finally(() => {
|
|
4204
4358
|
automaticSyncInFlight = false;
|
|
4205
4359
|
heartbeatBusyGrace = grantWorkerHeartbeatBusyGrace(lastServerHeartbeatAt, heartbeatBusyGrace);
|
|
4206
|
-
if (pendingAutomaticTrigger && !activeWorkspaceIncidentId) {
|
|
4360
|
+
if (pendingAutomaticTrigger && !activeWorkspaceIncidentId && !deferredWorkspaceConfiguration) {
|
|
4207
4361
|
scheduleAutomaticWorkspaceSync(pendingAutomaticTrigger, WORKSPACE_GIT_QUIET_MS);
|
|
4208
4362
|
}
|
|
4209
4363
|
});
|
|
@@ -4219,6 +4373,11 @@ async function startWorker(options) {
|
|
|
4219
4373
|
const targetMayMutateVisibleWorkspace = (target) => target.type === "project" || target.rootProfile === "visible_projects";
|
|
4220
4374
|
const resolveMessageTarget = (target) => {
|
|
4221
4375
|
if (!workspaceConfigured) throw new Error("Worker workspace configuration has not completed successfully");
|
|
4376
|
+
if (!deferredWorkspaceTargetIsAllowed(deferredWorkspaceConfiguration, activeWorkspaceIncidentId, target)) {
|
|
4377
|
+
throw new Error(
|
|
4378
|
+
`Worker workspace configuration is fenced for incident ${deferredWorkspaceConfiguration?.incidentId ?? activeWorkspaceIncidentId ?? "unknown"}; only canonical remediation commands are allowed`
|
|
4379
|
+
);
|
|
4380
|
+
}
|
|
4222
4381
|
if (target.type === "project" && !readyProjectIds.has(target.projectId)) {
|
|
4223
4382
|
throw new Error(`Project ${target.projectId} is not ready on this worker`);
|
|
4224
4383
|
}
|
|
@@ -4232,7 +4391,26 @@ async function startWorker(options) {
|
|
|
4232
4391
|
projectConfigById
|
|
4233
4392
|
});
|
|
4234
4393
|
};
|
|
4235
|
-
const configureWorkerWorkspace = async (message) => {
|
|
4394
|
+
const configureWorkerWorkspace = async (message, receiptGeneration) => {
|
|
4395
|
+
const incidentDeferral = workspaceConfigurationIncidentDeferral({
|
|
4396
|
+
requestedIncidentId: message.deferWorkspaceSyncForIncidentId,
|
|
4397
|
+
activeIncidentId: activeWorkspaceIncidentId
|
|
4398
|
+
});
|
|
4399
|
+
if (incidentDeferral.error) {
|
|
4400
|
+
workspaceConfigured = false;
|
|
4401
|
+
return {
|
|
4402
|
+
result: failedWorkspaceSyncResult(crypto.randomUUID(), { type: "connect" }, incidentDeferral.error),
|
|
4403
|
+
pending: incidentDeferredProjectCheckouts(message.projects),
|
|
4404
|
+
aheadOfOriginBranches: []
|
|
4405
|
+
};
|
|
4406
|
+
}
|
|
4407
|
+
if (incidentDeferral.incidentId) {
|
|
4408
|
+
workspaceConfigured = false;
|
|
4409
|
+
deferredWorkspaceConfiguration = {
|
|
4410
|
+
incidentId: incidentDeferral.incidentId,
|
|
4411
|
+
serverRefreshExpected: message.deferWorkspaceSyncForIncidentId === incidentDeferral.incidentId
|
|
4412
|
+
};
|
|
4413
|
+
}
|
|
4236
4414
|
const incomingCredentialGenerationFingerprint = credentialGenerationFingerprint({
|
|
4237
4415
|
...message,
|
|
4238
4416
|
workerBaseUrl: baseUrl,
|
|
@@ -4243,6 +4421,22 @@ async function startWorker(options) {
|
|
|
4243
4421
|
pendingFingerprintAtProcessStart: pendingCredentialGenerationIntentAtProcessStart?.fingerprint ?? null,
|
|
4244
4422
|
incomingFingerprint: incomingCredentialGenerationFingerprint
|
|
4245
4423
|
});
|
|
4424
|
+
if (deferredCredentialTransitionMustWait({
|
|
4425
|
+
incidentId: incidentDeferral.incidentId,
|
|
4426
|
+
transitionPhase: credentialTransitionPhase,
|
|
4427
|
+
canonicalRemediationActive: canonicalWorkspaceMutationIsActive(activeWorkspaceMutationTargets())
|
|
4428
|
+
})) {
|
|
4429
|
+
return {
|
|
4430
|
+
result: failedWorkspaceSyncResult(
|
|
4431
|
+
crypto.randomUUID(),
|
|
4432
|
+
{ type: "connect" },
|
|
4433
|
+
new Error("Workspace credential rotation is deferred until the active canonical remediation command finishes")
|
|
4434
|
+
),
|
|
4435
|
+
pending: incidentDeferredProjectCheckouts(message.projects),
|
|
4436
|
+
aheadOfOriginBranches: [],
|
|
4437
|
+
...incidentDeferral.incidentId ? { deferredWorkspaceSyncForIncidentId: incidentDeferral.incidentId } : {}
|
|
4438
|
+
};
|
|
4439
|
+
}
|
|
4246
4440
|
const credentialReapStatus = credentialReapContractStatus();
|
|
4247
4441
|
const verifiedCredentialReapContractNow = credentialReapStatus === "verified_systemd";
|
|
4248
4442
|
const credentialRestartIsContained = credentialGenerationRestartIsContained({
|
|
@@ -4356,6 +4550,12 @@ async function startWorker(options) {
|
|
|
4356
4550
|
workspaceSyncRequestsInFlight += 1;
|
|
4357
4551
|
try {
|
|
4358
4552
|
return await workspaceSyncSingleFlight.runExclusive(async () => {
|
|
4553
|
+
if (receiptGeneration !== workspaceConfigurationReceiptGeneration) {
|
|
4554
|
+
throw new Error("Workspace configuration was superseded by a newer server generation");
|
|
4555
|
+
}
|
|
4556
|
+
if (!workspaceConfigurationIncidentSnapshotIsCurrent(incidentDeferral.incidentId, activeWorkspaceIncidentId)) {
|
|
4557
|
+
throw new Error("Workspace configuration was superseded by a different active workspace incident");
|
|
4558
|
+
}
|
|
4359
4559
|
for (const project of message.projects) assertRepositoryTransitionState(project);
|
|
4360
4560
|
const busyConfigurationChanges = busyProjectConfigurationChangeIds({
|
|
4361
4561
|
currentProjects: [...projectConfigById.values()],
|
|
@@ -4377,10 +4577,12 @@ async function startWorker(options) {
|
|
|
4377
4577
|
const preserveOnlyBranches = message.projects.flatMap(
|
|
4378
4578
|
(project) => project.preserveOnlyBranches.map(({ branchId, branchName }) => ({ branchId, projectId: project.projectId, branchName }))
|
|
4379
4579
|
);
|
|
4380
|
-
|
|
4381
|
-
|
|
4382
|
-
|
|
4383
|
-
|
|
4580
|
+
if (!incidentDeferral.incidentId) {
|
|
4581
|
+
projectWorkspaceState = projectWorkspaceStateStore.reconcile({
|
|
4582
|
+
desiredProjects: message.projects,
|
|
4583
|
+
preserveOnlyBranches
|
|
4584
|
+
});
|
|
4585
|
+
}
|
|
4384
4586
|
const stillPendingCreatedBranches = new Map(
|
|
4385
4587
|
projectWorkspaceState.locallyPendingCreatedBranches.map((branch) => [
|
|
4386
4588
|
pendingCreatedBranchKey(branch.projectId, branch.branchName),
|
|
@@ -4440,6 +4642,12 @@ async function startWorker(options) {
|
|
|
4440
4642
|
void 0,
|
|
4441
4643
|
credentialPublicationPreauthorized
|
|
4442
4644
|
);
|
|
4645
|
+
if (receiptGeneration !== workspaceConfigurationReceiptGeneration) {
|
|
4646
|
+
throw new Error("Workspace configuration was superseded by a newer server generation");
|
|
4647
|
+
}
|
|
4648
|
+
if (!workspaceConfigurationIncidentSnapshotIsCurrent(incidentDeferral.incidentId, activeWorkspaceIncidentId)) {
|
|
4649
|
+
throw new Error("Workspace configuration was superseded by a different active workspace incident");
|
|
4650
|
+
}
|
|
4443
4651
|
configuredCredentialGenerationFingerprint = incomingCredentialGenerationFingerprint;
|
|
4444
4652
|
pendingCredentialGenerationIntentAtProcessStart = null;
|
|
4445
4653
|
bootstrapCredentialGenerationFingerprintAtProcessStart = null;
|
|
@@ -4454,6 +4662,34 @@ async function startWorker(options) {
|
|
|
4454
4662
|
branches: project.branches.filter(({ branchName }) => !pendingMirrorDeletes.has(projectBranchKey(project.projectId, branchName)))
|
|
4455
4663
|
}));
|
|
4456
4664
|
const nextProjectConfigById = new Map(effectiveProjects.map((project) => [project.projectId, project]));
|
|
4665
|
+
if (incidentDeferral.incidentId) {
|
|
4666
|
+
configureExistingWorkspaceGitForRemediation({
|
|
4667
|
+
workspacePath: workspaceShadowRoot,
|
|
4668
|
+
remoteUrl: message.workspaceRemoteUrl,
|
|
4669
|
+
credentialHelper: nextWorkspaceCredentialHelper,
|
|
4670
|
+
credentialUsername: workerCredentialUsername,
|
|
4671
|
+
gitIdentity: message.gitIdentity
|
|
4672
|
+
});
|
|
4673
|
+
projectConfigById.clear();
|
|
4674
|
+
for (const project of effectiveProjects) projectConfigById.set(project.projectId, project);
|
|
4675
|
+
readyProjectIds.clear();
|
|
4676
|
+
reconciledProjectConfigFingerprints.clear();
|
|
4677
|
+
pendingCheckouts.clear();
|
|
4678
|
+
const pending2 = incidentDeferredProjectCheckouts(message.projects);
|
|
4679
|
+
for (const checkout of pending2) pendingCheckouts.set(projectBranchKey(checkout.projectId, checkout.branchName), checkout);
|
|
4680
|
+
workspaceConfigured = activeWorkspaceIncidentId === incidentDeferral.incidentId;
|
|
4681
|
+
return {
|
|
4682
|
+
result: deferredIncidentWorkspaceConfigurationResult({
|
|
4683
|
+
attemptId: crypto.randomUUID(),
|
|
4684
|
+
workerLabel: label,
|
|
4685
|
+
head: workspaceLocalHead(),
|
|
4686
|
+
skippedMountIds: buildWorkspaceMounts().map(({ id }) => id)
|
|
4687
|
+
}),
|
|
4688
|
+
pending: pending2,
|
|
4689
|
+
aheadOfOriginBranches: [],
|
|
4690
|
+
deferredWorkspaceSyncForIncidentId: incidentDeferral.incidentId
|
|
4691
|
+
};
|
|
4692
|
+
}
|
|
4457
4693
|
ensureWorkspaceGitClone({
|
|
4458
4694
|
workspacePath: workspaceShadowRoot,
|
|
4459
4695
|
remoteUrl: message.workspaceRemoteUrl,
|
|
@@ -4521,10 +4757,24 @@ async function startWorker(options) {
|
|
|
4521
4757
|
{ ignoreBusy: true }
|
|
4522
4758
|
);
|
|
4523
4759
|
ensureConfiguredProjects();
|
|
4760
|
+
const configurationSyncAdmissionGeneration = workerAdmissionGeneration;
|
|
4761
|
+
const assertConfigurationSyncStillAdmitted = () => {
|
|
4762
|
+
if (receiptGeneration !== workspaceConfigurationReceiptGeneration || configurationSyncAdmissionGeneration !== workerAdmissionGeneration || !workspaceConfigurationIncidentSnapshotIsCurrent(incidentDeferral.incidentId, activeWorkspaceIncidentId) || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
|
|
4763
|
+
throw new Error("Workspace configuration synchronization admission changed while asynchronous work was in flight");
|
|
4764
|
+
}
|
|
4765
|
+
};
|
|
4524
4766
|
const result = await performWorkspaceSync({
|
|
4525
4767
|
attemptId: crypto.randomUUID(),
|
|
4526
|
-
trigger: { type: "connect" }
|
|
4768
|
+
trigger: { type: "connect" },
|
|
4769
|
+
assertStillAdmitted: assertConfigurationSyncStillAdmitted
|
|
4527
4770
|
});
|
|
4771
|
+
assertConfigurationSyncStillAdmitted();
|
|
4772
|
+
if (receiptGeneration !== workspaceConfigurationReceiptGeneration) {
|
|
4773
|
+
throw new Error("Workspace configuration was superseded by a newer server generation");
|
|
4774
|
+
}
|
|
4775
|
+
if (!workspaceConfigurationIncidentSnapshotIsCurrent(incidentDeferral.incidentId, activeWorkspaceIncidentId)) {
|
|
4776
|
+
throw new Error("Workspace configuration was superseded by a different active workspace incident");
|
|
4777
|
+
}
|
|
4528
4778
|
const publishedHead = result.publishedHead ?? result.localHead ?? result.startingHead;
|
|
4529
4779
|
if (publishedHead && ["no_change", "published", "updated", "conflict_reset", "reset"].includes(result.outcome)) {
|
|
4530
4780
|
const activeMountIds = new Set(result.activeMountIds ?? []);
|
|
@@ -4541,6 +4791,11 @@ async function startWorker(options) {
|
|
|
4541
4791
|
});
|
|
4542
4792
|
}
|
|
4543
4793
|
}
|
|
4794
|
+
if (deferredWorkspaceConfigurationRefreshTimer) {
|
|
4795
|
+
clearTimeout(deferredWorkspaceConfigurationRefreshTimer);
|
|
4796
|
+
deferredWorkspaceConfigurationRefreshTimer = void 0;
|
|
4797
|
+
}
|
|
4798
|
+
deferredWorkspaceConfiguration = null;
|
|
4544
4799
|
workspaceConfigured = true;
|
|
4545
4800
|
const pending = [...pendingCheckouts.values()].sort(
|
|
4546
4801
|
(left, right) => left.projectId.localeCompare(right.projectId) || left.branchName.localeCompare(right.branchName)
|
|
@@ -4548,6 +4803,13 @@ async function startWorker(options) {
|
|
|
4548
4803
|
return { result, pending, aheadOfOriginBranches: collectAheadOfOriginBranches() };
|
|
4549
4804
|
});
|
|
4550
4805
|
} catch (error) {
|
|
4806
|
+
if (receiptGeneration !== workspaceConfigurationReceiptGeneration) {
|
|
4807
|
+
return {
|
|
4808
|
+
result: failedWorkspaceSyncResult(crypto.randomUUID(), { type: "connect" }, error),
|
|
4809
|
+
pending: incidentDeferredProjectCheckouts(message.projects),
|
|
4810
|
+
aheadOfOriginBranches: []
|
|
4811
|
+
};
|
|
4812
|
+
}
|
|
4551
4813
|
if (error instanceof RegistryAuthConfigurationError) {
|
|
4552
4814
|
workspaceConfigured = false;
|
|
4553
4815
|
githubCredential = null;
|
|
@@ -4602,7 +4864,7 @@ async function startWorker(options) {
|
|
|
4602
4864
|
workspaceAutomaticTimer = void 0;
|
|
4603
4865
|
}
|
|
4604
4866
|
void (async () => {
|
|
4605
|
-
if (workspaceConfigured && !activeWorkspaceIncidentId) {
|
|
4867
|
+
if (workspaceConfigured && !activeWorkspaceIncidentId && !deferredWorkspaceConfiguration) {
|
|
4606
4868
|
await runWorkspaceSync({
|
|
4607
4869
|
trigger: { type: "manual", detail: "graceful worker shutdown" }
|
|
4608
4870
|
});
|
|
@@ -4651,7 +4913,9 @@ async function startWorker(options) {
|
|
|
4651
4913
|
updateClis: true,
|
|
4652
4914
|
browserPortForwarding: true,
|
|
4653
4915
|
execStdinV1: true,
|
|
4654
|
-
ptyEnvFilesV1: true
|
|
4916
|
+
ptyEnvFilesV1: true,
|
|
4917
|
+
workspaceRemediationAncestorGuardV1: true,
|
|
4918
|
+
workspaceIncidentConfigDeferralV1: true
|
|
4655
4919
|
},
|
|
4656
4920
|
projectRoot: projectsRoot,
|
|
4657
4921
|
artifactRoot,
|
|
@@ -4687,26 +4951,35 @@ async function startWorker(options) {
|
|
|
4687
4951
|
return;
|
|
4688
4952
|
}
|
|
4689
4953
|
if (message.type === "workspace_config") {
|
|
4954
|
+
const receiptGeneration = ++workspaceConfigurationReceiptGeneration;
|
|
4955
|
+
const refreshesDeferredConfiguration = deferredWorkspaceConfiguration !== null && activeWorkspaceIncidentId === null && message.deferWorkspaceSyncForIncidentId === void 0;
|
|
4690
4956
|
advanceWorkerAdmissionGeneration();
|
|
4691
|
-
const configured = await configureWorkerWorkspace(message);
|
|
4957
|
+
const configured = await configureWorkerWorkspace(message, receiptGeneration);
|
|
4692
4958
|
sendWorkerMessageFromCurrentSource(ws, {
|
|
4693
4959
|
type: "workspace_configured",
|
|
4694
4960
|
requestId: message.requestId,
|
|
4695
4961
|
result: configured.result,
|
|
4696
4962
|
pendingCheckouts: configured.pending,
|
|
4697
|
-
aheadOfOriginBranches: configured.aheadOfOriginBranches
|
|
4963
|
+
aheadOfOriginBranches: configured.aheadOfOriginBranches,
|
|
4964
|
+
...configured.deferredWorkspaceSyncForIncidentId ? { deferredWorkspaceSyncForIncidentId: configured.deferredWorkspaceSyncForIncidentId } : {}
|
|
4698
4965
|
});
|
|
4699
4966
|
process.stdout.write(
|
|
4700
4967
|
`[r5d-worker] workspace configured: ${message.projects.length} project(s), ${configured.pending.length} pending checkout(s)
|
|
4701
4968
|
`
|
|
4702
4969
|
);
|
|
4970
|
+
if (refreshesDeferredConfiguration && configured.result.outcome === "failed" && deferredWorkspaceConfiguration !== null && currentWorkerSocket === ws) {
|
|
4971
|
+
workspaceConfigured = false;
|
|
4972
|
+
advanceWorkerAdmissionGeneration();
|
|
4973
|
+
ws.close(1012, "Deferred workspace configuration refresh failed");
|
|
4974
|
+
return;
|
|
4975
|
+
}
|
|
4703
4976
|
if (!workspacePeriodicTimer) {
|
|
4704
4977
|
workspacePeriodicTimer = setInterval(() => {
|
|
4705
4978
|
scheduleAutomaticWorkspaceSync({ type: "periodic", detail: "periodic workspace reconciliation" }, 0);
|
|
4706
4979
|
}, WORKSPACE_GIT_PERIODIC_MS);
|
|
4707
4980
|
workspacePeriodicTimer.unref();
|
|
4708
4981
|
}
|
|
4709
|
-
if (pendingAutomaticTrigger && !activeWorkspaceIncidentId) {
|
|
4982
|
+
if (pendingAutomaticTrigger && !activeWorkspaceIncidentId && !deferredWorkspaceConfiguration) {
|
|
4710
4983
|
scheduleAutomaticWorkspaceSync(pendingAutomaticTrigger, 0);
|
|
4711
4984
|
}
|
|
4712
4985
|
return;
|
|
@@ -4718,11 +4991,20 @@ async function startWorker(options) {
|
|
|
4718
4991
|
trigger: message.trigger,
|
|
4719
4992
|
confirmedLargeDiff: message.confirmedLargeDiff,
|
|
4720
4993
|
confirmationReason: message.confirmationReason,
|
|
4721
|
-
resetToCanonical: message.resetToCanonical
|
|
4994
|
+
resetToCanonical: message.resetToCanonical,
|
|
4995
|
+
requiredAncestorHeads: message.requiredAncestorHeads
|
|
4722
4996
|
});
|
|
4723
4997
|
return;
|
|
4724
4998
|
}
|
|
4725
4999
|
if (message.type === "create_project_branch") {
|
|
5000
|
+
if (workspaceOperationsAreFenced(deferredWorkspaceConfiguration, activeWorkspaceIncidentId)) {
|
|
5001
|
+
sendWorkerMessage(ws, {
|
|
5002
|
+
type: "operation_result",
|
|
5003
|
+
requestId: message.requestId,
|
|
5004
|
+
error: `Project operations are deferred for workspace incident ${deferredWorkspaceConfiguration?.incidentId ?? activeWorkspaceIncidentId}`
|
|
5005
|
+
});
|
|
5006
|
+
return;
|
|
5007
|
+
}
|
|
4726
5008
|
try {
|
|
4727
5009
|
const pendingBranch = {
|
|
4728
5010
|
branchId: message.branchId,
|
|
@@ -4730,6 +5012,11 @@ async function startWorker(options) {
|
|
|
4730
5012
|
branchName: message.targetBranch
|
|
4731
5013
|
};
|
|
4732
5014
|
const created = await workspaceSyncSingleFlight.runMutation(() => {
|
|
5015
|
+
if (workspaceOperationsAreFenced(deferredWorkspaceConfiguration, activeWorkspaceIncidentId)) {
|
|
5016
|
+
throw new Error(
|
|
5017
|
+
`Project branch creation was fenced while queued by workspace incident ${deferredWorkspaceConfiguration?.incidentId ?? activeWorkspaceIncidentId}`
|
|
5018
|
+
);
|
|
5019
|
+
}
|
|
4733
5020
|
const project = projectConfigById.get(message.projectId);
|
|
4734
5021
|
if (!project) throw new Error(`Project ${message.projectId} is missing from the worker workspace configuration`);
|
|
4735
5022
|
assertRepositoryExecutionEnabled(project);
|
|
@@ -4813,9 +5100,22 @@ async function startWorker(options) {
|
|
|
4813
5100
|
return;
|
|
4814
5101
|
}
|
|
4815
5102
|
if (message.type === "delete_project_branch") {
|
|
5103
|
+
if (workspaceOperationsAreFenced(deferredWorkspaceConfiguration, activeWorkspaceIncidentId)) {
|
|
5104
|
+
sendWorkerMessage(ws, {
|
|
5105
|
+
type: "operation_result",
|
|
5106
|
+
requestId: message.requestId,
|
|
5107
|
+
error: `Project operations are deferred for workspace incident ${deferredWorkspaceConfiguration?.incidentId ?? activeWorkspaceIncidentId}`
|
|
5108
|
+
});
|
|
5109
|
+
return;
|
|
5110
|
+
}
|
|
4816
5111
|
try {
|
|
4817
5112
|
const deletionKey = projectBranchKey(message.projectId, message.branchName);
|
|
4818
5113
|
const deletionNeedsSync = await workspaceSyncSingleFlight.runMutation(() => {
|
|
5114
|
+
if (workspaceOperationsAreFenced(deferredWorkspaceConfiguration, activeWorkspaceIncidentId)) {
|
|
5115
|
+
throw new Error(
|
|
5116
|
+
`Project branch deletion was fenced while queued by workspace incident ${deferredWorkspaceConfiguration?.incidentId ?? activeWorkspaceIncidentId}`
|
|
5117
|
+
);
|
|
5118
|
+
}
|
|
4819
5119
|
const project = projectConfigById.get(message.projectId);
|
|
4820
5120
|
if (!project) throw new Error(`Project ${message.projectId} is missing from the worker workspace configuration`);
|
|
4821
5121
|
assertRepositoryExecutionEnabled(project);
|
|
@@ -4915,8 +5215,38 @@ async function startWorker(options) {
|
|
|
4915
5215
|
}
|
|
4916
5216
|
if (message.type === "workspace_incident_updated") {
|
|
4917
5217
|
const previousIncidentId = activeWorkspaceIncidentId;
|
|
4918
|
-
|
|
5218
|
+
const nextIncidentId = applyWorkspaceIncidentUpdate(activeWorkspaceIncidentId, message);
|
|
5219
|
+
if (nextIncidentId !== previousIncidentId) advanceWorkerAdmissionGeneration();
|
|
5220
|
+
activeWorkspaceIncidentId = nextIncidentId;
|
|
4919
5221
|
if (previousIncidentId && !activeWorkspaceIncidentId && (message.status === "resolved" || message.status === "confirmed" || message.status === "reset")) {
|
|
5222
|
+
const disposition = workspaceIncidentTerminalClearDisposition({
|
|
5223
|
+
deferredConfiguration: deferredWorkspaceConfiguration,
|
|
5224
|
+
clearedIncidentId: previousIncidentId
|
|
5225
|
+
});
|
|
5226
|
+
if (disposition !== "ordinary_resume") {
|
|
5227
|
+
pendingAutomaticTrigger ??= { type: "periodic", detail: "resume after workspace incident" };
|
|
5228
|
+
workspaceConfigured = false;
|
|
5229
|
+
advanceWorkerAdmissionGeneration();
|
|
5230
|
+
if (disposition === "reconnect_for_refresh") {
|
|
5231
|
+
ws.close(1012, "Refreshing deferred workspace configuration");
|
|
5232
|
+
} else {
|
|
5233
|
+
if (deferredWorkspaceConfigurationRefreshTimer) clearTimeout(deferredWorkspaceConfigurationRefreshTimer);
|
|
5234
|
+
const deferredIncidentId = deferredWorkspaceConfiguration?.incidentId;
|
|
5235
|
+
deferredWorkspaceConfigurationRefreshTimer = setTimeout(() => {
|
|
5236
|
+
deferredWorkspaceConfigurationRefreshTimer = void 0;
|
|
5237
|
+
if (!deferredIncidentId || !deferredWorkspaceRefreshWatchdogIsCurrent({
|
|
5238
|
+
capturedIncidentId: deferredIncidentId,
|
|
5239
|
+
deferredConfiguration: deferredWorkspaceConfiguration,
|
|
5240
|
+
activeIncidentId: activeWorkspaceIncidentId
|
|
5241
|
+
}) || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
|
|
5242
|
+
return;
|
|
5243
|
+
}
|
|
5244
|
+
ws.close(1012, "Timed out waiting for deferred workspace configuration refresh");
|
|
5245
|
+
}, WORKSPACE_INCIDENT_CONFIG_REFRESH_TIMEOUT_MS);
|
|
5246
|
+
deferredWorkspaceConfigurationRefreshTimer.unref();
|
|
5247
|
+
}
|
|
5248
|
+
return;
|
|
5249
|
+
}
|
|
4920
5250
|
scheduleAutomaticWorkspaceSync(pendingAutomaticTrigger ?? { type: "periodic", detail: "resume after workspace incident" }, 0);
|
|
4921
5251
|
}
|
|
4922
5252
|
return;
|
|
@@ -5020,6 +5350,12 @@ async function startWorker(options) {
|
|
|
5020
5350
|
sendAck({ error: `Process run ${message.runId} is not active on this worker` });
|
|
5021
5351
|
return;
|
|
5022
5352
|
}
|
|
5353
|
+
if (!deferredWorkspaceTargetIsAllowed(deferredWorkspaceConfiguration, activeWorkspaceIncidentId, active.target)) {
|
|
5354
|
+
sendAck({
|
|
5355
|
+
error: `Process input is deferred for workspace incident ${deferredWorkspaceConfiguration?.incidentId ?? activeWorkspaceIncidentId ?? "unknown"}`
|
|
5356
|
+
});
|
|
5357
|
+
return;
|
|
5358
|
+
}
|
|
5023
5359
|
if (!active.interactive || !active.stdin) {
|
|
5024
5360
|
sendAck({
|
|
5025
5361
|
error: `Process run ${message.runId} has no open stdin. Start a new shell command with "interactive": true to write to its stdin.`
|
|
@@ -5065,6 +5401,15 @@ async function startWorker(options) {
|
|
|
5065
5401
|
});
|
|
5066
5402
|
return;
|
|
5067
5403
|
}
|
|
5404
|
+
if (workspaceOperationsAreFenced(deferredWorkspaceConfiguration, activeWorkspaceIncidentId)) {
|
|
5405
|
+
sendWorkerMessage(ws, {
|
|
5406
|
+
type: "pty_error",
|
|
5407
|
+
requestId: message.requestId,
|
|
5408
|
+
ptyId: message.ptyId,
|
|
5409
|
+
error: `Shells are deferred for workspace incident ${deferredWorkspaceConfiguration?.incidentId ?? activeWorkspaceIncidentId}`
|
|
5410
|
+
});
|
|
5411
|
+
return;
|
|
5412
|
+
}
|
|
5068
5413
|
await reserveWorkspaceCommandAfterCurrentSync(message.target, workspaceSyncSingleFlight, () => {
|
|
5069
5414
|
workspaceSyncPriorityPtyTargets.set(message.ptyId, message.target);
|
|
5070
5415
|
if (targetMayMutateVisibleWorkspace(message.target)) recordVisibleWorkspaceMutation(message.target);
|
|
@@ -5073,6 +5418,11 @@ async function startWorker(options) {
|
|
|
5073
5418
|
let mutationLeaseTransferred = false;
|
|
5074
5419
|
try {
|
|
5075
5420
|
releaseWorkspaceMutation = await acquireWorkspaceCommandMutation(message.target, workspaceSyncSingleFlight);
|
|
5421
|
+
if (workspaceOperationsAreFenced(deferredWorkspaceConfiguration, activeWorkspaceIncidentId)) {
|
|
5422
|
+
throw new Error(
|
|
5423
|
+
`Shell opening was fenced while queued by workspace incident ${deferredWorkspaceConfiguration?.incidentId ?? activeWorkspaceIncidentId}`
|
|
5424
|
+
);
|
|
5425
|
+
}
|
|
5076
5426
|
const resolvedTarget = resolveMessageTarget(message.target);
|
|
5077
5427
|
process.stdout.write(`[r5d-worker] pty ${message.ptyId}: ${describeWorkerSessionTarget(message.target)}
|
|
5078
5428
|
`);
|
|
@@ -5105,6 +5455,14 @@ async function startWorker(options) {
|
|
|
5105
5455
|
}
|
|
5106
5456
|
if (message.type === "pty_input") {
|
|
5107
5457
|
const activePty = activePtys.get(message.ptyId);
|
|
5458
|
+
if (activePty && workspaceOperationsAreFenced(deferredWorkspaceConfiguration, activeWorkspaceIncidentId)) {
|
|
5459
|
+
sendWorkerMessage(ws, {
|
|
5460
|
+
type: "pty_error",
|
|
5461
|
+
ptyId: message.ptyId,
|
|
5462
|
+
error: `Shell input is deferred for workspace incident ${deferredWorkspaceConfiguration?.incidentId ?? activeWorkspaceIncidentId ?? "unknown"}`
|
|
5463
|
+
});
|
|
5464
|
+
return;
|
|
5465
|
+
}
|
|
5108
5466
|
if (activePty && targetMayMutateVisibleWorkspace(activePty.target)) {
|
|
5109
5467
|
markWorkspaceDirty({ type: "shell_inline", detail: `pty ${message.ptyId}` }, false, activePty.target);
|
|
5110
5468
|
}
|
|
@@ -5134,6 +5492,17 @@ async function startWorker(options) {
|
|
|
5134
5492
|
});
|
|
5135
5493
|
return;
|
|
5136
5494
|
}
|
|
5495
|
+
if (!workspaceCommandTransportIsAllowed(deferredWorkspaceConfiguration, activeWorkspaceIncidentId, "exec")) {
|
|
5496
|
+
sendWorkerMessage(ws, {
|
|
5497
|
+
type: "exec_result",
|
|
5498
|
+
requestId: message.requestId,
|
|
5499
|
+
stdout: "",
|
|
5500
|
+
stderr: "",
|
|
5501
|
+
exitCode: 1,
|
|
5502
|
+
error: `One-shot commands are deferred for workspace incident ${deferredWorkspaceConfiguration?.incidentId ?? activeWorkspaceIncidentId}`
|
|
5503
|
+
});
|
|
5504
|
+
return;
|
|
5505
|
+
}
|
|
5137
5506
|
let result;
|
|
5138
5507
|
let targetReserved = false;
|
|
5139
5508
|
const hasWorkspaceEffect = workerCommandHasWorkspaceEffect(message);
|
|
@@ -5146,6 +5515,11 @@ async function startWorker(options) {
|
|
|
5146
5515
|
});
|
|
5147
5516
|
}
|
|
5148
5517
|
const runCommand = async () => {
|
|
5518
|
+
if (!workspaceCommandTransportIsAllowed(deferredWorkspaceConfiguration, activeWorkspaceIncidentId, "exec")) {
|
|
5519
|
+
throw new Error(
|
|
5520
|
+
`One-shot command was fenced while queued by workspace incident ${deferredWorkspaceConfiguration?.incidentId ?? activeWorkspaceIncidentId}`
|
|
5521
|
+
);
|
|
5522
|
+
}
|
|
5149
5523
|
const resolvedTarget = resolveMessageTarget(message.target);
|
|
5150
5524
|
process.stdout.write(`[r5d-worker] exec ${message.runId}: ${message.argv.join(" ")}
|
|
5151
5525
|
`);
|
|
@@ -5156,7 +5530,14 @@ async function startWorker(options) {
|
|
|
5156
5530
|
token,
|
|
5157
5531
|
artifactRoot,
|
|
5158
5532
|
planRoot,
|
|
5159
|
-
assertAdmission:
|
|
5533
|
+
assertAdmission: () => {
|
|
5534
|
+
if (!workspaceCommandTransportIsAllowed(deferredWorkspaceConfiguration, activeWorkspaceIncidentId, "exec")) {
|
|
5535
|
+
throw new Error(
|
|
5536
|
+
`One-shot command was fenced before spawn by workspace incident ${deferredWorkspaceConfiguration?.incidentId ?? activeWorkspaceIncidentId}`
|
|
5537
|
+
);
|
|
5538
|
+
}
|
|
5539
|
+
assertMessageAdmission();
|
|
5540
|
+
}
|
|
5160
5541
|
});
|
|
5161
5542
|
};
|
|
5162
5543
|
result = await runWorkspaceCommand(message.target, workspaceSyncSingleFlight, runCommand);
|
|
@@ -5256,6 +5637,14 @@ async function startWorker(options) {
|
|
|
5256
5637
|
return;
|
|
5257
5638
|
}
|
|
5258
5639
|
if (message.type === "read" || message.type === "write" || message.type === "edit" || message.type === "grep" || message.type === "find" || message.type === "ls" || message.type === "view_file_bytes" || message.type === "code_list" || message.type === "code_read") {
|
|
5640
|
+
if (workspaceOperationsAreFenced(deferredWorkspaceConfiguration, activeWorkspaceIncidentId)) {
|
|
5641
|
+
sendWorkerMessage(ws, {
|
|
5642
|
+
type: "operation_result",
|
|
5643
|
+
requestId: message.requestId,
|
|
5644
|
+
error: `Workspace operations are deferred for incident ${deferredWorkspaceConfiguration?.incidentId ?? activeWorkspaceIncidentId}; use a canonical remediation shell`
|
|
5645
|
+
});
|
|
5646
|
+
return;
|
|
5647
|
+
}
|
|
5259
5648
|
const reservesVisibleWorkspace = targetMayMutateVisibleWorkspace(message.target);
|
|
5260
5649
|
const mutatesVisibleWorkspace = (message.type === "write" || message.type === "edit") && targetMayMutateVisibleWorkspace(message.target);
|
|
5261
5650
|
if (reservesVisibleWorkspace) {
|
|
@@ -5274,7 +5663,8 @@ async function startWorker(options) {
|
|
|
5274
5663
|
baseUrl,
|
|
5275
5664
|
token,
|
|
5276
5665
|
artifactRoot,
|
|
5277
|
-
planRoot
|
|
5666
|
+
planRoot,
|
|
5667
|
+
assertAdmission: assertMessageAdmission
|
|
5278
5668
|
});
|
|
5279
5669
|
});
|
|
5280
5670
|
ws.send(
|
|
@@ -5304,7 +5694,7 @@ async function startWorker(options) {
|
|
|
5304
5694
|
} finally {
|
|
5305
5695
|
if (reservesVisibleWorkspace) {
|
|
5306
5696
|
workspaceSyncPriorityOperationTargets.delete(message.requestId);
|
|
5307
|
-
if (pendingAutomaticTrigger && !activeWorkspaceIncidentId) {
|
|
5697
|
+
if (pendingAutomaticTrigger && !activeWorkspaceIncidentId && !deferredWorkspaceConfiguration) {
|
|
5308
5698
|
scheduleAutomaticWorkspaceSync(pendingAutomaticTrigger, dirtyTrigger ? WORKSPACE_GIT_QUIET_MS : 0);
|
|
5309
5699
|
}
|
|
5310
5700
|
}
|
|
@@ -5340,6 +5730,10 @@ async function startWorker(options) {
|
|
|
5340
5730
|
clearInterval(workspacePeriodicTimer);
|
|
5341
5731
|
workspacePeriodicTimer = void 0;
|
|
5342
5732
|
}
|
|
5733
|
+
if (deferredWorkspaceConfigurationRefreshTimer) {
|
|
5734
|
+
clearTimeout(deferredWorkspaceConfigurationRefreshTimer);
|
|
5735
|
+
deferredWorkspaceConfigurationRefreshTimer = void 0;
|
|
5736
|
+
}
|
|
5343
5737
|
if (terminalReplayTimer) {
|
|
5344
5738
|
clearInterval(terminalReplayTimer);
|
|
5345
5739
|
terminalReplayTimer = void 0;
|