@ricsam/r5d-worker 0.0.122 → 0.0.124
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -3
- package/dist/cjs/command-launcher.cjs +76 -2
- package/dist/cjs/control-command-policy.cjs +177 -0
- package/dist/cjs/main.cjs +541 -247
- package/dist/cjs/package.json +1 -1
- package/dist/cjs/project-checkout-garbage.cjs +32 -3
- package/dist/cjs/project-workspace-state.cjs +51 -35
- package/dist/cjs/project-worktrees.cjs +79 -34
- package/dist/cjs/recovery-journal-protocol.cjs +56 -0
- package/dist/cjs/recovery-journal-runtime.cjs +133 -0
- package/dist/cjs/recovery-journal-thread.cjs +9 -0
- package/dist/cjs/recovery-journal.cjs +735 -0
- package/dist/cjs/recovery-store.cjs +8 -0
- package/dist/cjs/session-file-mutations.cjs +61 -0
- package/dist/cjs/working-tree-mirror.cjs +1 -0
- package/dist/cjs/workspace-command-sync-policy.cjs +37 -8
- package/dist/cjs/workspace-filesystem-executor-thread.cjs +36 -0
- package/dist/cjs/workspace-filesystem-executor.cjs +327 -0
- package/dist/cjs/workspace-filesystem-job-types.cjs +134 -0
- package/dist/cjs/workspace-filesystem-jobs.cjs +57 -0
- package/dist/cjs/workspace-git-sync.cjs +275 -201
- package/dist/cjs/workspace-mount-hold-fence.cjs +120 -0
- package/dist/mjs/command-launcher.mjs +75 -2
- package/dist/mjs/control-command-policy.mjs +146 -0
- package/dist/mjs/main.mjs +549 -253
- package/dist/mjs/package.json +1 -1
- package/dist/mjs/project-checkout-garbage.mjs +30 -2
- package/dist/mjs/project-workspace-state.mjs +51 -35
- package/dist/mjs/project-worktrees.mjs +74 -34
- package/dist/mjs/recovery-journal-protocol.mjs +30 -0
- package/dist/mjs/recovery-journal-runtime.mjs +112 -0
- package/dist/mjs/recovery-journal-thread.mjs +8 -0
- package/dist/mjs/recovery-journal.mjs +687 -0
- package/dist/mjs/recovery-store.mjs +8 -0
- package/dist/mjs/session-file-mutations.mjs +37 -0
- package/dist/mjs/working-tree-mirror.mjs +1 -0
- package/dist/mjs/workspace-command-sync-policy.mjs +37 -8
- package/dist/mjs/workspace-filesystem-executor-thread.mjs +38 -0
- package/dist/mjs/workspace-filesystem-executor.mjs +287 -0
- package/dist/mjs/workspace-filesystem-job-types.mjs +106 -0
- package/dist/mjs/workspace-filesystem-jobs.mjs +51 -0
- package/dist/mjs/workspace-git-sync.mjs +264 -202
- package/dist/mjs/workspace-mount-hold-fence.mjs +95 -0
- package/dist/types/command-launcher.d.ts +42 -0
- package/dist/types/control-command-policy.d.ts +53 -0
- package/dist/types/main.d.ts +31 -19
- package/dist/types/project-checkout-garbage.d.ts +19 -2
- package/dist/types/project-workspace-state.d.ts +9 -9
- package/dist/types/project-worktrees.d.ts +49 -5
- package/dist/types/recovery-journal-protocol.d.ts +35 -0
- package/dist/types/recovery-journal-runtime.d.ts +12 -0
- package/dist/types/recovery-journal-stall-fixture.d.ts +1 -0
- package/dist/types/recovery-journal-thread.d.ts +1 -0
- package/dist/types/recovery-journal.d.ts +246 -0
- package/dist/types/recovery-store.d.ts +6 -0
- package/dist/types/session-file-mutations.d.ts +22 -0
- package/dist/types/workspace-command-sync-policy.d.ts +22 -7
- package/dist/types/workspace-filesystem-executor-thread.d.ts +1 -0
- package/dist/types/workspace-filesystem-executor.d.ts +123 -0
- package/dist/types/workspace-filesystem-job-types.d.ts +246 -0
- package/dist/types/workspace-filesystem-jobs.d.ts +7 -0
- package/dist/types/workspace-git-sync.d.ts +113 -7
- package/dist/types/workspace-mount-hold-fence.d.ts +42 -0
- package/package.json +1 -1
- package/dist/cjs/project-snapshot-recovery-runner.cjs +0 -171
- package/dist/mjs/project-snapshot-recovery-runner.mjs +0 -135
- package/dist/types/project-snapshot-recovery-runner.d.ts +0 -10
package/dist/mjs/main.mjs
CHANGED
|
@@ -4,7 +4,9 @@ import { getWorkerPlanPath, getWorkerPlansPath, validatePlanId } from "./plan-pa
|
|
|
4
4
|
import { createPlanMarkdown, updateTaskCheckbox } from "./plan-parser.mjs";
|
|
5
5
|
import { lintPlans, formatPlanLintErrors } from "./plan-lint.mjs";
|
|
6
6
|
import fs from "node:fs";
|
|
7
|
-
import {
|
|
7
|
+
import { WorkerRecoveryJournal, WorkerRecoveryJournalBusyError } from "./recovery-journal.mjs";
|
|
8
|
+
import { SessionFileMutationTracker } from "./session-file-mutations.mjs";
|
|
9
|
+
import { workerOperationDigest } from "./recovery-store.mjs";
|
|
8
10
|
import { WorkerOutputWindow } from "./recovery-output-window.mjs";
|
|
9
11
|
import {
|
|
10
12
|
createWorkerCommandLauncher,
|
|
@@ -26,6 +28,12 @@ import { installCliUpdate, readInstalledCliVersion } from "./cli-update.mjs";
|
|
|
26
28
|
import { gitTransportSecurityArgs, workerGitProcessEnvironment } from "./git-process-environment.mjs";
|
|
27
29
|
import { terminateProcessTree } from "./process-tree.mjs";
|
|
28
30
|
import { ProjectCheckoutGarbageCollector } from "./project-checkout-garbage.mjs";
|
|
31
|
+
import {
|
|
32
|
+
assertControlCommandArgv,
|
|
33
|
+
CONTROL_COMMAND_RUNTIME_MS,
|
|
34
|
+
controlCommandEnvironment,
|
|
35
|
+
resolveControlCommandExecutable
|
|
36
|
+
} from "./control-command-policy.mjs";
|
|
29
37
|
import { createPtyOutputCoalescer } from "./pty-output-coalescer.mjs";
|
|
30
38
|
import { openWorkerPortForwardRelay } from "./port-forward-client.mjs";
|
|
31
39
|
import {
|
|
@@ -52,9 +60,11 @@ import {
|
|
|
52
60
|
runWorkspaceCommand
|
|
53
61
|
} from "./workspace-command-sync-policy.mjs";
|
|
54
62
|
import { WorkspaceMutationGate } from "./workspace-mutation-gate.mjs";
|
|
63
|
+
import { WorkspaceMountHoldAbortedError, WorkspaceMountHoldFence } from "./workspace-mount-hold-fence.mjs";
|
|
64
|
+
import { installWorkspaceFilesystemExecutorFailureHandler, workspaceFilesystemExecutor } from "./workspace-filesystem-executor.mjs";
|
|
55
65
|
import { WorkspaceProjectionLedger } from "./workspace-projection-ledger.mjs";
|
|
56
66
|
import { WorkspaceSyncCoalescer } from "./workspace-sync-coalescer.mjs";
|
|
57
|
-
import { checkoutPathMovePublicationComplete
|
|
67
|
+
import { checkoutPathMovePublicationComplete } from "./workspace-path-move.mjs";
|
|
58
68
|
import { busyProjectConfigurationChangeIds, deferredProjectConfigurationPendingBranches } from "./workspace-project-config-policy.mjs";
|
|
59
69
|
import { creatorLocalProjectBranchIsAuthorized, workerProjectBranchDisposition } from "./workspace-preserve-only-policy.mjs";
|
|
60
70
|
import { assertProjectBranchDeletionIncarnation } from "./workspace-branch-incarnation-policy.mjs";
|
|
@@ -84,6 +94,7 @@ import {
|
|
|
84
94
|
deleteProjectMirrorBranch,
|
|
85
95
|
ensureProjectWorktrees,
|
|
86
96
|
inventoryProjectCheckouts,
|
|
97
|
+
PROJECT_WORKTREE_SNAPSHOT_OWNER_SESSION_ID,
|
|
87
98
|
observeProjectMirrorHeads,
|
|
88
99
|
projectBranchMayExistAfterCreateFailure,
|
|
89
100
|
projectOriginBranchName,
|
|
@@ -96,11 +107,6 @@ import {
|
|
|
96
107
|
parseProjectMirrorRefsTokensMessage,
|
|
97
108
|
planProjectMirrorObservationFetch
|
|
98
109
|
} from "./project-mirror-fetch-policy.mjs";
|
|
99
|
-
import {
|
|
100
|
-
PROJECT_SNAPSHOT_RECOVERY_HELPER_COMMAND,
|
|
101
|
-
recoverProjectSnapshotsInChild,
|
|
102
|
-
runProjectSnapshotRecoveryHelper
|
|
103
|
-
} from "./project-snapshot-recovery-runner.mjs";
|
|
104
110
|
import {
|
|
105
111
|
ProjectWorkspacePendingBranchPathChangeError,
|
|
106
112
|
ProjectWorkspaceStateStore,
|
|
@@ -129,7 +135,7 @@ function workerServerHeartbeatAction(input) {
|
|
|
129
135
|
return input.probeElapsedMs >= input.timeoutMs ? "terminate" : "wait";
|
|
130
136
|
}
|
|
131
137
|
const WORKSPACE_HYDRATION_LEASE_OPERATION_PREFIX = "hydration:";
|
|
132
|
-
function createWorkspaceHydrationLeaseHooks(send, options
|
|
138
|
+
function createWorkspaceHydrationLeaseHooks(send, options) {
|
|
133
139
|
const yieldToEventLoop = options.yieldToEventLoop ?? (() => new Promise((resolve) => setImmediate(resolve)));
|
|
134
140
|
const log = options.log ?? ((line) => process.stdout.write(`${line}
|
|
135
141
|
`));
|
|
@@ -150,13 +156,17 @@ function createWorkspaceHydrationLeaseHooks(send, options = {}) {
|
|
|
150
156
|
log(
|
|
151
157
|
`[r5d-worker] released hydration lease ${context.transactionId} after ${startedAt === void 0 ? "?" : Math.round(now() - startedAt)}ms`
|
|
152
158
|
);
|
|
159
|
+
},
|
|
160
|
+
holdMounts(mountIds, description) {
|
|
161
|
+
return options.fence.hold(mountIds, description);
|
|
153
162
|
}
|
|
154
163
|
};
|
|
155
164
|
}
|
|
156
165
|
const workerLifecycleTestHarness = {
|
|
157
166
|
boundedWorkerLifecycleProgress,
|
|
158
167
|
workerServerHeartbeatAction,
|
|
159
|
-
createWorkspaceHydrationLeaseHooks
|
|
168
|
+
createWorkspaceHydrationLeaseHooks,
|
|
169
|
+
workerCommandHasWorkspaceEffect
|
|
160
170
|
};
|
|
161
171
|
class ProjectWorkspaceConfigurationDeferredError extends Error {
|
|
162
172
|
}
|
|
@@ -177,6 +187,26 @@ const PTY_FOREGROUND_POLL_MS = 1e3;
|
|
|
177
187
|
const PTY_FOREGROUND_IDLE_ENABLED = process.env.R5D_PTY_FOREGROUND_IDLE !== "0";
|
|
178
188
|
const PTY_TMP_PATH_PREFIX = "r5d-worker-tmp://";
|
|
179
189
|
let workerCommandLauncher;
|
|
190
|
+
const trustedControlPath = process.env.PATH;
|
|
191
|
+
let capacityReportTimer;
|
|
192
|
+
let capacityReportSocket = null;
|
|
193
|
+
let lastCapacityReport;
|
|
194
|
+
function scheduleCapacityReport(force = false) {
|
|
195
|
+
if (force) lastCapacityReport = void 0;
|
|
196
|
+
if (capacityReportTimer) return;
|
|
197
|
+
capacityReportTimer = setTimeout(() => {
|
|
198
|
+
capacityReportTimer = void 0;
|
|
199
|
+
const ws = capacityReportSocket;
|
|
200
|
+
if (!ws || ws.readyState !== WebSocket.OPEN || !workerCommandLauncher) return;
|
|
201
|
+
const capacity = { ...workerCommandLauncher.capacityReport(), reportedAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
202
|
+
const { reportedAt: _reportedAt, ...comparable } = capacity;
|
|
203
|
+
const serialized = JSON.stringify(comparable);
|
|
204
|
+
if (serialized === lastCapacityReport) return;
|
|
205
|
+
lastCapacityReport = serialized;
|
|
206
|
+
sendWorkerMessage(ws, { type: "capacity_report", capacity });
|
|
207
|
+
}, 100);
|
|
208
|
+
capacityReportTimer.unref?.();
|
|
209
|
+
}
|
|
180
210
|
const activeProcesses = /* @__PURE__ */ new Map();
|
|
181
211
|
const credentialBearingProcessGroups = /* @__PURE__ */ new Map();
|
|
182
212
|
const credentialBearingProcessGroupTargets = /* @__PURE__ */ new Map();
|
|
@@ -189,7 +219,7 @@ const workspaceSyncPriorityPtyTargets = /* @__PURE__ */ new Map();
|
|
|
189
219
|
let currentWorkerSocket = null;
|
|
190
220
|
const workerRuntimeId = crypto.randomUUID();
|
|
191
221
|
const workerOutputWindow = new WorkerOutputWindow();
|
|
192
|
-
let
|
|
222
|
+
let recoveryJournal;
|
|
193
223
|
let workerReconnectAttempt = 0;
|
|
194
224
|
let workerRecoveryId;
|
|
195
225
|
let workerExecutionLeaseDeadline = 0;
|
|
@@ -201,13 +231,13 @@ let runtimeExitPromise;
|
|
|
201
231
|
function fenceWorkerRuntimeOperations() {
|
|
202
232
|
launcherFailureInProgress = true;
|
|
203
233
|
workerAdmissionGeneration += 1;
|
|
204
|
-
for (const operation of
|
|
205
|
-
if (operation.state !== "completed")
|
|
234
|
+
for (const operation of recoveryJournal?.activeAndPendingOperations() ?? []) {
|
|
235
|
+
if (operation.state !== "completed") recoveryJournal?.unknown(operation.requestId);
|
|
206
236
|
}
|
|
207
237
|
for (const runId of workerCommandLauncher?.activeIds ?? []) {
|
|
208
238
|
infrastructureStoppedRunIds.add(runId);
|
|
209
239
|
cancelledProcessRuns.add(runId);
|
|
210
|
-
|
|
240
|
+
recoveryJournal?.unknownRun(runId);
|
|
211
241
|
}
|
|
212
242
|
for (const runId of activeProcesses.keys()) infrastructureStoppedRunIds.add(runId);
|
|
213
243
|
}
|
|
@@ -216,13 +246,21 @@ function exitWorkerRuntime(code) {
|
|
|
216
246
|
fenceWorkerRuntimeOperations();
|
|
217
247
|
runtimeExitPromise = (async () => {
|
|
218
248
|
await Promise.race([
|
|
219
|
-
Promise.allSettled([terminateActiveCredentialBearingChildren(), workerCommandLauncher?.close()]),
|
|
249
|
+
Promise.allSettled([terminateActiveCredentialBearingChildren(), workerCommandLauncher?.close(), recoveryJournal?.flush()]),
|
|
220
250
|
Bun.sleep(4e3)
|
|
221
251
|
]);
|
|
222
252
|
process.exit(code);
|
|
223
253
|
})();
|
|
224
254
|
return runtimeExitPromise;
|
|
225
255
|
}
|
|
256
|
+
function handleRecoveryJournalFailure(error) {
|
|
257
|
+
if (launcherFailureInProgress) return;
|
|
258
|
+
fenceWorkerRuntimeOperations();
|
|
259
|
+
process.stderr.write(`[r5d-worker] ${error.message}; stopping runtime for recovery journal restart
|
|
260
|
+
`);
|
|
261
|
+
currentWorkerSocket?.close(1012, "Recovery journal failure");
|
|
262
|
+
void exitWorkerRuntime(WORKER_RECONNECT_EXIT_CODE);
|
|
263
|
+
}
|
|
226
264
|
function handleWorkerCommandLauncherFailure(error) {
|
|
227
265
|
if (!workerCommandLauncher || launcherFailureInProgress) return;
|
|
228
266
|
fenceWorkerRuntimeOperations();
|
|
@@ -243,7 +281,7 @@ async function retryWorkerServerRead(read) {
|
|
|
243
281
|
try {
|
|
244
282
|
return await read();
|
|
245
283
|
} catch (error) {
|
|
246
|
-
if (!
|
|
284
|
+
if (!recoveryJournal || !isWorkerCommunicationFailure(error instanceof Error ? error.message : String(error))) throw error;
|
|
247
285
|
deadline ??= Date.now() + WORKER_RECOVERY_TIMEOUT_MS;
|
|
248
286
|
if (workerLeaseExpired || Date.now() >= deadline) throw new WorkerServerUnavailableError("Worker communication recovery expired");
|
|
249
287
|
currentWorkerSocket?.close(1012, "Communication recovery");
|
|
@@ -265,9 +303,10 @@ function renewWorkerExecutionLease(leaseMs) {
|
|
|
265
303
|
`);
|
|
266
304
|
});
|
|
267
305
|
closeAllPtys();
|
|
268
|
-
for (const operation of
|
|
306
|
+
for (const operation of recoveryJournal?.activeAndPendingOperations() ?? []) {
|
|
269
307
|
if (operation.sessionId && operation.state !== "completed") {
|
|
270
|
-
|
|
308
|
+
void recoveryJournal?.cancelSession(operation.sessionId);
|
|
309
|
+
abortPendingReservations("Execution lease expired before command start", operation.sessionId);
|
|
271
310
|
void workerCommandLauncher?.cancelSession(operation.sessionId).catch((error) => {
|
|
272
311
|
process.stderr.write(`[r5d-worker] expired command launcher cancellation failed: ${error}
|
|
273
312
|
`);
|
|
@@ -277,7 +316,7 @@ function renewWorkerExecutionLease(leaseMs) {
|
|
|
277
316
|
for (const [runId, active] of activeProcesses) {
|
|
278
317
|
if (!active.sessionId) continue;
|
|
279
318
|
infrastructureStoppedRunIds.add(runId);
|
|
280
|
-
|
|
319
|
+
recoveryJournal?.unknownRun(runId);
|
|
281
320
|
cancelledProcessRuns.add(runId);
|
|
282
321
|
closeProcessStdin(active);
|
|
283
322
|
void terminateProcessTree(active.process).catch(
|
|
@@ -292,6 +331,31 @@ function renewWorkerExecutionLease(leaseMs) {
|
|
|
292
331
|
}
|
|
293
332
|
let workerAdmissionGeneration = 0;
|
|
294
333
|
const workspaceMutationGate = new WorkspaceMutationGate();
|
|
334
|
+
const workspaceMountHoldFence = new WorkspaceMountHoldFence();
|
|
335
|
+
const projectMountId = (projectId, branchName) => `project:${projectId}:${encodeURIComponent(branchName)}`;
|
|
336
|
+
const pendingReservationAborts = /* @__PURE__ */ new Map();
|
|
337
|
+
const sessionFileMutations = new SessionFileMutationTracker();
|
|
338
|
+
function registerPendingReservation(key, sessionId) {
|
|
339
|
+
abortPendingReservation(key, "Reservation superseded by a newer request with the same id");
|
|
340
|
+
const controller = new AbortController();
|
|
341
|
+
pendingReservationAborts.set(key, { controller, sessionId });
|
|
342
|
+
return controller;
|
|
343
|
+
}
|
|
344
|
+
function unregisterPendingReservation(key, controller) {
|
|
345
|
+
if (pendingReservationAborts.get(key)?.controller === controller) pendingReservationAborts.delete(key);
|
|
346
|
+
}
|
|
347
|
+
function abortPendingReservation(key, reason) {
|
|
348
|
+
const pending = pendingReservationAborts.get(key);
|
|
349
|
+
if (!pending) return;
|
|
350
|
+
pendingReservationAborts.delete(key);
|
|
351
|
+
pending.controller.abort(new WorkspaceMountHoldAbortedError(reason));
|
|
352
|
+
}
|
|
353
|
+
function abortPendingReservations(reason, sessionId) {
|
|
354
|
+
for (const [key, pending] of [...pendingReservationAborts]) {
|
|
355
|
+
if (sessionId !== void 0 && pending.sessionId !== sessionId) continue;
|
|
356
|
+
abortPendingReservation(key, reason);
|
|
357
|
+
}
|
|
358
|
+
}
|
|
295
359
|
let workspaceSyncQueue = Promise.resolve();
|
|
296
360
|
let startupProjectSnapshotRecoveryCompleted = false;
|
|
297
361
|
const checkoutGarbageCollectors = /* @__PURE__ */ new Map();
|
|
@@ -326,6 +390,17 @@ const workspaceSyncSingleFlight = {
|
|
|
326
390
|
},
|
|
327
391
|
afterCurrent() {
|
|
328
392
|
return workspaceSyncQueue;
|
|
393
|
+
},
|
|
394
|
+
mountHold(target, signal) {
|
|
395
|
+
if (target.type !== "project") return null;
|
|
396
|
+
const mountIds = [projectMountId(target.projectId, target.branchName)];
|
|
397
|
+
if (!workspaceMountHoldFence.isHeld(mountIds)) return null;
|
|
398
|
+
const holder = workspaceMountHoldFence.describeHold(mountIds);
|
|
399
|
+
process.stdout.write(
|
|
400
|
+
`[r5d-worker] command for ${target.projectId}/${target.branchName} waits for ${holder ?? "a workspace filesystem job"}
|
|
401
|
+
`
|
|
402
|
+
);
|
|
403
|
+
return workspaceMountHoldFence.waitForRelease(mountIds, { signal });
|
|
329
404
|
}
|
|
330
405
|
};
|
|
331
406
|
let githubCredential = null;
|
|
@@ -982,14 +1057,15 @@ async function runGitAsync(args, options = {}) {
|
|
|
982
1057
|
env: workerGitProcessEnvironment(),
|
|
983
1058
|
...bounded ? { detached: true } : {}
|
|
984
1059
|
});
|
|
985
|
-
const completion = Promise.all([
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
subprocess.exited
|
|
989
|
-
]).then(([stdout, stderr, exitCode]) => ({ stdout, stderr, exitCode }));
|
|
1060
|
+
const completion = Promise.all([new Response(subprocess.stdout).text(), new Response(subprocess.stderr).text(), subprocess.exited]).then(
|
|
1061
|
+
([stdout, stderr, exitCode]) => ({ stdout, stderr, exitCode })
|
|
1062
|
+
);
|
|
990
1063
|
let timer;
|
|
991
1064
|
const timedOut = bounded ? new Promise((_resolve, reject) => {
|
|
992
|
-
timer = setTimeout(
|
|
1065
|
+
timer = setTimeout(
|
|
1066
|
+
() => reject(new GitTransportTimeoutError(`git ${args.join(" ")} timed out after ${options.timeoutMs}ms`)),
|
|
1067
|
+
options.timeoutMs
|
|
1068
|
+
);
|
|
993
1069
|
}) : null;
|
|
994
1070
|
try {
|
|
995
1071
|
const { stdout, stderr, exitCode } = timedOut ? await Promise.race([completion, timedOut]) : await completion;
|
|
@@ -2078,12 +2154,12 @@ async function streamCommandOutput(stream, onData) {
|
|
|
2078
2154
|
}
|
|
2079
2155
|
const text = decoder.decode(chunk.value, { stream: true });
|
|
2080
2156
|
if (text.length > 0) {
|
|
2081
|
-
onData(text);
|
|
2157
|
+
await onData(text);
|
|
2082
2158
|
}
|
|
2083
2159
|
}
|
|
2084
2160
|
const trailing = decoder.decode();
|
|
2085
2161
|
if (trailing.length > 0) {
|
|
2086
|
-
onData(trailing);
|
|
2162
|
+
await onData(trailing);
|
|
2087
2163
|
}
|
|
2088
2164
|
} finally {
|
|
2089
2165
|
reader.releaseLock();
|
|
@@ -2222,12 +2298,21 @@ function mutationQueueKey(target, resolved) {
|
|
|
2222
2298
|
}
|
|
2223
2299
|
return `${describeWorkerSessionTarget(target)}:${path.posix.normalize(resolved.repoRelativePath)}`;
|
|
2224
2300
|
}
|
|
2225
|
-
function
|
|
2301
|
+
async function statIfExists(targetPath) {
|
|
2302
|
+
try {
|
|
2303
|
+
return await fs.promises.stat(targetPath);
|
|
2304
|
+
} catch (error) {
|
|
2305
|
+
const code = error.code;
|
|
2306
|
+
if (code === "ENOENT" || code === "ENOTDIR") return null;
|
|
2307
|
+
throw error;
|
|
2308
|
+
}
|
|
2309
|
+
}
|
|
2310
|
+
async function readWorkerTextFile(branchPath, filePath, offset, limit, builtInPaths) {
|
|
2226
2311
|
const resolved = resolveWorkerFilePath(branchPath, filePath, builtInPaths);
|
|
2227
|
-
if (!
|
|
2312
|
+
if (!(await statIfExists(resolved.absolutePath))?.isFile()) {
|
|
2228
2313
|
throw new Error(`File not found: ${resolved.displayPath}`);
|
|
2229
2314
|
}
|
|
2230
|
-
const buffer = fs.
|
|
2315
|
+
const buffer = await fs.promises.readFile(resolved.absolutePath);
|
|
2231
2316
|
const formatted = formatLineNumberedContent(buffer.toString("utf8"), offset, limit);
|
|
2232
2317
|
return {
|
|
2233
2318
|
type: "read",
|
|
@@ -2237,25 +2322,25 @@ function readWorkerTextFile(branchPath, filePath, offset, limit, builtInPaths) {
|
|
|
2237
2322
|
...formatted
|
|
2238
2323
|
};
|
|
2239
2324
|
}
|
|
2240
|
-
function writeWorkerTextFile(branchPath, filePath, content, builtInPaths) {
|
|
2325
|
+
async function writeWorkerTextFile(branchPath, filePath, content, builtInPaths) {
|
|
2241
2326
|
const resolved = resolveWorkerFilePath(branchPath, filePath, builtInPaths);
|
|
2242
|
-
fs.
|
|
2243
|
-
fs.
|
|
2327
|
+
await fs.promises.mkdir(path.dirname(resolved.absolutePath), { recursive: true });
|
|
2328
|
+
await fs.promises.writeFile(resolved.absolutePath, content, "utf8");
|
|
2244
2329
|
return {
|
|
2245
2330
|
type: "write",
|
|
2246
2331
|
file: resolved.displayPath,
|
|
2247
2332
|
gitBlobHash: getGitBlobHashForContent(content)
|
|
2248
2333
|
};
|
|
2249
2334
|
}
|
|
2250
|
-
function editWorkerTextFile(branchPath, filePath, edits, builtInPaths) {
|
|
2335
|
+
async function editWorkerTextFile(branchPath, filePath, edits, builtInPaths) {
|
|
2251
2336
|
const resolved = resolveWorkerFilePath(branchPath, filePath, builtInPaths);
|
|
2252
2337
|
if (!Array.isArray(edits) || edits.length === 0) {
|
|
2253
2338
|
throw new Error("edit requires at least one replacement");
|
|
2254
2339
|
}
|
|
2255
|
-
if (!
|
|
2340
|
+
if (!(await statIfExists(resolved.absolutePath))?.isFile()) {
|
|
2256
2341
|
throw new Error(`File not found: ${resolved.displayPath}`);
|
|
2257
2342
|
}
|
|
2258
|
-
const originalContent = fs.
|
|
2343
|
+
const originalContent = await fs.promises.readFile(resolved.absolutePath, "utf8");
|
|
2259
2344
|
const ranges = edits.map((edit, index) => {
|
|
2260
2345
|
if (edit.oldText === edit.newText) {
|
|
2261
2346
|
throw new Error(`edits[${index}].oldText and edits[${index}].newText must be different`);
|
|
@@ -2287,7 +2372,7 @@ function editWorkerTextFile(branchPath, filePath, edits, builtInPaths) {
|
|
|
2287
2372
|
for (const range of [...sortedRanges].reverse()) {
|
|
2288
2373
|
nextContent = nextContent.slice(0, range.start) + range.edit.newText + nextContent.slice(range.end);
|
|
2289
2374
|
}
|
|
2290
|
-
fs.
|
|
2375
|
+
await fs.promises.writeFile(resolved.absolutePath, nextContent, "utf8");
|
|
2291
2376
|
return {
|
|
2292
2377
|
type: "edit",
|
|
2293
2378
|
file: resolved.displayPath,
|
|
@@ -2320,7 +2405,7 @@ async function executePlanOperation(input) {
|
|
|
2320
2405
|
let created = false;
|
|
2321
2406
|
if (message.createIfMissing) {
|
|
2322
2407
|
try {
|
|
2323
|
-
fs.
|
|
2408
|
+
await fs.promises.writeFile(resolved.absolutePath, createPlanMarkdown(message.planId), { encoding: "utf8", flag: "wx" });
|
|
2324
2409
|
created = true;
|
|
2325
2410
|
} catch (error) {
|
|
2326
2411
|
if (error.code !== "EEXIST") throw error;
|
|
@@ -2329,11 +2414,16 @@ async function executePlanOperation(input) {
|
|
|
2329
2414
|
return {
|
|
2330
2415
|
type: "read_plan",
|
|
2331
2416
|
created,
|
|
2332
|
-
read: readWorkerTextFile(resolvedTarget.rootPath, filePath, void 0, void 0, builtInPaths)
|
|
2417
|
+
read: await readWorkerTextFile(resolvedTarget.rootPath, filePath, void 0, void 0, builtInPaths)
|
|
2333
2418
|
};
|
|
2334
2419
|
}
|
|
2335
|
-
const content = fs.
|
|
2336
|
-
writeWorkerTextFile(
|
|
2420
|
+
const content = await fs.promises.readFile(resolved.absolutePath, "utf8");
|
|
2421
|
+
await writeWorkerTextFile(
|
|
2422
|
+
resolvedTarget.rootPath,
|
|
2423
|
+
filePath,
|
|
2424
|
+
updateTaskCheckbox(content, message.taskId, message.checked),
|
|
2425
|
+
builtInPaths
|
|
2426
|
+
);
|
|
2337
2427
|
return {
|
|
2338
2428
|
type: "update_task_status",
|
|
2339
2429
|
planId: message.planId,
|
|
@@ -2354,7 +2444,11 @@ async function executeReadFileOperation(input) {
|
|
|
2354
2444
|
rootDir: input.rootDir,
|
|
2355
2445
|
access: "read"
|
|
2356
2446
|
});
|
|
2357
|
-
|
|
2447
|
+
const resolved = resolveWorkerFilePath(input.resolvedTarget.rootPath, input.message.filePath, builtInPaths);
|
|
2448
|
+
return withFileMutationQueue(
|
|
2449
|
+
mutationQueueKey(input.resolvedTarget.target, resolved),
|
|
2450
|
+
() => readWorkerTextFile(input.resolvedTarget.rootPath, input.message.filePath, input.message.offset, input.message.limit, builtInPaths)
|
|
2451
|
+
);
|
|
2358
2452
|
}
|
|
2359
2453
|
async function executeWriteFileOperation(input) {
|
|
2360
2454
|
const builtInPaths = await prepareBuiltInToolPathsForTarget({
|
|
@@ -2407,16 +2501,16 @@ function wildcardToRegex(pattern) {
|
|
|
2407
2501
|
function normalizeFindPattern(pattern) {
|
|
2408
2502
|
return /[*?]/.test(pattern) ? pattern : `*${pattern}*`;
|
|
2409
2503
|
}
|
|
2410
|
-
function walkWorkerEntries(branchPath, start) {
|
|
2504
|
+
async function walkWorkerEntries(branchPath, start) {
|
|
2411
2505
|
const entries = [];
|
|
2412
2506
|
const visitedDirectories = /* @__PURE__ */ new Set();
|
|
2413
|
-
const realVirtualRoot = start.scope === "virtual" ? fs.
|
|
2414
|
-
const visit = (absolutePath, isRoot) => {
|
|
2507
|
+
const realVirtualRoot = start.scope === "virtual" ? await fs.promises.realpath(start.virtualRootPath) : null;
|
|
2508
|
+
const visit = async (absolutePath, isRoot) => {
|
|
2415
2509
|
let stat;
|
|
2416
2510
|
try {
|
|
2417
|
-
stat = fs.
|
|
2511
|
+
stat = await fs.promises.stat(absolutePath);
|
|
2418
2512
|
if (realVirtualRoot) {
|
|
2419
|
-
assertInsideRoot(realVirtualRoot, fs.
|
|
2513
|
+
assertInsideRoot(realVirtualRoot, await fs.promises.realpath(absolutePath), `${start.displayPath} path`);
|
|
2420
2514
|
}
|
|
2421
2515
|
} catch (error) {
|
|
2422
2516
|
if (isRoot) throw error;
|
|
@@ -2429,7 +2523,7 @@ function walkWorkerEntries(branchPath, start) {
|
|
|
2429
2523
|
if (!stat.isDirectory()) return;
|
|
2430
2524
|
let realDirectoryPath;
|
|
2431
2525
|
try {
|
|
2432
|
-
realDirectoryPath = fs.
|
|
2526
|
+
realDirectoryPath = await fs.promises.realpath(absolutePath);
|
|
2433
2527
|
} catch (error) {
|
|
2434
2528
|
if (isRoot) throw error;
|
|
2435
2529
|
return;
|
|
@@ -2438,25 +2532,25 @@ function walkWorkerEntries(branchPath, start) {
|
|
|
2438
2532
|
visitedDirectories.add(realDirectoryPath);
|
|
2439
2533
|
let children;
|
|
2440
2534
|
try {
|
|
2441
|
-
children = fs.
|
|
2535
|
+
children = (await fs.promises.readdir(absolutePath, { withFileTypes: true })).sort((a, b) => a.name.localeCompare(b.name));
|
|
2442
2536
|
} catch (error) {
|
|
2443
2537
|
if (isRoot) throw error;
|
|
2444
2538
|
return;
|
|
2445
2539
|
}
|
|
2446
2540
|
for (const child of children) {
|
|
2447
2541
|
if (IGNORED_ENTRY_NAMES.has(child.name)) continue;
|
|
2448
|
-
visit(path.join(absolutePath, child.name), false);
|
|
2542
|
+
await visit(path.join(absolutePath, child.name), false);
|
|
2449
2543
|
}
|
|
2450
2544
|
};
|
|
2451
|
-
visit(start.absolutePath, true);
|
|
2545
|
+
await visit(start.absolutePath, true);
|
|
2452
2546
|
return entries;
|
|
2453
2547
|
}
|
|
2454
2548
|
function isProbablyText(bytes) {
|
|
2455
2549
|
return !bytes.subarray(0, Math.min(bytes.length, 4096)).includes(0);
|
|
2456
2550
|
}
|
|
2457
|
-
function grepWorkerFiles(branchPath, input, builtInPaths) {
|
|
2551
|
+
async function grepWorkerFiles(branchPath, input, builtInPaths) {
|
|
2458
2552
|
const start = resolveWorkerFilePath(branchPath, input.path ?? ".", builtInPaths);
|
|
2459
|
-
if (!
|
|
2553
|
+
if (!await statIfExists(start.absolutePath)) {
|
|
2460
2554
|
throw new Error(`Path not found: ${start.displayPath}`);
|
|
2461
2555
|
}
|
|
2462
2556
|
const limit = normalizeToolLimit(input.limit, 100, 1e3);
|
|
@@ -2466,12 +2560,12 @@ function grepWorkerFiles(branchPath, input, builtInPaths) {
|
|
|
2466
2560
|
const lines = [];
|
|
2467
2561
|
let matchCount = 0;
|
|
2468
2562
|
let truncated = false;
|
|
2469
|
-
for (const entry of walkWorkerEntries(branchPath, start)) {
|
|
2563
|
+
for (const entry of await walkWorkerEntries(branchPath, start)) {
|
|
2470
2564
|
if (!entry.stat.isFile()) continue;
|
|
2471
2565
|
if (globRegex && !globRegex.test(entry.matchPath)) continue;
|
|
2472
2566
|
let bytes;
|
|
2473
2567
|
try {
|
|
2474
|
-
bytes = fs.
|
|
2568
|
+
bytes = await fs.promises.readFile(entry.absolutePath);
|
|
2475
2569
|
} catch {
|
|
2476
2570
|
continue;
|
|
2477
2571
|
}
|
|
@@ -2524,9 +2618,9 @@ async function executeGrepOperation(input) {
|
|
|
2524
2618
|
builtInPaths
|
|
2525
2619
|
);
|
|
2526
2620
|
}
|
|
2527
|
-
function findWorkerFiles(branchPath, input, builtInPaths) {
|
|
2621
|
+
async function findWorkerFiles(branchPath, input, builtInPaths) {
|
|
2528
2622
|
const start = resolveWorkerFilePath(branchPath, input.path ?? ".", builtInPaths);
|
|
2529
|
-
if (!
|
|
2623
|
+
if (!(await statIfExists(start.absolutePath))?.isDirectory()) {
|
|
2530
2624
|
throw new Error(`Directory not found: ${start.displayPath}`);
|
|
2531
2625
|
}
|
|
2532
2626
|
const pattern = normalizeFindPattern(input.pattern ?? "*");
|
|
@@ -2536,7 +2630,7 @@ function findWorkerFiles(branchPath, input, builtInPaths) {
|
|
|
2536
2630
|
const matches = [];
|
|
2537
2631
|
let count = 0;
|
|
2538
2632
|
let truncated = false;
|
|
2539
|
-
for (const entry of walkWorkerEntries(branchPath, start)) {
|
|
2633
|
+
for (const entry of await walkWorkerEntries(branchPath, start)) {
|
|
2540
2634
|
if (entry.absolutePath === start.absolutePath) continue;
|
|
2541
2635
|
const isDirectory = entry.stat.isDirectory();
|
|
2542
2636
|
if (entryType === "file" && !entry.stat.isFile()) continue;
|
|
@@ -2581,13 +2675,13 @@ async function executeFindOperation(input) {
|
|
|
2581
2675
|
builtInPaths
|
|
2582
2676
|
);
|
|
2583
2677
|
}
|
|
2584
|
-
function listWorkerDirectory(branchPath, inputPath = ".", inputLimit, builtInPaths) {
|
|
2678
|
+
async function listWorkerDirectory(branchPath, inputPath = ".", inputLimit, builtInPaths) {
|
|
2585
2679
|
const resolved = resolveWorkerFilePath(branchPath, inputPath, builtInPaths);
|
|
2586
|
-
if (!
|
|
2680
|
+
if (!(await statIfExists(resolved.absolutePath))?.isDirectory()) {
|
|
2587
2681
|
throw new Error(`Directory not found: ${resolved.displayPath}`);
|
|
2588
2682
|
}
|
|
2589
2683
|
const limit = normalizeToolLimit(inputLimit, 200, 1e3);
|
|
2590
|
-
const entries = fs.
|
|
2684
|
+
const entries = (await fs.promises.readdir(resolved.absolutePath, { withFileTypes: true })).filter((entry) => !IGNORED_ENTRY_NAMES.has(entry.name)).sort((a, b) => Number(b.isDirectory()) - Number(a.isDirectory()) || a.name.localeCompare(b.name));
|
|
2591
2685
|
const displayed = entries.slice(0, limit).map((entry) => `${entry.name}${entry.isDirectory() ? "/" : ""}`);
|
|
2592
2686
|
const truncated = entries.length > displayed.length;
|
|
2593
2687
|
const hint = truncated ? "\n... truncated. Increase limit to continue." : "";
|
|
@@ -2638,36 +2732,40 @@ function resolveWorkerCodePath(branchPath, inputPath) {
|
|
|
2638
2732
|
}
|
|
2639
2733
|
return { absolutePath, displayPath };
|
|
2640
2734
|
}
|
|
2641
|
-
function listWorkerCodeDirectory(branchPath, inputPath) {
|
|
2735
|
+
async function listWorkerCodeDirectory(branchPath, inputPath) {
|
|
2642
2736
|
const resolved = resolveWorkerCodePath(branchPath, inputPath);
|
|
2643
|
-
if (!fs.
|
|
2737
|
+
if (!(await fs.promises.stat(resolved.absolutePath)).isDirectory()) {
|
|
2644
2738
|
throw new Error(`Code directory not found: ${resolved.displayPath}`);
|
|
2645
2739
|
}
|
|
2646
|
-
const
|
|
2740
|
+
const listed = (await fs.promises.readdir(resolved.absolutePath, { withFileTypes: true })).filter(
|
|
2741
|
+
(entry) => entry.name.toLowerCase() !== ".git"
|
|
2742
|
+
);
|
|
2743
|
+
const entries = [];
|
|
2744
|
+
for (const entry of listed) {
|
|
2647
2745
|
const entryPath = path.posix.join(resolved.displayPath, entry.name);
|
|
2648
2746
|
try {
|
|
2649
2747
|
const resolvedEntry = resolveWorkerCodePath(branchPath, entryPath);
|
|
2650
|
-
const stats = fs.
|
|
2651
|
-
if (!stats.isFile() && !stats.isDirectory())
|
|
2652
|
-
|
|
2653
|
-
|
|
2654
|
-
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
|
|
2658
|
-
}
|
|
2659
|
-
];
|
|
2748
|
+
const stats = await fs.promises.stat(resolvedEntry.absolutePath);
|
|
2749
|
+
if (!stats.isFile() && !stats.isDirectory()) continue;
|
|
2750
|
+
entries.push({
|
|
2751
|
+
name: entry.name,
|
|
2752
|
+
type: stats.isDirectory() ? "directory" : "file",
|
|
2753
|
+
size: stats.isDirectory() ? null : stats.size,
|
|
2754
|
+
mtime: stats.mtime.toISOString()
|
|
2755
|
+
});
|
|
2660
2756
|
} catch {
|
|
2661
|
-
return [];
|
|
2662
2757
|
}
|
|
2663
|
-
}
|
|
2758
|
+
}
|
|
2759
|
+
entries.sort(
|
|
2760
|
+
(left, right) => Number(right.type === "directory") - Number(left.type === "directory") || left.name.localeCompare(right.name)
|
|
2761
|
+
);
|
|
2664
2762
|
return { type: "code_list", path: resolved.displayPath, entries };
|
|
2665
2763
|
}
|
|
2666
|
-
function readWorkerCodeFile(branchPath, inputPath) {
|
|
2764
|
+
async function readWorkerCodeFile(branchPath, inputPath) {
|
|
2667
2765
|
const resolved = resolveWorkerCodePath(branchPath, inputPath);
|
|
2668
|
-
const stats = fs.
|
|
2766
|
+
const stats = await fs.promises.stat(resolved.absolutePath);
|
|
2669
2767
|
if (!stats.isFile()) throw new Error(`Code file not found: ${resolved.displayPath}`);
|
|
2670
|
-
const bytes = fs.
|
|
2768
|
+
const bytes = await fs.promises.readFile(resolved.absolutePath);
|
|
2671
2769
|
return {
|
|
2672
2770
|
type: "code_read",
|
|
2673
2771
|
path: resolved.displayPath,
|
|
@@ -2676,17 +2774,17 @@ function readWorkerCodeFile(branchPath, inputPath) {
|
|
|
2676
2774
|
base64: bytes.toString("base64")
|
|
2677
2775
|
};
|
|
2678
2776
|
}
|
|
2679
|
-
function executeCodeListOperation(input) {
|
|
2777
|
+
async function executeCodeListOperation(input) {
|
|
2680
2778
|
if (input.resolvedTarget.target.type !== "project") throw new Error("Code listing requires a project branch target");
|
|
2681
2779
|
return listWorkerCodeDirectory(input.resolvedTarget.rootPath, input.message.path);
|
|
2682
2780
|
}
|
|
2683
|
-
function executeCodeReadOperation(input) {
|
|
2781
|
+
async function executeCodeReadOperation(input) {
|
|
2684
2782
|
if (input.resolvedTarget.target.type !== "project") throw new Error("Code reading requires a project branch target");
|
|
2685
2783
|
return readWorkerCodeFile(input.resolvedTarget.rootPath, input.message.path);
|
|
2686
2784
|
}
|
|
2687
|
-
function readWorkerImageFile(branchPath, filePath, builtInPaths) {
|
|
2785
|
+
async function readWorkerImageFile(branchPath, filePath, builtInPaths) {
|
|
2688
2786
|
const resolved = resolveWorkerFilePath(branchPath, filePath, builtInPaths);
|
|
2689
|
-
if (!
|
|
2787
|
+
if (!(await statIfExists(resolved.absolutePath))?.isFile()) {
|
|
2690
2788
|
throw new Error(`File not found: ${resolved.displayPath}`);
|
|
2691
2789
|
}
|
|
2692
2790
|
const extension = path.extname(resolved.absolutePath).toLowerCase();
|
|
@@ -2694,7 +2792,7 @@ function readWorkerImageFile(branchPath, filePath, builtInPaths) {
|
|
|
2694
2792
|
if (!mediaType) {
|
|
2695
2793
|
throw new Error("read supports PNG and JPEG image inputs only");
|
|
2696
2794
|
}
|
|
2697
|
-
const bytes = fs.
|
|
2795
|
+
const bytes = await fs.promises.readFile(resolved.absolutePath);
|
|
2698
2796
|
const dimensions = readImageDimensions(bytes, mediaType);
|
|
2699
2797
|
return {
|
|
2700
2798
|
type: "view_file_bytes",
|
|
@@ -2794,6 +2892,10 @@ function cancelWorkerCommandLaunch(resources) {
|
|
|
2794
2892
|
});
|
|
2795
2893
|
}
|
|
2796
2894
|
async function acquireWorkerCommandLaunch(ws, message, assertAdmission) {
|
|
2895
|
+
if (message.type === "exec_start" && message.commandClass === "control") {
|
|
2896
|
+
assertControlCommandArgv(message.argv);
|
|
2897
|
+
if (message.interactive) throw new Error("Control commands cannot be interactive");
|
|
2898
|
+
}
|
|
2797
2899
|
const id = message.type === "pty_open" ? `pty:${message.ptyId}` : message.runId;
|
|
2798
2900
|
const operationId = message.type === "pty_open" ? id : `exec:${id}`;
|
|
2799
2901
|
sendWorkerMessage(ws, { type: "heartbeat_lease", operationId, active: true });
|
|
@@ -2802,6 +2904,7 @@ async function acquireWorkerCommandLaunch(ws, message, assertAdmission) {
|
|
|
2802
2904
|
id,
|
|
2803
2905
|
kind: message.type,
|
|
2804
2906
|
..."workspaceEffect" in message && message.workspaceEffect === "none" ? { workspaceEffect: "none" } : {},
|
|
2907
|
+
...message.type === "exec_start" && message.commandClass === "control" ? { commandClass: "control" } : {},
|
|
2805
2908
|
..."sessionId" in message && message.sessionId ? { sessionId: message.sessionId } : {},
|
|
2806
2909
|
assertAdmission: () => {
|
|
2807
2910
|
assertAdmission();
|
|
@@ -2949,8 +3052,17 @@ async function executeStreamingCommand(input) {
|
|
|
2949
3052
|
});
|
|
2950
3053
|
const cwd = resolveCommandCwd(input.resolvedTarget.rootPath, input.message.cwd);
|
|
2951
3054
|
const interactive = input.message.interactive === true;
|
|
3055
|
+
const control = input.message.commandClass === "control";
|
|
3056
|
+
let argv = input.message.argv;
|
|
3057
|
+
let environment = workerChildProcessEnvironment([githubProcessEnv(), input.message.env ?? {}, targetProcessEnv2]);
|
|
3058
|
+
if (control) {
|
|
3059
|
+
assertControlCommandArgv(argv);
|
|
3060
|
+
if (interactive) throw new Error("Control commands cannot be interactive");
|
|
3061
|
+
argv = [resolveControlCommandExecutable(trustedControlPath, (program, options) => Bun.which(program, options)), ...argv.slice(1)];
|
|
3062
|
+
environment = controlCommandEnvironment(environment, trustedControlPath);
|
|
3063
|
+
}
|
|
2952
3064
|
input.assertAdmission();
|
|
2953
|
-
const subprocess = Bun.spawn(input.resources.wrap(
|
|
3065
|
+
const subprocess = Bun.spawn(input.resources.wrap(argv), {
|
|
2954
3066
|
cwd,
|
|
2955
3067
|
// Without an explicit stdin the process reads /dev/null and interactive
|
|
2956
3068
|
// prompts see immediate EOF; "pipe" keeps stdin open for exec_stdin.
|
|
@@ -2958,7 +3070,7 @@ async function executeStreamingCommand(input) {
|
|
|
2958
3070
|
stdout: "pipe",
|
|
2959
3071
|
stderr: "pipe",
|
|
2960
3072
|
detached: true,
|
|
2961
|
-
env:
|
|
3073
|
+
env: environment
|
|
2962
3074
|
});
|
|
2963
3075
|
spawnedProcess = subprocess;
|
|
2964
3076
|
credentialBearingProcessGroups.set(subprocess.pid, subprocess);
|
|
@@ -2973,12 +3085,13 @@ async function executeStreamingCommand(input) {
|
|
|
2973
3085
|
pid: subprocess.pid,
|
|
2974
3086
|
processGroupId: process.platform === "win32" ? void 0 : subprocess.pid,
|
|
2975
3087
|
credentialId: input.message.credentialId,
|
|
2976
|
-
argv
|
|
3088
|
+
argv,
|
|
2977
3089
|
command: input.message.command,
|
|
2978
3090
|
cwd,
|
|
2979
3091
|
startedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2980
3092
|
...interactive ? { interactive: true, stdin: subprocess.stdin } : {},
|
|
2981
|
-
...input.message.workspaceEffect === "none" ? { workspaceEffect: "none" } : {}
|
|
3093
|
+
...input.message.workspaceEffect === "none" ? { workspaceEffect: "none" } : {},
|
|
3094
|
+
...control ? { commandClass: "control" } : {}
|
|
2982
3095
|
});
|
|
2983
3096
|
settlePreparation();
|
|
2984
3097
|
started = true;
|
|
@@ -2990,7 +3103,8 @@ async function executeStreamingCommand(input) {
|
|
|
2990
3103
|
pid: subprocess.pid,
|
|
2991
3104
|
...process.platform === "win32" ? {} : { processGroupId: subprocess.pid }
|
|
2992
3105
|
});
|
|
2993
|
-
|
|
3106
|
+
const timeoutMs = control ? Math.min(input.message.timeoutMs ?? CONTROL_COMMAND_RUNTIME_MS, CONTROL_COMMAND_RUNTIME_MS) : input.message.timeoutMs;
|
|
3107
|
+
if (timeoutMs) {
|
|
2994
3108
|
timeout = setTimeout(() => {
|
|
2995
3109
|
timedOut = true;
|
|
2996
3110
|
cancelWorkerCommandLaunch(input.resources);
|
|
@@ -3001,30 +3115,32 @@ async function executeStreamingCommand(input) {
|
|
|
3001
3115
|
`
|
|
3002
3116
|
);
|
|
3003
3117
|
});
|
|
3004
|
-
},
|
|
3118
|
+
}, timeoutMs);
|
|
3005
3119
|
}
|
|
3006
3120
|
const [exitCode] = await Promise.all([
|
|
3007
3121
|
subprocess.exited,
|
|
3008
|
-
streamCommandOutput(
|
|
3009
|
-
|
|
3122
|
+
streamCommandOutput(
|
|
3123
|
+
subprocess.stdout,
|
|
3124
|
+
(data) => sendWorkerOutput(input.ws, {
|
|
3010
3125
|
type: "exec_output",
|
|
3011
3126
|
runId: input.message.runId,
|
|
3012
3127
|
stream: "stdout",
|
|
3013
3128
|
data
|
|
3014
|
-
})
|
|
3015
|
-
|
|
3016
|
-
streamCommandOutput(
|
|
3017
|
-
|
|
3129
|
+
})
|
|
3130
|
+
),
|
|
3131
|
+
streamCommandOutput(
|
|
3132
|
+
subprocess.stderr,
|
|
3133
|
+
(data) => sendWorkerOutput(input.ws, {
|
|
3018
3134
|
type: "exec_output",
|
|
3019
3135
|
runId: input.message.runId,
|
|
3020
3136
|
stream: "stderr",
|
|
3021
3137
|
data
|
|
3022
|
-
})
|
|
3023
|
-
|
|
3138
|
+
})
|
|
3139
|
+
)
|
|
3024
3140
|
]);
|
|
3025
3141
|
const diagnostic = await input.resources.diagnose();
|
|
3026
3142
|
if (diagnostic) {
|
|
3027
|
-
|
|
3143
|
+
await sendWorkerOutput(input.ws, {
|
|
3028
3144
|
type: "exec_output",
|
|
3029
3145
|
runId: input.message.runId,
|
|
3030
3146
|
stream: "stderr",
|
|
@@ -3096,6 +3212,33 @@ async function reapCompletedCredentialBearingProcessGroup(runId, subprocess) {
|
|
|
3096
3212
|
);
|
|
3097
3213
|
}
|
|
3098
3214
|
}
|
|
3215
|
+
async function cancelProcessRun(runId) {
|
|
3216
|
+
let resourceCancellationError;
|
|
3217
|
+
try {
|
|
3218
|
+
await workerCommandLauncher?.cancel(runId);
|
|
3219
|
+
} catch (error) {
|
|
3220
|
+
resourceCancellationError = error instanceof Error ? error.message : String(error);
|
|
3221
|
+
}
|
|
3222
|
+
const active = activeProcesses.get(runId);
|
|
3223
|
+
let cancelled = resourceCancellationError === void 0;
|
|
3224
|
+
let cancelMessage;
|
|
3225
|
+
if (active) {
|
|
3226
|
+
try {
|
|
3227
|
+
closeProcessStdin(active);
|
|
3228
|
+
await terminateProcessTree(active.process);
|
|
3229
|
+
cancelMessage = `Stopped ${active.command}`;
|
|
3230
|
+
} catch (error) {
|
|
3231
|
+
cancelled = false;
|
|
3232
|
+
cancelMessage = `Failed to stop ${active.command}: ${error instanceof Error ? error.message : String(error)}`;
|
|
3233
|
+
}
|
|
3234
|
+
} else {
|
|
3235
|
+
cancelledProcessRuns.add(runId);
|
|
3236
|
+
abortPendingReservation(`run:${runId}`, "Cancellation queued before command start");
|
|
3237
|
+
cancelMessage = "Cancellation queued before command start";
|
|
3238
|
+
}
|
|
3239
|
+
if (resourceCancellationError) cancelMessage += `; job resource termination failed: ${resourceCancellationError}`;
|
|
3240
|
+
return { cancelled, message: cancelMessage };
|
|
3241
|
+
}
|
|
3099
3242
|
function closeProcessStdin(active) {
|
|
3100
3243
|
if (!active?.stdin) {
|
|
3101
3244
|
return;
|
|
@@ -3121,21 +3264,35 @@ function sendReplayWorkerMessage(ws, message) {
|
|
|
3121
3264
|
}
|
|
3122
3265
|
function sendWorkerMessage(ws, message) {
|
|
3123
3266
|
if (launcherFailureInProgress) return;
|
|
3124
|
-
if (recoveryStore && "requestId" in message && typeof message.requestId === "string" && recoveryStore.isUnknown(message.requestId))
|
|
3125
|
-
return;
|
|
3126
3267
|
const transportError = "error" in message && typeof message.error === "string" ? message.error : message.type === "workspace_sync_result" && message.result.outcome === "failed" ? message.result.error : void 0;
|
|
3127
|
-
if (
|
|
3128
|
-
|
|
3268
|
+
if (recoveryJournal && transportError && isWorkerCommunicationFailure(transportError) && "requestId" in message && typeof message.requestId === "string") {
|
|
3269
|
+
recoveryJournal.unknown(message.requestId);
|
|
3129
3270
|
currentWorkerSocket?.close(1012, "Communication recovery");
|
|
3130
3271
|
return;
|
|
3131
3272
|
}
|
|
3132
|
-
|
|
3133
|
-
|
|
3134
|
-
recoveryStore?.unknownRun(message.runId);
|
|
3273
|
+
if (!recoveryJournal) {
|
|
3274
|
+
sendReplayWorkerMessage(ws, message);
|
|
3135
3275
|
return;
|
|
3136
3276
|
}
|
|
3137
|
-
|
|
3138
|
-
|
|
3277
|
+
recoveryJournal.send(message);
|
|
3278
|
+
}
|
|
3279
|
+
function sendWorkerOutput(ws, frame) {
|
|
3280
|
+
if (launcherFailureInProgress) return Promise.resolve();
|
|
3281
|
+
if (!recoveryJournal) {
|
|
3282
|
+
sendReplayWorkerMessage(ws, frame);
|
|
3283
|
+
return Promise.resolve();
|
|
3284
|
+
}
|
|
3285
|
+
return recoveryJournal.sendOutput(frame);
|
|
3286
|
+
}
|
|
3287
|
+
function deliverJournaledWorkerMessage(message) {
|
|
3288
|
+
const target = currentWorkerSocket;
|
|
3289
|
+
if ((message.type === "exec_exit" || message.type === "exec_error") && typeof message.runId === "string" && infrastructureStoppedRunIds.has(message.runId)) {
|
|
3290
|
+
recoveryJournal?.unknownRun(message.runId);
|
|
3291
|
+
return true;
|
|
3292
|
+
}
|
|
3293
|
+
if (!target || target.readyState !== WebSocket.OPEN) return false;
|
|
3294
|
+
if (message.type === "exec_output" && workerRecoveryId !== void 0) return false;
|
|
3295
|
+
return sendReplayWorkerMessage(target, message);
|
|
3139
3296
|
}
|
|
3140
3297
|
function sendSerializedWorkerMessage(ws, serialized) {
|
|
3141
3298
|
sendWorkerMessage(ws, JSON.parse(serialized));
|
|
@@ -3164,7 +3321,8 @@ function buildActiveProcessReports() {
|
|
|
3164
3321
|
...active.cwd ? { cwd: active.cwd } : {},
|
|
3165
3322
|
startedAt: active.startedAt,
|
|
3166
3323
|
...active.interactive ? { interactive: true } : {},
|
|
3167
|
-
...active.workspaceEffect === "none" ? { workspaceEffect: "none" } : {}
|
|
3324
|
+
...active.workspaceEffect === "none" ? { workspaceEffect: "none" } : {},
|
|
3325
|
+
...active.commandClass ? { commandClass: active.commandClass } : {}
|
|
3168
3326
|
}));
|
|
3169
3327
|
}
|
|
3170
3328
|
function sendActiveProcessReport(ws) {
|
|
@@ -3686,6 +3844,7 @@ async function startWorker(options, projectRuntime = {
|
|
|
3686
3844
|
validateLabel(label);
|
|
3687
3845
|
if (!workerCommandLauncher) {
|
|
3688
3846
|
const launcher = await createWorkerCommandLauncher({
|
|
3847
|
+
onCapacityChange: () => scheduleCapacityReport(),
|
|
3689
3848
|
configuration: process.env.R5D_WORKER_COMMAND_LAUNCHER,
|
|
3690
3849
|
env: commandLauncherEnvironment(),
|
|
3691
3850
|
onFailure: handleWorkerCommandLauncherFailure
|
|
@@ -3696,9 +3855,19 @@ async function startWorker(options, projectRuntime = {
|
|
|
3696
3855
|
`);
|
|
3697
3856
|
}
|
|
3698
3857
|
const rootDir = path.resolve(options.rootDir ?? process.env.R5D_ROOT ?? defaultRootDir());
|
|
3699
|
-
|
|
3700
|
-
|
|
3701
|
-
|
|
3858
|
+
if (!recoveryJournal) {
|
|
3859
|
+
const journal = await WorkerRecoveryJournal.open({
|
|
3860
|
+
filename: path.join(
|
|
3861
|
+
rootDir,
|
|
3862
|
+
".worker-recovery",
|
|
3863
|
+
createHash("sha256").update(`${baseUrl}\0${label}`).digest("hex"),
|
|
3864
|
+
"protocol.sqlite"
|
|
3865
|
+
)
|
|
3866
|
+
});
|
|
3867
|
+
journal.deliver = deliverJournaledWorkerMessage;
|
|
3868
|
+
journal.onFailure = handleRecoveryJournalFailure;
|
|
3869
|
+
recoveryJournal = journal;
|
|
3870
|
+
}
|
|
3702
3871
|
const projectsRoot = path.join(rootDir, "projects");
|
|
3703
3872
|
const syncRoot = path.join(rootDir, "sync");
|
|
3704
3873
|
const workspaceShadowRoot = path.join(syncRoot, "workspace");
|
|
@@ -3801,7 +3970,6 @@ async function startWorker(options, projectRuntime = {
|
|
|
3801
3970
|
const bearerAuthHeader = `Authorization: Bearer ${token}`;
|
|
3802
3971
|
const workerCredentialUsername = credentialUsernameForAuthHeader(bearerAuthHeader);
|
|
3803
3972
|
const projectBranchKey = (projectId, branchName) => `${projectId}\0${branchName}`;
|
|
3804
|
-
const projectMountId = (projectId, branchName) => `project:${projectId}:${encodeURIComponent(branchName)}`;
|
|
3805
3973
|
const projectPlanMountId = (projectId, branchName) => `plan:${projectId}:${encodeURIComponent(branchName)}`;
|
|
3806
3974
|
const workspaceProjectRelativePath = (projectId, branchName) => path.posix.join("projects", projectId, "branches", encodeURIComponent(branchName));
|
|
3807
3975
|
const workspacePlanRelativePath = (projectId, branchName) => path.posix.join("plans", projectId, encodeURIComponent(branchName));
|
|
@@ -3884,7 +4052,8 @@ async function startWorker(options, projectRuntime = {
|
|
|
3884
4052
|
sourcePath: configuredProjectBranchPath(projectsRoot, project, branchName),
|
|
3885
4053
|
planSourcePath: path.join(planRoot, project.projectId, ...branchName.split("/"))
|
|
3886
4054
|
}));
|
|
3887
|
-
removeProjectWorktrees({ projectRoot, branchNames: branches.map(({ branchName }) => branchName) });
|
|
4055
|
+
const removed = removeProjectWorktrees({ projectsRoot, projectRoot, branchNames: branches.map(({ branchName }) => branchName) });
|
|
4056
|
+
if (removed.stagedForCollectionPath) checkoutGarbageCollector.enqueue(removed.stagedForCollectionPath);
|
|
3888
4057
|
for (const branch of branches) {
|
|
3889
4058
|
fs.rmSync(branch.planSourcePath, { recursive: true, force: true });
|
|
3890
4059
|
const key = projectBranchKey(project.projectId, branch.branchName);
|
|
@@ -3972,14 +4141,14 @@ async function startWorker(options, projectRuntime = {
|
|
|
3972
4141
|
)
|
|
3973
4142
|
};
|
|
3974
4143
|
};
|
|
3975
|
-
const stageDurableWorkspaceDeletions = () => {
|
|
4144
|
+
const stageDurableWorkspaceDeletions = async () => {
|
|
3976
4145
|
for (const deletion of projectWorkspaceState.pendingTreeDeletions) {
|
|
3977
4146
|
const cleanupConfig = cleanupConfigForDurableDeletion(deletion);
|
|
3978
4147
|
if (deletion.kind === "project") {
|
|
3979
4148
|
if (!deletion.projectDeleted) {
|
|
3980
4149
|
const nextProject = projectConfigById.get(deletion.projectId);
|
|
3981
4150
|
if (!nextProject) throw new Error(`Checkout-path move lost desired project ${deletion.projectId}`);
|
|
3982
|
-
|
|
4151
|
+
await workspaceFilesystemExecutor().run("project_checkout_path_move", {
|
|
3983
4152
|
oldProjectRoot: deletion.managedPath,
|
|
3984
4153
|
newProjectRoot: configuredProjectRoot(projectsRoot, nextProject),
|
|
3985
4154
|
projectsDurabilityRoot: projectsRoot,
|
|
@@ -4346,7 +4515,7 @@ async function startWorker(options, projectRuntime = {
|
|
|
4346
4515
|
}
|
|
4347
4516
|
}
|
|
4348
4517
|
if (transitionReseedRequired) {
|
|
4349
|
-
projectWorkspaceState = projectWorkspaceStateStore.recordEnabledRepositoryTransition({
|
|
4518
|
+
projectWorkspaceState = await projectWorkspaceStateStore.recordEnabledRepositoryTransition({
|
|
4350
4519
|
projectId: project.projectId,
|
|
4351
4520
|
transitionId: project.repositoryTransitionId
|
|
4352
4521
|
});
|
|
@@ -4515,7 +4684,7 @@ async function startWorker(options, projectRuntime = {
|
|
|
4515
4684
|
)
|
|
4516
4685
|
];
|
|
4517
4686
|
if (publishedTombstoneIds.length > 0) {
|
|
4518
|
-
projectWorkspaceState = projectWorkspaceStateStore.recordTreePublication({
|
|
4687
|
+
projectWorkspaceState = await projectWorkspaceStateStore.recordTreePublication({
|
|
4519
4688
|
tombstoneIds: publishedTombstoneIds,
|
|
4520
4689
|
publishedHead
|
|
4521
4690
|
});
|
|
@@ -4534,7 +4703,7 @@ async function startWorker(options, projectRuntime = {
|
|
|
4534
4703
|
forgetLocalProjectMirrorRefsTokens(deletion.projectId, deletion.branchName);
|
|
4535
4704
|
}
|
|
4536
4705
|
if (deletion.tombstoneId && !deletion.projectDeleted) {
|
|
4537
|
-
projectWorkspaceState = projectWorkspaceStateStore.recordMirrorRefDeletion({
|
|
4706
|
+
projectWorkspaceState = await projectWorkspaceStateStore.recordMirrorRefDeletion({
|
|
4538
4707
|
tombstoneId: deletion.tombstoneId,
|
|
4539
4708
|
branchName: deletion.branchName
|
|
4540
4709
|
});
|
|
@@ -4731,7 +4900,10 @@ async function startWorker(options, projectRuntime = {
|
|
|
4731
4900
|
};
|
|
4732
4901
|
};
|
|
4733
4902
|
const workspaceHydrationTransactionHooks = createWorkspaceHydrationLeaseHooks(
|
|
4734
|
-
(message) => sendWorkerMessageFromCurrentSource(ws, message)
|
|
4903
|
+
(message) => sendWorkerMessageFromCurrentSource(ws, message),
|
|
4904
|
+
{
|
|
4905
|
+
fence: workspaceMountHoldFence
|
|
4906
|
+
}
|
|
4735
4907
|
);
|
|
4736
4908
|
const performWorkspaceSync = async (input) => {
|
|
4737
4909
|
if (!workspaceRemoteUrl || !workspaceCredentialHelper || !workspaceCredentialUsername || !workspaceGitIdentity) {
|
|
@@ -5389,7 +5561,7 @@ async function startWorker(options, projectRuntime = {
|
|
|
5389
5561
|
(project) => project.preserveOnlyBranches.map(({ branchId, branchName }) => ({ branchId, projectId: project.projectId, branchName }))
|
|
5390
5562
|
);
|
|
5391
5563
|
if (!incidentDeferral.incidentId) {
|
|
5392
|
-
projectWorkspaceState = projectWorkspaceStateStore.reconcile({
|
|
5564
|
+
projectWorkspaceState = await projectWorkspaceStateStore.reconcile({
|
|
5393
5565
|
desiredProjects: message.projects,
|
|
5394
5566
|
preserveOnlyBranches
|
|
5395
5567
|
});
|
|
@@ -5400,7 +5572,7 @@ async function startWorker(options, projectRuntime = {
|
|
|
5400
5572
|
(candidate) => candidate.projectId === project.projectId && candidate.branchName === branch.branchName
|
|
5401
5573
|
);
|
|
5402
5574
|
if (pending2 && pending2.branchId === branch.branchId) {
|
|
5403
|
-
projectWorkspaceState = projectWorkspaceStateStore.clearPendingCreatedBranch(pending2);
|
|
5575
|
+
projectWorkspaceState = await projectWorkspaceStateStore.clearPendingCreatedBranch(pending2);
|
|
5404
5576
|
}
|
|
5405
5577
|
}
|
|
5406
5578
|
}
|
|
@@ -5568,7 +5740,7 @@ async function startWorker(options, projectRuntime = {
|
|
|
5568
5740
|
for (const projectId of [...readyProjectIds]) {
|
|
5569
5741
|
if (!projectConfigById.has(projectId)) readyProjectIds.delete(projectId);
|
|
5570
5742
|
}
|
|
5571
|
-
stageDurableWorkspaceDeletions();
|
|
5743
|
+
await stageDurableWorkspaceDeletions();
|
|
5572
5744
|
const mountsBeforeGit = buildWorkspaceMounts();
|
|
5573
5745
|
await hydrateWorkspaceGitMounts(
|
|
5574
5746
|
workspaceShadowRoot,
|
|
@@ -5614,7 +5786,12 @@ async function startWorker(options, projectRuntime = {
|
|
|
5614
5786
|
const connectSyncMs = Date.now() - connectSyncStartedAtMs;
|
|
5615
5787
|
const result = syncResult.telemetry ? syncResult : {
|
|
5616
5788
|
...syncResult,
|
|
5617
|
-
telemetry: {
|
|
5789
|
+
telemetry: {
|
|
5790
|
+
totalMs: connectSyncMs,
|
|
5791
|
+
queueMs: 0,
|
|
5792
|
+
prepareMs: lastWorkspaceSyncMirrorObservationMs,
|
|
5793
|
+
synchronizeMs: connectSyncMs
|
|
5794
|
+
}
|
|
5618
5795
|
};
|
|
5619
5796
|
assertConfigurationSyncStillAdmitted();
|
|
5620
5797
|
if (receiptGeneration !== workspaceConfigurationReceiptGeneration) {
|
|
@@ -5633,7 +5810,7 @@ async function startWorker(options, projectRuntime = {
|
|
|
5633
5810
|
) ? [deletion.tombstoneId] : []
|
|
5634
5811
|
);
|
|
5635
5812
|
if (layoutCleanupIds.length > 0) {
|
|
5636
|
-
projectWorkspaceState = projectWorkspaceStateStore.recordTreePublication({
|
|
5813
|
+
projectWorkspaceState = await projectWorkspaceStateStore.recordTreePublication({
|
|
5637
5814
|
tombstoneIds: layoutCleanupIds,
|
|
5638
5815
|
publishedHead
|
|
5639
5816
|
});
|
|
@@ -5740,26 +5917,33 @@ async function startWorker(options, projectRuntime = {
|
|
|
5740
5917
|
if (startupProjectSnapshotRecoveryCompleted) return;
|
|
5741
5918
|
sendLifecycleProgress({ phase: "recovering", operationId, detail: "Checking interrupted workspace transactions" });
|
|
5742
5919
|
const snapshotRecovery = await workspaceSyncSingleFlight.runExclusive(
|
|
5743
|
-
async () => await
|
|
5744
|
-
|
|
5745
|
-
|
|
5746
|
-
|
|
5747
|
-
|
|
5748
|
-
|
|
5749
|
-
|
|
5750
|
-
|
|
5751
|
-
|
|
5752
|
-
|
|
5753
|
-
|
|
5754
|
-
|
|
5755
|
-
|
|
5756
|
-
|
|
5757
|
-
|
|
5758
|
-
|
|
5920
|
+
async () => await workspaceFilesystemExecutor().run(
|
|
5921
|
+
"project_snapshots_recover",
|
|
5922
|
+
{
|
|
5923
|
+
projectsRoot,
|
|
5924
|
+
temporaryRoot: projectWorktreeSnapshotsRoot,
|
|
5925
|
+
currentProcessId: process.pid,
|
|
5926
|
+
currentOwnerSessionId: PROJECT_WORKTREE_SNAPSHOT_OWNER_SESSION_ID
|
|
5927
|
+
},
|
|
5928
|
+
{
|
|
5929
|
+
onProgress: (progress) => {
|
|
5930
|
+
const lifecycleProgress = {
|
|
5931
|
+
phase: "recovering",
|
|
5932
|
+
operationId,
|
|
5933
|
+
detail: describeSnapshotProgress(progress),
|
|
5934
|
+
completedBytes: progress.completedBytes,
|
|
5935
|
+
completedItems: progress.completedBranches,
|
|
5936
|
+
totalItems: progress.totalBranches,
|
|
5937
|
+
...progress.totalBytes === void 0 ? {} : { totalBytes: progress.totalBytes }
|
|
5938
|
+
};
|
|
5939
|
+
sendLifecycleProgress(lifecycleProgress);
|
|
5940
|
+
process.stdout.write(
|
|
5941
|
+
`[r5d-worker] ${lifecycleProgress.detail}: ${progress.completedBranches}/${progress.totalBranches} branch(es), ${progress.completedBytes}/${progress.totalBytes ?? "?"} bytes
|
|
5759
5942
|
`
|
|
5760
|
-
|
|
5943
|
+
);
|
|
5944
|
+
}
|
|
5761
5945
|
}
|
|
5762
|
-
|
|
5946
|
+
)
|
|
5763
5947
|
);
|
|
5764
5948
|
if (snapshotRecovery.restored.length > 0) {
|
|
5765
5949
|
process.stdout.write(`[r5d-worker] restored ${snapshotRecovery.restored.length} interrupted project snapshot(s)
|
|
@@ -5854,7 +6038,7 @@ async function startWorker(options, projectRuntime = {
|
|
|
5854
6038
|
type: "hello",
|
|
5855
6039
|
resumableProtocol: WORKER_RESUMABLE_PROTOCOL,
|
|
5856
6040
|
runtimeId: workerRuntimeId,
|
|
5857
|
-
ledgerId:
|
|
6041
|
+
ledgerId: recoveryJournal.ledgerId,
|
|
5858
6042
|
startupReady: startupRecoveryComplete,
|
|
5859
6043
|
hostInfo: {
|
|
5860
6044
|
hostname: hostname(),
|
|
@@ -5873,6 +6057,7 @@ async function startWorker(options, projectRuntime = {
|
|
|
5873
6057
|
workspaceConfigResetToCanonicalV1: true,
|
|
5874
6058
|
projectBranchDeletionFastAckV1: true,
|
|
5875
6059
|
projectBranchDeletionStagedRemovalV1: true,
|
|
6060
|
+
commandControlLaneV1: workerCommandLauncher?.supportsControlLane === true,
|
|
5876
6061
|
projectMirrorLeaseV1: true,
|
|
5877
6062
|
projectMirrorRefsTokensV1: true,
|
|
5878
6063
|
projectBranchWorkingTreeModeV1: true
|
|
@@ -5881,16 +6066,39 @@ async function startWorker(options, projectRuntime = {
|
|
|
5881
6066
|
}
|
|
5882
6067
|
};
|
|
5883
6068
|
sendSerializedWorkerMessage(ws, JSON.stringify(hello));
|
|
6069
|
+
let outputPumpInFlight = false;
|
|
5884
6070
|
outputReplayTimer = setInterval(() => {
|
|
5885
|
-
if (workerRecoveryId !== void 0 || currentWorkerSocket !== ws) return;
|
|
5886
|
-
|
|
5887
|
-
|
|
5888
|
-
|
|
6071
|
+
if (workerRecoveryId !== void 0 || currentWorkerSocket !== ws || outputPumpInFlight || !recoveryJournal) return;
|
|
6072
|
+
outputPumpInFlight = true;
|
|
6073
|
+
void (async () => {
|
|
6074
|
+
try {
|
|
6075
|
+
for await (const page of recoveryJournal.outputReplay(void 0)) {
|
|
6076
|
+
for (const output of page) {
|
|
6077
|
+
if (workerRecoveryId !== void 0 || currentWorkerSocket !== ws || !sendReplayWorkerMessage(ws, output)) return;
|
|
6078
|
+
}
|
|
6079
|
+
}
|
|
6080
|
+
} catch (error) {
|
|
6081
|
+
process.stderr.write(`[r5d-worker] output replay pump failed: ${error instanceof Error ? error.message : String(error)}
|
|
6082
|
+
`);
|
|
6083
|
+
} finally {
|
|
6084
|
+
outputPumpInFlight = false;
|
|
6085
|
+
}
|
|
6086
|
+
})();
|
|
5889
6087
|
}, 250);
|
|
5890
6088
|
outputReplayTimer.unref();
|
|
6089
|
+
let terminalPumpInFlight = false;
|
|
5891
6090
|
terminalReplayTimer = setInterval(() => {
|
|
5892
|
-
if (workerRecoveryId !== void 0) return;
|
|
5893
|
-
|
|
6091
|
+
if (workerRecoveryId !== void 0 || terminalPumpInFlight || !recoveryJournal) return;
|
|
6092
|
+
terminalPumpInFlight = true;
|
|
6093
|
+
void recoveryJournal.terminals().then((terminals) => {
|
|
6094
|
+
if (workerRecoveryId !== void 0 || currentWorkerSocket !== ws) return;
|
|
6095
|
+
for (const terminal of terminals) sendReplayWorkerMessage(ws, terminal);
|
|
6096
|
+
}).catch((error) => {
|
|
6097
|
+
process.stderr.write(`[r5d-worker] terminal replay pump failed: ${error instanceof Error ? error.message : String(error)}
|
|
6098
|
+
`);
|
|
6099
|
+
}).finally(() => {
|
|
6100
|
+
terminalPumpInFlight = false;
|
|
6101
|
+
});
|
|
5894
6102
|
}, 5e3);
|
|
5895
6103
|
terminalReplayTimer.unref();
|
|
5896
6104
|
process.stdout.write("[r5d-worker] connected\n");
|
|
@@ -5903,28 +6111,37 @@ async function startWorker(options, projectRuntime = {
|
|
|
5903
6111
|
if (message.type === "recovery_request") {
|
|
5904
6112
|
workerRecoveryId = message.recoveryId;
|
|
5905
6113
|
sendActiveProcessReport(ws);
|
|
5906
|
-
|
|
5907
|
-
|
|
5908
|
-
|
|
5909
|
-
|
|
5910
|
-
|
|
5911
|
-
|
|
5912
|
-
|
|
5913
|
-
|
|
5914
|
-
|
|
5915
|
-
|
|
5916
|
-
|
|
5917
|
-
|
|
5918
|
-
|
|
5919
|
-
|
|
5920
|
-
|
|
5921
|
-
|
|
5922
|
-
|
|
6114
|
+
recoveryJournal.pauseDelivery();
|
|
6115
|
+
try {
|
|
6116
|
+
const state = await recoveryJournal.recoveryState(message.requests);
|
|
6117
|
+
if (currentWorkerSocket !== ws || workerRecoveryId !== message.recoveryId) return;
|
|
6118
|
+
sendWorkerMessage(ws, {
|
|
6119
|
+
type: "recovery_state",
|
|
6120
|
+
recoveryId: message.recoveryId,
|
|
6121
|
+
runtimeId: workerRuntimeId,
|
|
6122
|
+
ledgerId: recoveryJournal.ledgerId,
|
|
6123
|
+
operations: state.operations
|
|
6124
|
+
});
|
|
6125
|
+
const cursors = new Map(state.cursors.map((cursor) => [cursor.runId, cursor]));
|
|
6126
|
+
for (const cursor of message.outputOffsets) cursors.set(cursor.runId, cursor);
|
|
6127
|
+
workerOutputWindow.reset([...cursors.values()]);
|
|
6128
|
+
for (const response of state.responses) sendReplayWorkerMessage(ws, response);
|
|
6129
|
+
for await (const page of recoveryJournal.outputReplay(message.outputOffsets)) {
|
|
6130
|
+
for (const output of page) {
|
|
6131
|
+
while (true) {
|
|
6132
|
+
if (currentWorkerSocket !== ws || workerRecoveryId !== message.recoveryId || ws.readyState !== WebSocket.OPEN || workerLeaseExpired)
|
|
6133
|
+
return;
|
|
6134
|
+
if (sendReplayWorkerMessage(ws, output)) break;
|
|
6135
|
+
await Bun.sleep(10);
|
|
6136
|
+
}
|
|
6137
|
+
}
|
|
5923
6138
|
}
|
|
6139
|
+
if (currentWorkerSocket !== ws || workerRecoveryId !== message.recoveryId) return;
|
|
6140
|
+
for (const terminal of state.terminals) sendReplayWorkerMessage(ws, terminal);
|
|
6141
|
+
sendWorkerMessage(ws, { type: "recovery_complete", recoveryId: message.recoveryId });
|
|
6142
|
+
} finally {
|
|
6143
|
+
recoveryJournal.resumeDelivery();
|
|
5924
6144
|
}
|
|
5925
|
-
if (currentWorkerSocket !== ws || workerRecoveryId !== message.recoveryId) return;
|
|
5926
|
-
for (const terminal of recoveryStore.terminals()) sendReplayWorkerMessage(ws, terminal);
|
|
5927
|
-
sendWorkerMessage(ws, { type: "recovery_complete", recoveryId: message.recoveryId });
|
|
5928
6145
|
return;
|
|
5929
6146
|
}
|
|
5930
6147
|
if (message.type === "recovery_ready") {
|
|
@@ -5939,17 +6156,23 @@ async function startWorker(options, projectRuntime = {
|
|
|
5939
6156
|
return;
|
|
5940
6157
|
}
|
|
5941
6158
|
if (message.type === "operation_result_ack") {
|
|
5942
|
-
|
|
6159
|
+
recoveryJournal.acknowledgeResult(message.requestId);
|
|
5943
6160
|
return;
|
|
5944
6161
|
}
|
|
5945
6162
|
if (message.type === "exec_output_ack") {
|
|
5946
|
-
|
|
6163
|
+
recoveryJournal.acknowledgeOutput(message.runId, message.stream, message.nextOffset);
|
|
5947
6164
|
workerOutputWindow.acknowledge(message.runId, message.stream, message.nextOffset);
|
|
5948
6165
|
return;
|
|
5949
6166
|
}
|
|
5950
6167
|
if (message.type === "cancel_session") {
|
|
5951
|
-
|
|
6168
|
+
const durableCancellation = recoveryJournal.cancelSession(message.sessionId);
|
|
6169
|
+
abortPendingReservations("Session cancelled before command start", message.sessionId);
|
|
5952
6170
|
await Promise.all([
|
|
6171
|
+
durableCancellation,
|
|
6172
|
+
// File writes already in flight for the session settle before the
|
|
6173
|
+
// acknowledgement; queued ones re-check the cancellation before
|
|
6174
|
+
// their first write and never start.
|
|
6175
|
+
sessionFileMutations.settled(message.sessionId),
|
|
5953
6176
|
workerCommandLauncher?.cancelSession(message.sessionId),
|
|
5954
6177
|
...[...activeProcesses].filter(([, active]) => active.sessionId === message.sessionId).map(async ([runId, active]) => {
|
|
5955
6178
|
cancelledProcessRuns.add(runId);
|
|
@@ -5964,27 +6187,74 @@ async function startWorker(options, projectRuntime = {
|
|
|
5964
6187
|
if (!startupRecoveryComplete && message.type !== "ping") return;
|
|
5965
6188
|
if (currentWorkerSocket !== ws) return;
|
|
5966
6189
|
const operationRequestId = "requestId" in message && typeof message.requestId === "string" && message.type !== "workspace_config" ? message.requestId : void 0;
|
|
6190
|
+
if (message.type === "cancel") {
|
|
6191
|
+
const admission = recoveryJournal.admit({ ...message, requestId: message.requestId });
|
|
6192
|
+
const cancelled = await cancelProcessRun(message.runId);
|
|
6193
|
+
let admitted;
|
|
6194
|
+
try {
|
|
6195
|
+
admitted = await admission;
|
|
6196
|
+
} catch (error) {
|
|
6197
|
+
if (!(error instanceof WorkerRecoveryJournalBusyError)) throw error;
|
|
6198
|
+
process.stderr.write(`[r5d-worker] cancel ${message.runId} stopped the run but was not durably admitted: ${error.message}
|
|
6199
|
+
`);
|
|
6200
|
+
return;
|
|
6201
|
+
}
|
|
6202
|
+
if (admitted.admission === "conflict") {
|
|
6203
|
+
ws.close(1008, "Operation identity conflict");
|
|
6204
|
+
return;
|
|
6205
|
+
}
|
|
6206
|
+
sendWorkerMessage(ws, { type: "operation_received", requestId: message.requestId });
|
|
6207
|
+
if (admitted.admission !== "new") {
|
|
6208
|
+
if (admitted.response && admitted.admission !== "unknown") sendReplayWorkerMessage(ws, admitted.response);
|
|
6209
|
+
return;
|
|
6210
|
+
}
|
|
6211
|
+
sendSerializedWorkerMessage(
|
|
6212
|
+
ws,
|
|
6213
|
+
JSON.stringify({
|
|
6214
|
+
type: "cancel_result",
|
|
6215
|
+
requestId: message.requestId,
|
|
6216
|
+
runId: message.runId,
|
|
6217
|
+
cancelled: cancelled.cancelled,
|
|
6218
|
+
message: cancelled.message
|
|
6219
|
+
})
|
|
6220
|
+
);
|
|
6221
|
+
return;
|
|
6222
|
+
}
|
|
5967
6223
|
if (operationRequestId) {
|
|
5968
|
-
|
|
5969
|
-
|
|
6224
|
+
let admitted;
|
|
6225
|
+
try {
|
|
6226
|
+
admitted = await recoveryJournal.admit({ ...message, requestId: operationRequestId });
|
|
6227
|
+
} catch (error) {
|
|
6228
|
+
if (!(error instanceof WorkerRecoveryJournalBusyError)) throw error;
|
|
6229
|
+
process.stderr.write(`[r5d-worker] ${message.type} ${operationRequestId} was not admitted: ${error.message}
|
|
6230
|
+
`);
|
|
6231
|
+
return;
|
|
6232
|
+
}
|
|
6233
|
+
if (admitted.admission === "conflict") {
|
|
5970
6234
|
ws.close(1008, "Operation identity conflict");
|
|
5971
6235
|
return;
|
|
5972
6236
|
}
|
|
5973
6237
|
sendWorkerMessage(ws, { type: "operation_received", requestId: operationRequestId });
|
|
5974
|
-
if (admission !== "new") {
|
|
5975
|
-
|
|
5976
|
-
if (
|
|
5977
|
-
|
|
6238
|
+
if (admitted.admission !== "new") {
|
|
6239
|
+
if (admitted.response && admitted.admission !== "unknown") sendReplayWorkerMessage(ws, admitted.response);
|
|
6240
|
+
if (!(admitted.admission === "unknown" && message.type === "delete_project_branch" && await recoveryJournal.readmitUnknownBranchDeletion(
|
|
6241
|
+
operationRequestId,
|
|
6242
|
+
"sessionId" in message && typeof message.sessionId === "string" ? message.sessionId : void 0
|
|
6243
|
+
))) {
|
|
5978
6244
|
return;
|
|
5979
6245
|
}
|
|
5980
6246
|
}
|
|
6247
|
+
if (recoveryJournal.isRequestCancelled(operationRequestId)) {
|
|
6248
|
+
recoveryJournal.unknown(operationRequestId);
|
|
6249
|
+
return;
|
|
6250
|
+
}
|
|
5981
6251
|
}
|
|
5982
6252
|
const messageAdmissionGeneration = workerAdmissionGeneration;
|
|
5983
6253
|
const admissionCredentials = configuredCredentialGenerationFingerprint;
|
|
5984
6254
|
const assertMessageAdmission = () => {
|
|
5985
6255
|
if (operationRequestId) {
|
|
5986
|
-
if (workerLeaseExpired ||
|
|
5987
|
-
|
|
6256
|
+
if (workerLeaseExpired || recoveryJournal.isRequestCancelled(operationRequestId) || admissionCredentials !== configuredCredentialGenerationFingerprint || shutdownAfterClose || !workspaceConfigured) {
|
|
6257
|
+
recoveryJournal.unknown(operationRequestId);
|
|
5988
6258
|
throw new StaleWorkerAdmissionError();
|
|
5989
6259
|
}
|
|
5990
6260
|
return;
|
|
@@ -5999,6 +6269,8 @@ async function startWorker(options, projectRuntime = {
|
|
|
5999
6269
|
});
|
|
6000
6270
|
};
|
|
6001
6271
|
if (message.type === "connected") {
|
|
6272
|
+
capacityReportSocket = ws;
|
|
6273
|
+
scheduleCapacityReport(true);
|
|
6002
6274
|
return;
|
|
6003
6275
|
}
|
|
6004
6276
|
if (message.type === "project_mirror_refs_tokens") {
|
|
@@ -6119,7 +6391,7 @@ async function startWorker(options, projectRuntime = {
|
|
|
6119
6391
|
}
|
|
6120
6392
|
recoveryDeadline ??= Date.now() + WORKER_RECOVERY_TIMEOUT_MS;
|
|
6121
6393
|
if (workerLeaseExpired || Date.now() >= recoveryDeadline) {
|
|
6122
|
-
|
|
6394
|
+
recoveryJournal.unknown(message.requestId);
|
|
6123
6395
|
return;
|
|
6124
6396
|
}
|
|
6125
6397
|
currentWorkerSocket?.close(1012, "Communication recovery");
|
|
@@ -6149,7 +6421,7 @@ async function startWorker(options, projectRuntime = {
|
|
|
6149
6421
|
projectId: message.projectId,
|
|
6150
6422
|
branchName: message.targetBranch
|
|
6151
6423
|
};
|
|
6152
|
-
const created = await workspaceSyncSingleFlight.runBranchMutation(() => {
|
|
6424
|
+
const created = await workspaceSyncSingleFlight.runBranchMutation(async () => {
|
|
6153
6425
|
if (workspaceOperationsAreFenced(deferredWorkspaceConfiguration, activeWorkspaceIncidentId)) {
|
|
6154
6426
|
throw new Error(
|
|
6155
6427
|
`Project branch creation was fenced while queued by workspace incident ${deferredWorkspaceConfiguration?.incidentId ?? activeWorkspaceIncidentId}`
|
|
@@ -6164,7 +6436,7 @@ async function startWorker(options, projectRuntime = {
|
|
|
6164
6436
|
const pendingWasRecorded = projectWorkspaceState.locallyPendingCreatedBranches.some(
|
|
6165
6437
|
({ projectId, branchName }) => projectId === message.projectId && branchName === message.targetBranch
|
|
6166
6438
|
);
|
|
6167
|
-
projectWorkspaceState = projectWorkspaceStateStore.recordPendingCreatedBranch(pendingBranch);
|
|
6439
|
+
projectWorkspaceState = await projectWorkspaceStateStore.recordPendingCreatedBranch(pendingBranch);
|
|
6168
6440
|
let gitBranchCreated = false;
|
|
6169
6441
|
try {
|
|
6170
6442
|
if (!readyProjectIds.has(project.projectId)) {
|
|
@@ -6185,7 +6457,7 @@ async function startWorker(options, projectRuntime = {
|
|
|
6185
6457
|
}
|
|
6186
6458
|
return { branchPath: existingPath, baseCommitHash: existingBranch.baseCommitHash };
|
|
6187
6459
|
}
|
|
6188
|
-
const createdBranch = createOrRetryLinkedProjectBranch({
|
|
6460
|
+
const createdBranch = await createOrRetryLinkedProjectBranch({
|
|
6189
6461
|
projectRoot: configuredProjectRoot(projectsRoot, project),
|
|
6190
6462
|
primaryBranchName: primaryProjectBranch(project),
|
|
6191
6463
|
sourceBranchName: message.sourceBranch,
|
|
@@ -6210,7 +6482,7 @@ async function startWorker(options, projectRuntime = {
|
|
|
6210
6482
|
return createdBranch;
|
|
6211
6483
|
} catch (error) {
|
|
6212
6484
|
if (!pendingWasRecorded && !gitBranchCreated && !projectBranchMayExistAfterCreateFailure(error)) {
|
|
6213
|
-
projectWorkspaceState = projectWorkspaceStateStore.rollbackPendingCreatedBranch(pendingBranch);
|
|
6485
|
+
projectWorkspaceState = await projectWorkspaceStateStore.rollbackPendingCreatedBranch(pendingBranch);
|
|
6214
6486
|
}
|
|
6215
6487
|
throw error;
|
|
6216
6488
|
}
|
|
@@ -6254,7 +6526,7 @@ async function startWorker(options, projectRuntime = {
|
|
|
6254
6526
|
});
|
|
6255
6527
|
try {
|
|
6256
6528
|
const deletionKey = projectBranchKey(message.projectId, message.branchName);
|
|
6257
|
-
const deletionNeedsSync = await workspaceSyncSingleFlight.runBranchMutation(() => {
|
|
6529
|
+
const deletionNeedsSync = await workspaceSyncSingleFlight.runBranchMutation(async () => {
|
|
6258
6530
|
if (workspaceOperationsAreFenced(deferredWorkspaceConfiguration, activeWorkspaceIncidentId)) {
|
|
6259
6531
|
throw new Error(
|
|
6260
6532
|
`Project branch deletion was fenced while queued by workspace incident ${deferredWorkspaceConfiguration?.incidentId ?? activeWorkspaceIncidentId}`
|
|
@@ -6281,12 +6553,12 @@ async function startWorker(options, projectRuntime = {
|
|
|
6281
6553
|
if (!configuredBranch && !pendingLocalBranch && !hasDurableTombstone && !pendingMirrorDeletes.has(deletionKey)) {
|
|
6282
6554
|
return false;
|
|
6283
6555
|
}
|
|
6284
|
-
projectWorkspaceState = projectWorkspaceStateStore.beginBranchDeletion({
|
|
6556
|
+
projectWorkspaceState = await projectWorkspaceStateStore.beginBranchDeletion({
|
|
6285
6557
|
projectId: message.projectId,
|
|
6286
6558
|
branchName: message.branchName
|
|
6287
6559
|
});
|
|
6288
6560
|
if (pendingLocalBranch) {
|
|
6289
|
-
projectWorkspaceState = projectWorkspaceStateStore.clearPendingCreatedBranch(pendingLocalBranch);
|
|
6561
|
+
projectWorkspaceState = await projectWorkspaceStateStore.clearPendingCreatedBranch(pendingLocalBranch);
|
|
6290
6562
|
}
|
|
6291
6563
|
pendingCreatedBranchPublicationNotBefore.delete(pendingCreatedBranchKey(message.projectId, message.branchName));
|
|
6292
6564
|
if (pendingMirrorDeletes.has(deletionKey)) return true;
|
|
@@ -6390,7 +6662,7 @@ async function startWorker(options, projectRuntime = {
|
|
|
6390
6662
|
}
|
|
6391
6663
|
if (message.type === "exec_terminal_ack") {
|
|
6392
6664
|
pendingProcessTerminals.delete(message.runId);
|
|
6393
|
-
|
|
6665
|
+
recoveryJournal.acknowledgeTerminal(message.runId);
|
|
6394
6666
|
return;
|
|
6395
6667
|
}
|
|
6396
6668
|
if (message.type === "port_forward_connect") {
|
|
@@ -6445,42 +6717,6 @@ async function startWorker(options, projectRuntime = {
|
|
|
6445
6717
|
sendSerializedWorkerMessage(ws, JSON.stringify({ type: "pong" }));
|
|
6446
6718
|
return;
|
|
6447
6719
|
}
|
|
6448
|
-
if (message.type === "cancel") {
|
|
6449
|
-
let resourceCancellationError;
|
|
6450
|
-
try {
|
|
6451
|
-
await workerCommandLauncher?.cancel(message.runId);
|
|
6452
|
-
} catch (error) {
|
|
6453
|
-
resourceCancellationError = error instanceof Error ? error.message : String(error);
|
|
6454
|
-
}
|
|
6455
|
-
const active = activeProcesses.get(message.runId);
|
|
6456
|
-
let cancelled = resourceCancellationError === void 0;
|
|
6457
|
-
let cancelMessage;
|
|
6458
|
-
if (active) {
|
|
6459
|
-
try {
|
|
6460
|
-
closeProcessStdin(active);
|
|
6461
|
-
await terminateProcessTree(active.process);
|
|
6462
|
-
cancelMessage = `Stopped ${active.command}`;
|
|
6463
|
-
} catch (error) {
|
|
6464
|
-
cancelled = false;
|
|
6465
|
-
cancelMessage = `Failed to stop ${active.command}: ${error instanceof Error ? error.message : String(error)}`;
|
|
6466
|
-
}
|
|
6467
|
-
} else {
|
|
6468
|
-
cancelledProcessRuns.add(message.runId);
|
|
6469
|
-
cancelMessage = "Cancellation queued before command start";
|
|
6470
|
-
}
|
|
6471
|
-
if (resourceCancellationError) cancelMessage += `; job resource termination failed: ${resourceCancellationError}`;
|
|
6472
|
-
sendSerializedWorkerMessage(
|
|
6473
|
-
ws,
|
|
6474
|
-
JSON.stringify({
|
|
6475
|
-
type: "cancel_result",
|
|
6476
|
-
requestId: message.requestId,
|
|
6477
|
-
runId: message.runId,
|
|
6478
|
-
cancelled,
|
|
6479
|
-
message: cancelMessage
|
|
6480
|
-
})
|
|
6481
|
-
);
|
|
6482
|
-
return;
|
|
6483
|
-
}
|
|
6484
6720
|
if (message.type === "exec_stdin") {
|
|
6485
6721
|
const sendAck = (payload) => sendSerializedWorkerMessage(
|
|
6486
6722
|
ws,
|
|
@@ -6561,10 +6797,20 @@ async function startWorker(options, projectRuntime = {
|
|
|
6561
6797
|
let mutationLeaseTransferred = false;
|
|
6562
6798
|
try {
|
|
6563
6799
|
resources = await acquireWorkerCommandLaunch(ws, message, assertMessageAdmission);
|
|
6564
|
-
|
|
6565
|
-
|
|
6566
|
-
|
|
6567
|
-
|
|
6800
|
+
const reservationAbort = registerPendingReservation(`pty:${message.ptyId}`, void 0);
|
|
6801
|
+
try {
|
|
6802
|
+
await reserveWorkspaceCommandAfterCurrentSync(
|
|
6803
|
+
message.target,
|
|
6804
|
+
workspaceSyncSingleFlight,
|
|
6805
|
+
() => {
|
|
6806
|
+
workspaceSyncPriorityPtyTargets.set(message.ptyId, message.target);
|
|
6807
|
+
if (targetMayMutateVisibleWorkspace(message.target)) recordVisibleWorkspaceMutation(message.target);
|
|
6808
|
+
},
|
|
6809
|
+
{ signal: reservationAbort.signal }
|
|
6810
|
+
);
|
|
6811
|
+
} finally {
|
|
6812
|
+
unregisterPendingReservation(`pty:${message.ptyId}`, reservationAbort);
|
|
6813
|
+
}
|
|
6568
6814
|
releaseWorkspaceMutation = await acquireWorkspaceCommandMutation(message.target, workspaceSyncSingleFlight);
|
|
6569
6815
|
if (workspaceOperationsAreFenced(deferredWorkspaceConfiguration, activeWorkspaceIncidentId)) {
|
|
6570
6816
|
throw new Error(
|
|
@@ -6629,6 +6875,7 @@ async function startWorker(options, projectRuntime = {
|
|
|
6629
6875
|
}
|
|
6630
6876
|
if (message.type === "pty_close") {
|
|
6631
6877
|
const activePty = activePtys.get(message.ptyId);
|
|
6878
|
+
abortPendingReservation(`pty:${message.ptyId}`, "Shell closed before it started");
|
|
6632
6879
|
await closePty(message);
|
|
6633
6880
|
if (activePty && targetMayMutateVisibleWorkspace(activePty.target)) {
|
|
6634
6881
|
markWorkspaceDirty({ type: "process_terminal", detail: `pty ${message.ptyId} closed` }, true, activePty.target);
|
|
@@ -6665,11 +6912,21 @@ async function startWorker(options, projectRuntime = {
|
|
|
6665
6912
|
try {
|
|
6666
6913
|
resources = await acquireWorkerCommandLaunch(ws, message, assertMessageAdmission);
|
|
6667
6914
|
if (hasWorkspaceEffect) {
|
|
6668
|
-
|
|
6669
|
-
|
|
6670
|
-
|
|
6671
|
-
|
|
6672
|
-
|
|
6915
|
+
const reservationAbort = registerPendingReservation(`run:${message.runId}`, message.sessionId);
|
|
6916
|
+
try {
|
|
6917
|
+
await reserveWorkspaceCommandAfterCurrentSync(
|
|
6918
|
+
message.target,
|
|
6919
|
+
workspaceSyncSingleFlight,
|
|
6920
|
+
() => {
|
|
6921
|
+
workspaceSyncPriorityProcessTargets.set(message.runId, message.target);
|
|
6922
|
+
targetReserved = true;
|
|
6923
|
+
if (targetMayMutateVisibleWorkspace(message.target)) recordVisibleWorkspaceMutation(message.target);
|
|
6924
|
+
},
|
|
6925
|
+
{ signal: reservationAbort.signal }
|
|
6926
|
+
);
|
|
6927
|
+
} finally {
|
|
6928
|
+
unregisterPendingReservation(`run:${message.runId}`, reservationAbort);
|
|
6929
|
+
}
|
|
6673
6930
|
}
|
|
6674
6931
|
const runCommand = async () => {
|
|
6675
6932
|
if (!workspaceCommandTransportIsAllowed(deferredWorkspaceConfiguration, activeWorkspaceIncidentId, "exec")) {
|
|
@@ -6721,7 +6978,7 @@ async function startWorker(options, projectRuntime = {
|
|
|
6721
6978
|
cancelledProcessRuns.delete(message.runId);
|
|
6722
6979
|
}
|
|
6723
6980
|
if (infrastructureStoppedRunIds.has(message.runId)) {
|
|
6724
|
-
|
|
6981
|
+
recoveryJournal.unknown(message.requestId);
|
|
6725
6982
|
return;
|
|
6726
6983
|
}
|
|
6727
6984
|
sendSerializedWorkerMessage(ws, JSON.stringify(result));
|
|
@@ -6754,8 +7011,10 @@ async function startWorker(options, projectRuntime = {
|
|
|
6754
7011
|
const runCommand = async () => {
|
|
6755
7012
|
try {
|
|
6756
7013
|
const resolvedTarget = resolveMessageTarget(message.target);
|
|
6757
|
-
process.stdout.write(
|
|
6758
|
-
`)
|
|
7014
|
+
process.stdout.write(
|
|
7015
|
+
`[r5d-worker] exec_start ${message.runId}${message.commandClass === "control" ? " (control)" : ""}: ${message.argv.join(" ")}
|
|
7016
|
+
`
|
|
7017
|
+
);
|
|
6759
7018
|
await executeStreamingCommand({
|
|
6760
7019
|
resources,
|
|
6761
7020
|
ws,
|
|
@@ -6793,11 +7052,21 @@ async function startWorker(options, projectRuntime = {
|
|
|
6793
7052
|
try {
|
|
6794
7053
|
resources = await acquireWorkerCommandLaunch(ws, message, assertMessageAdmission);
|
|
6795
7054
|
if (hasWorkspaceEffect) {
|
|
6796
|
-
|
|
6797
|
-
|
|
6798
|
-
|
|
6799
|
-
|
|
6800
|
-
|
|
7055
|
+
const reservationAbort = registerPendingReservation(`run:${message.runId}`, message.sessionId);
|
|
7056
|
+
try {
|
|
7057
|
+
await reserveWorkspaceCommandAfterCurrentSync(
|
|
7058
|
+
message.target,
|
|
7059
|
+
workspaceSyncSingleFlight,
|
|
7060
|
+
() => {
|
|
7061
|
+
workspaceSyncPriorityProcessTargets.set(message.runId, message.target);
|
|
7062
|
+
targetReserved = true;
|
|
7063
|
+
if (targetMayMutateVisibleWorkspace(message.target)) recordVisibleWorkspaceMutation(message.target);
|
|
7064
|
+
},
|
|
7065
|
+
{ signal: reservationAbort.signal }
|
|
7066
|
+
);
|
|
7067
|
+
} finally {
|
|
7068
|
+
unregisterPendingReservation(`run:${message.runId}`, reservationAbort);
|
|
7069
|
+
}
|
|
6801
7070
|
}
|
|
6802
7071
|
await runReservedWorkspaceCommand(message.target, workspaceSyncSingleFlight, runCommand, () => {
|
|
6803
7072
|
if (targetReserved) {
|
|
@@ -6832,16 +7101,40 @@ async function startWorker(options, projectRuntime = {
|
|
|
6832
7101
|
return;
|
|
6833
7102
|
}
|
|
6834
7103
|
const reservesVisibleWorkspace = targetMayMutateVisibleWorkspace(message.target);
|
|
6835
|
-
const
|
|
7104
|
+
const mutatesFiles = message.type === "write" || message.type === "edit" || message.type === "update_task_status" || message.type === "read_plan" && message.createIfMissing;
|
|
7105
|
+
const mutatesVisibleWorkspace = mutatesFiles && targetMayMutateVisibleWorkspace(message.target);
|
|
6836
7106
|
if (reservesVisibleWorkspace) {
|
|
6837
|
-
|
|
6838
|
-
|
|
6839
|
-
|
|
6840
|
-
|
|
7107
|
+
const reservationAbort = registerPendingReservation(
|
|
7108
|
+
`op:${message.requestId}`,
|
|
7109
|
+
"sessionId" in message && typeof message.sessionId === "string" ? message.sessionId : void 0
|
|
7110
|
+
);
|
|
7111
|
+
try {
|
|
7112
|
+
await reserveWorkspaceCommandAfterCurrentSync(
|
|
7113
|
+
message.target,
|
|
7114
|
+
workspaceSyncSingleFlight,
|
|
7115
|
+
() => {
|
|
7116
|
+
workspaceSyncPriorityOperationTargets.set(message.requestId, message.target);
|
|
7117
|
+
if (mutatesVisibleWorkspace) recordVisibleWorkspaceMutation(message.target);
|
|
7118
|
+
},
|
|
7119
|
+
{ signal: reservationAbort.signal }
|
|
7120
|
+
);
|
|
7121
|
+
} catch (error) {
|
|
7122
|
+
sendSerializedWorkerMessage(
|
|
7123
|
+
ws,
|
|
7124
|
+
JSON.stringify({
|
|
7125
|
+
type: "operation_result",
|
|
7126
|
+
requestId: message.requestId,
|
|
7127
|
+
error: error instanceof Error ? error.message : String(error)
|
|
7128
|
+
})
|
|
7129
|
+
);
|
|
7130
|
+
return;
|
|
7131
|
+
} finally {
|
|
7132
|
+
unregisterPendingReservation(`op:${message.requestId}`, reservationAbort);
|
|
7133
|
+
}
|
|
6841
7134
|
}
|
|
6842
7135
|
let dirtyTrigger;
|
|
6843
7136
|
try {
|
|
6844
|
-
const
|
|
7137
|
+
const operation = runWorkspaceCommand(message.target, workspaceSyncSingleFlight, async () => {
|
|
6845
7138
|
const resolvedTarget = resolveMessageTarget(message.target);
|
|
6846
7139
|
return await executeOperation({
|
|
6847
7140
|
message,
|
|
@@ -6852,6 +7145,8 @@ async function startWorker(options, projectRuntime = {
|
|
|
6852
7145
|
assertAdmission: assertMessageAdmission
|
|
6853
7146
|
});
|
|
6854
7147
|
});
|
|
7148
|
+
if (mutatesFiles && typeof message.sessionId === "string") sessionFileMutations.track(message.sessionId, operation);
|
|
7149
|
+
const result = await operation;
|
|
6855
7150
|
sendSerializedWorkerMessage(
|
|
6856
7151
|
ws,
|
|
6857
7152
|
JSON.stringify({
|
|
@@ -6935,6 +7230,7 @@ async function startWorker(options, projectRuntime = {
|
|
|
6935
7230
|
projectRuntime.configuration.deferred = deferredWorkspaceConfiguration;
|
|
6936
7231
|
}
|
|
6937
7232
|
if (currentWorkerSocket === ws) currentWorkerSocket = null;
|
|
7233
|
+
abortPendingReservations("Worker connection closed before command start");
|
|
6938
7234
|
workerRecoveryId = "reconnecting";
|
|
6939
7235
|
const reason = event.reason ? `: ${event.reason}` : "";
|
|
6940
7236
|
process.stderr.write(`[r5d-worker] disconnected (${event.code}${reason})
|
|
@@ -6968,6 +7264,11 @@ async function startWorker(options, projectRuntime = {
|
|
|
6968
7264
|
});
|
|
6969
7265
|
}
|
|
6970
7266
|
function installWorkerRuntimeFailureLogging() {
|
|
7267
|
+
installWorkspaceFilesystemExecutorFailureHandler((error) => {
|
|
7268
|
+
process.stderr.write(`[r5d-worker] ${error.message}; restarting the worker runtime
|
|
7269
|
+
`);
|
|
7270
|
+
void exitWorkerRuntime(WORKER_RECONNECT_EXIT_CODE);
|
|
7271
|
+
});
|
|
6971
7272
|
process.on("uncaughtException", (error) => {
|
|
6972
7273
|
process.stderr.write(`[r5d-worker] uncaught exception: ${error instanceof Error ? error.stack ?? error.message : String(error)}
|
|
6973
7274
|
`);
|
|
@@ -6980,11 +7281,6 @@ function installWorkerRuntimeFailureLogging() {
|
|
|
6980
7281
|
});
|
|
6981
7282
|
}
|
|
6982
7283
|
async function main() {
|
|
6983
|
-
if (process.argv[2] === PROJECT_SNAPSHOT_RECOVERY_HELPER_COMMAND) {
|
|
6984
|
-
const exitCode2 = runProjectSnapshotRecoveryHelper(process.argv.slice(3));
|
|
6985
|
-
if (exitCode2 !== 0) process.exit(exitCode2);
|
|
6986
|
-
return;
|
|
6987
|
-
}
|
|
6988
7284
|
const parsed = parseArgs(process.argv.slice(2));
|
|
6989
7285
|
if (parsed.command === "help") {
|
|
6990
7286
|
printHelp();
|