@ricsam/r5d-worker 0.0.122 → 0.0.124

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/README.md +6 -3
  2. package/dist/cjs/command-launcher.cjs +76 -2
  3. package/dist/cjs/control-command-policy.cjs +177 -0
  4. package/dist/cjs/main.cjs +541 -247
  5. package/dist/cjs/package.json +1 -1
  6. package/dist/cjs/project-checkout-garbage.cjs +32 -3
  7. package/dist/cjs/project-workspace-state.cjs +51 -35
  8. package/dist/cjs/project-worktrees.cjs +79 -34
  9. package/dist/cjs/recovery-journal-protocol.cjs +56 -0
  10. package/dist/cjs/recovery-journal-runtime.cjs +133 -0
  11. package/dist/cjs/recovery-journal-thread.cjs +9 -0
  12. package/dist/cjs/recovery-journal.cjs +735 -0
  13. package/dist/cjs/recovery-store.cjs +8 -0
  14. package/dist/cjs/session-file-mutations.cjs +61 -0
  15. package/dist/cjs/working-tree-mirror.cjs +1 -0
  16. package/dist/cjs/workspace-command-sync-policy.cjs +37 -8
  17. package/dist/cjs/workspace-filesystem-executor-thread.cjs +36 -0
  18. package/dist/cjs/workspace-filesystem-executor.cjs +327 -0
  19. package/dist/cjs/workspace-filesystem-job-types.cjs +134 -0
  20. package/dist/cjs/workspace-filesystem-jobs.cjs +57 -0
  21. package/dist/cjs/workspace-git-sync.cjs +275 -201
  22. package/dist/cjs/workspace-mount-hold-fence.cjs +120 -0
  23. package/dist/mjs/command-launcher.mjs +75 -2
  24. package/dist/mjs/control-command-policy.mjs +146 -0
  25. package/dist/mjs/main.mjs +549 -253
  26. package/dist/mjs/package.json +1 -1
  27. package/dist/mjs/project-checkout-garbage.mjs +30 -2
  28. package/dist/mjs/project-workspace-state.mjs +51 -35
  29. package/dist/mjs/project-worktrees.mjs +74 -34
  30. package/dist/mjs/recovery-journal-protocol.mjs +30 -0
  31. package/dist/mjs/recovery-journal-runtime.mjs +112 -0
  32. package/dist/mjs/recovery-journal-thread.mjs +8 -0
  33. package/dist/mjs/recovery-journal.mjs +687 -0
  34. package/dist/mjs/recovery-store.mjs +8 -0
  35. package/dist/mjs/session-file-mutations.mjs +37 -0
  36. package/dist/mjs/working-tree-mirror.mjs +1 -0
  37. package/dist/mjs/workspace-command-sync-policy.mjs +37 -8
  38. package/dist/mjs/workspace-filesystem-executor-thread.mjs +38 -0
  39. package/dist/mjs/workspace-filesystem-executor.mjs +287 -0
  40. package/dist/mjs/workspace-filesystem-job-types.mjs +106 -0
  41. package/dist/mjs/workspace-filesystem-jobs.mjs +51 -0
  42. package/dist/mjs/workspace-git-sync.mjs +264 -202
  43. package/dist/mjs/workspace-mount-hold-fence.mjs +95 -0
  44. package/dist/types/command-launcher.d.ts +42 -0
  45. package/dist/types/control-command-policy.d.ts +53 -0
  46. package/dist/types/main.d.ts +31 -19
  47. package/dist/types/project-checkout-garbage.d.ts +19 -2
  48. package/dist/types/project-workspace-state.d.ts +9 -9
  49. package/dist/types/project-worktrees.d.ts +49 -5
  50. package/dist/types/recovery-journal-protocol.d.ts +35 -0
  51. package/dist/types/recovery-journal-runtime.d.ts +12 -0
  52. package/dist/types/recovery-journal-stall-fixture.d.ts +1 -0
  53. package/dist/types/recovery-journal-thread.d.ts +1 -0
  54. package/dist/types/recovery-journal.d.ts +246 -0
  55. package/dist/types/recovery-store.d.ts +6 -0
  56. package/dist/types/session-file-mutations.d.ts +22 -0
  57. package/dist/types/workspace-command-sync-policy.d.ts +22 -7
  58. package/dist/types/workspace-filesystem-executor-thread.d.ts +1 -0
  59. package/dist/types/workspace-filesystem-executor.d.ts +123 -0
  60. package/dist/types/workspace-filesystem-job-types.d.ts +246 -0
  61. package/dist/types/workspace-filesystem-jobs.d.ts +7 -0
  62. package/dist/types/workspace-git-sync.d.ts +113 -7
  63. package/dist/types/workspace-mount-hold-fence.d.ts +42 -0
  64. package/package.json +1 -1
  65. package/dist/cjs/project-snapshot-recovery-runner.cjs +0 -171
  66. package/dist/mjs/project-snapshot-recovery-runner.mjs +0 -135
  67. package/dist/types/project-snapshot-recovery-runner.d.ts +0 -10
package/dist/mjs/main.mjs CHANGED
@@ -4,7 +4,9 @@ import { getWorkerPlanPath, getWorkerPlansPath, validatePlanId } from "./plan-pa
4
4
  import { createPlanMarkdown, updateTaskCheckbox } from "./plan-parser.mjs";
5
5
  import { lintPlans, formatPlanLintErrors } from "./plan-lint.mjs";
6
6
  import fs from "node:fs";
7
- import { WorkerRecoveryStore, workerOperationDigest } from "./recovery-store.mjs";
7
+ import { WorkerRecoveryJournal, WorkerRecoveryJournalBusyError } from "./recovery-journal.mjs";
8
+ import { SessionFileMutationTracker } from "./session-file-mutations.mjs";
9
+ import { workerOperationDigest } from "./recovery-store.mjs";
8
10
  import { WorkerOutputWindow } from "./recovery-output-window.mjs";
9
11
  import {
10
12
  createWorkerCommandLauncher,
@@ -26,6 +28,12 @@ import { installCliUpdate, readInstalledCliVersion } from "./cli-update.mjs";
26
28
  import { gitTransportSecurityArgs, workerGitProcessEnvironment } from "./git-process-environment.mjs";
27
29
  import { terminateProcessTree } from "./process-tree.mjs";
28
30
  import { ProjectCheckoutGarbageCollector } from "./project-checkout-garbage.mjs";
31
+ import {
32
+ assertControlCommandArgv,
33
+ CONTROL_COMMAND_RUNTIME_MS,
34
+ controlCommandEnvironment,
35
+ resolveControlCommandExecutable
36
+ } from "./control-command-policy.mjs";
29
37
  import { createPtyOutputCoalescer } from "./pty-output-coalescer.mjs";
30
38
  import { openWorkerPortForwardRelay } from "./port-forward-client.mjs";
31
39
  import {
@@ -52,9 +60,11 @@ import {
52
60
  runWorkspaceCommand
53
61
  } from "./workspace-command-sync-policy.mjs";
54
62
  import { WorkspaceMutationGate } from "./workspace-mutation-gate.mjs";
63
+ import { WorkspaceMountHoldAbortedError, WorkspaceMountHoldFence } from "./workspace-mount-hold-fence.mjs";
64
+ import { installWorkspaceFilesystemExecutorFailureHandler, workspaceFilesystemExecutor } from "./workspace-filesystem-executor.mjs";
55
65
  import { WorkspaceProjectionLedger } from "./workspace-projection-ledger.mjs";
56
66
  import { WorkspaceSyncCoalescer } from "./workspace-sync-coalescer.mjs";
57
- import { checkoutPathMovePublicationComplete, preserveProjectCheckoutPathMove } from "./workspace-path-move.mjs";
67
+ import { checkoutPathMovePublicationComplete } from "./workspace-path-move.mjs";
58
68
  import { busyProjectConfigurationChangeIds, deferredProjectConfigurationPendingBranches } from "./workspace-project-config-policy.mjs";
59
69
  import { creatorLocalProjectBranchIsAuthorized, workerProjectBranchDisposition } from "./workspace-preserve-only-policy.mjs";
60
70
  import { assertProjectBranchDeletionIncarnation } from "./workspace-branch-incarnation-policy.mjs";
@@ -84,6 +94,7 @@ import {
84
94
  deleteProjectMirrorBranch,
85
95
  ensureProjectWorktrees,
86
96
  inventoryProjectCheckouts,
97
+ PROJECT_WORKTREE_SNAPSHOT_OWNER_SESSION_ID,
87
98
  observeProjectMirrorHeads,
88
99
  projectBranchMayExistAfterCreateFailure,
89
100
  projectOriginBranchName,
@@ -96,11 +107,6 @@ import {
96
107
  parseProjectMirrorRefsTokensMessage,
97
108
  planProjectMirrorObservationFetch
98
109
  } from "./project-mirror-fetch-policy.mjs";
99
- import {
100
- PROJECT_SNAPSHOT_RECOVERY_HELPER_COMMAND,
101
- recoverProjectSnapshotsInChild,
102
- runProjectSnapshotRecoveryHelper
103
- } from "./project-snapshot-recovery-runner.mjs";
104
110
  import {
105
111
  ProjectWorkspacePendingBranchPathChangeError,
106
112
  ProjectWorkspaceStateStore,
@@ -129,7 +135,7 @@ function workerServerHeartbeatAction(input) {
129
135
  return input.probeElapsedMs >= input.timeoutMs ? "terminate" : "wait";
130
136
  }
131
137
  const WORKSPACE_HYDRATION_LEASE_OPERATION_PREFIX = "hydration:";
132
- function createWorkspaceHydrationLeaseHooks(send, options = {}) {
138
+ function createWorkspaceHydrationLeaseHooks(send, options) {
133
139
  const yieldToEventLoop = options.yieldToEventLoop ?? (() => new Promise((resolve) => setImmediate(resolve)));
134
140
  const log = options.log ?? ((line) => process.stdout.write(`${line}
135
141
  `));
@@ -150,13 +156,17 @@ function createWorkspaceHydrationLeaseHooks(send, options = {}) {
150
156
  log(
151
157
  `[r5d-worker] released hydration lease ${context.transactionId} after ${startedAt === void 0 ? "?" : Math.round(now() - startedAt)}ms`
152
158
  );
159
+ },
160
+ holdMounts(mountIds, description) {
161
+ return options.fence.hold(mountIds, description);
153
162
  }
154
163
  };
155
164
  }
156
165
  const workerLifecycleTestHarness = {
157
166
  boundedWorkerLifecycleProgress,
158
167
  workerServerHeartbeatAction,
159
- createWorkspaceHydrationLeaseHooks
168
+ createWorkspaceHydrationLeaseHooks,
169
+ workerCommandHasWorkspaceEffect
160
170
  };
161
171
  class ProjectWorkspaceConfigurationDeferredError extends Error {
162
172
  }
@@ -177,6 +187,26 @@ const PTY_FOREGROUND_POLL_MS = 1e3;
177
187
  const PTY_FOREGROUND_IDLE_ENABLED = process.env.R5D_PTY_FOREGROUND_IDLE !== "0";
178
188
  const PTY_TMP_PATH_PREFIX = "r5d-worker-tmp://";
179
189
  let workerCommandLauncher;
190
+ const trustedControlPath = process.env.PATH;
191
+ let capacityReportTimer;
192
+ let capacityReportSocket = null;
193
+ let lastCapacityReport;
194
+ function scheduleCapacityReport(force = false) {
195
+ if (force) lastCapacityReport = void 0;
196
+ if (capacityReportTimer) return;
197
+ capacityReportTimer = setTimeout(() => {
198
+ capacityReportTimer = void 0;
199
+ const ws = capacityReportSocket;
200
+ if (!ws || ws.readyState !== WebSocket.OPEN || !workerCommandLauncher) return;
201
+ const capacity = { ...workerCommandLauncher.capacityReport(), reportedAt: (/* @__PURE__ */ new Date()).toISOString() };
202
+ const { reportedAt: _reportedAt, ...comparable } = capacity;
203
+ const serialized = JSON.stringify(comparable);
204
+ if (serialized === lastCapacityReport) return;
205
+ lastCapacityReport = serialized;
206
+ sendWorkerMessage(ws, { type: "capacity_report", capacity });
207
+ }, 100);
208
+ capacityReportTimer.unref?.();
209
+ }
180
210
  const activeProcesses = /* @__PURE__ */ new Map();
181
211
  const credentialBearingProcessGroups = /* @__PURE__ */ new Map();
182
212
  const credentialBearingProcessGroupTargets = /* @__PURE__ */ new Map();
@@ -189,7 +219,7 @@ const workspaceSyncPriorityPtyTargets = /* @__PURE__ */ new Map();
189
219
  let currentWorkerSocket = null;
190
220
  const workerRuntimeId = crypto.randomUUID();
191
221
  const workerOutputWindow = new WorkerOutputWindow();
192
- let recoveryStore;
222
+ let recoveryJournal;
193
223
  let workerReconnectAttempt = 0;
194
224
  let workerRecoveryId;
195
225
  let workerExecutionLeaseDeadline = 0;
@@ -201,13 +231,13 @@ let runtimeExitPromise;
201
231
  function fenceWorkerRuntimeOperations() {
202
232
  launcherFailureInProgress = true;
203
233
  workerAdmissionGeneration += 1;
204
- for (const operation of recoveryStore?.operations() ?? []) {
205
- if (operation.state !== "completed") recoveryStore?.unknown(operation.requestId);
234
+ for (const operation of recoveryJournal?.activeAndPendingOperations() ?? []) {
235
+ if (operation.state !== "completed") recoveryJournal?.unknown(operation.requestId);
206
236
  }
207
237
  for (const runId of workerCommandLauncher?.activeIds ?? []) {
208
238
  infrastructureStoppedRunIds.add(runId);
209
239
  cancelledProcessRuns.add(runId);
210
- recoveryStore?.unknownRun(runId);
240
+ recoveryJournal?.unknownRun(runId);
211
241
  }
212
242
  for (const runId of activeProcesses.keys()) infrastructureStoppedRunIds.add(runId);
213
243
  }
@@ -216,13 +246,21 @@ function exitWorkerRuntime(code) {
216
246
  fenceWorkerRuntimeOperations();
217
247
  runtimeExitPromise = (async () => {
218
248
  await Promise.race([
219
- Promise.allSettled([terminateActiveCredentialBearingChildren(), workerCommandLauncher?.close()]),
249
+ Promise.allSettled([terminateActiveCredentialBearingChildren(), workerCommandLauncher?.close(), recoveryJournal?.flush()]),
220
250
  Bun.sleep(4e3)
221
251
  ]);
222
252
  process.exit(code);
223
253
  })();
224
254
  return runtimeExitPromise;
225
255
  }
256
+ function handleRecoveryJournalFailure(error) {
257
+ if (launcherFailureInProgress) return;
258
+ fenceWorkerRuntimeOperations();
259
+ process.stderr.write(`[r5d-worker] ${error.message}; stopping runtime for recovery journal restart
260
+ `);
261
+ currentWorkerSocket?.close(1012, "Recovery journal failure");
262
+ void exitWorkerRuntime(WORKER_RECONNECT_EXIT_CODE);
263
+ }
226
264
  function handleWorkerCommandLauncherFailure(error) {
227
265
  if (!workerCommandLauncher || launcherFailureInProgress) return;
228
266
  fenceWorkerRuntimeOperations();
@@ -243,7 +281,7 @@ async function retryWorkerServerRead(read) {
243
281
  try {
244
282
  return await read();
245
283
  } catch (error) {
246
- if (!recoveryStore || !isWorkerCommunicationFailure(error instanceof Error ? error.message : String(error))) throw error;
284
+ if (!recoveryJournal || !isWorkerCommunicationFailure(error instanceof Error ? error.message : String(error))) throw error;
247
285
  deadline ??= Date.now() + WORKER_RECOVERY_TIMEOUT_MS;
248
286
  if (workerLeaseExpired || Date.now() >= deadline) throw new WorkerServerUnavailableError("Worker communication recovery expired");
249
287
  currentWorkerSocket?.close(1012, "Communication recovery");
@@ -265,9 +303,10 @@ function renewWorkerExecutionLease(leaseMs) {
265
303
  `);
266
304
  });
267
305
  closeAllPtys();
268
- for (const operation of recoveryStore?.operations() ?? []) {
306
+ for (const operation of recoveryJournal?.activeAndPendingOperations() ?? []) {
269
307
  if (operation.sessionId && operation.state !== "completed") {
270
- recoveryStore?.cancelSession(operation.sessionId);
308
+ void recoveryJournal?.cancelSession(operation.sessionId);
309
+ abortPendingReservations("Execution lease expired before command start", operation.sessionId);
271
310
  void workerCommandLauncher?.cancelSession(operation.sessionId).catch((error) => {
272
311
  process.stderr.write(`[r5d-worker] expired command launcher cancellation failed: ${error}
273
312
  `);
@@ -277,7 +316,7 @@ function renewWorkerExecutionLease(leaseMs) {
277
316
  for (const [runId, active] of activeProcesses) {
278
317
  if (!active.sessionId) continue;
279
318
  infrastructureStoppedRunIds.add(runId);
280
- recoveryStore?.unknownRun(runId);
319
+ recoveryJournal?.unknownRun(runId);
281
320
  cancelledProcessRuns.add(runId);
282
321
  closeProcessStdin(active);
283
322
  void terminateProcessTree(active.process).catch(
@@ -292,6 +331,31 @@ function renewWorkerExecutionLease(leaseMs) {
292
331
  }
293
332
  let workerAdmissionGeneration = 0;
294
333
  const workspaceMutationGate = new WorkspaceMutationGate();
334
+ const workspaceMountHoldFence = new WorkspaceMountHoldFence();
335
+ const projectMountId = (projectId, branchName) => `project:${projectId}:${encodeURIComponent(branchName)}`;
336
+ const pendingReservationAborts = /* @__PURE__ */ new Map();
337
+ const sessionFileMutations = new SessionFileMutationTracker();
338
+ function registerPendingReservation(key, sessionId) {
339
+ abortPendingReservation(key, "Reservation superseded by a newer request with the same id");
340
+ const controller = new AbortController();
341
+ pendingReservationAborts.set(key, { controller, sessionId });
342
+ return controller;
343
+ }
344
+ function unregisterPendingReservation(key, controller) {
345
+ if (pendingReservationAborts.get(key)?.controller === controller) pendingReservationAborts.delete(key);
346
+ }
347
+ function abortPendingReservation(key, reason) {
348
+ const pending = pendingReservationAborts.get(key);
349
+ if (!pending) return;
350
+ pendingReservationAborts.delete(key);
351
+ pending.controller.abort(new WorkspaceMountHoldAbortedError(reason));
352
+ }
353
+ function abortPendingReservations(reason, sessionId) {
354
+ for (const [key, pending] of [...pendingReservationAborts]) {
355
+ if (sessionId !== void 0 && pending.sessionId !== sessionId) continue;
356
+ abortPendingReservation(key, reason);
357
+ }
358
+ }
295
359
  let workspaceSyncQueue = Promise.resolve();
296
360
  let startupProjectSnapshotRecoveryCompleted = false;
297
361
  const checkoutGarbageCollectors = /* @__PURE__ */ new Map();
@@ -326,6 +390,17 @@ const workspaceSyncSingleFlight = {
326
390
  },
327
391
  afterCurrent() {
328
392
  return workspaceSyncQueue;
393
+ },
394
+ mountHold(target, signal) {
395
+ if (target.type !== "project") return null;
396
+ const mountIds = [projectMountId(target.projectId, target.branchName)];
397
+ if (!workspaceMountHoldFence.isHeld(mountIds)) return null;
398
+ const holder = workspaceMountHoldFence.describeHold(mountIds);
399
+ process.stdout.write(
400
+ `[r5d-worker] command for ${target.projectId}/${target.branchName} waits for ${holder ?? "a workspace filesystem job"}
401
+ `
402
+ );
403
+ return workspaceMountHoldFence.waitForRelease(mountIds, { signal });
329
404
  }
330
405
  };
331
406
  let githubCredential = null;
@@ -982,14 +1057,15 @@ async function runGitAsync(args, options = {}) {
982
1057
  env: workerGitProcessEnvironment(),
983
1058
  ...bounded ? { detached: true } : {}
984
1059
  });
985
- const completion = Promise.all([
986
- new Response(subprocess.stdout).text(),
987
- new Response(subprocess.stderr).text(),
988
- subprocess.exited
989
- ]).then(([stdout, stderr, exitCode]) => ({ stdout, stderr, exitCode }));
1060
+ const completion = Promise.all([new Response(subprocess.stdout).text(), new Response(subprocess.stderr).text(), subprocess.exited]).then(
1061
+ ([stdout, stderr, exitCode]) => ({ stdout, stderr, exitCode })
1062
+ );
990
1063
  let timer;
991
1064
  const timedOut = bounded ? new Promise((_resolve, reject) => {
992
- timer = setTimeout(() => reject(new GitTransportTimeoutError(`git ${args.join(" ")} timed out after ${options.timeoutMs}ms`)), options.timeoutMs);
1065
+ timer = setTimeout(
1066
+ () => reject(new GitTransportTimeoutError(`git ${args.join(" ")} timed out after ${options.timeoutMs}ms`)),
1067
+ options.timeoutMs
1068
+ );
993
1069
  }) : null;
994
1070
  try {
995
1071
  const { stdout, stderr, exitCode } = timedOut ? await Promise.race([completion, timedOut]) : await completion;
@@ -2078,12 +2154,12 @@ async function streamCommandOutput(stream, onData) {
2078
2154
  }
2079
2155
  const text = decoder.decode(chunk.value, { stream: true });
2080
2156
  if (text.length > 0) {
2081
- onData(text);
2157
+ await onData(text);
2082
2158
  }
2083
2159
  }
2084
2160
  const trailing = decoder.decode();
2085
2161
  if (trailing.length > 0) {
2086
- onData(trailing);
2162
+ await onData(trailing);
2087
2163
  }
2088
2164
  } finally {
2089
2165
  reader.releaseLock();
@@ -2222,12 +2298,21 @@ function mutationQueueKey(target, resolved) {
2222
2298
  }
2223
2299
  return `${describeWorkerSessionTarget(target)}:${path.posix.normalize(resolved.repoRelativePath)}`;
2224
2300
  }
2225
- function readWorkerTextFile(branchPath, filePath, offset, limit, builtInPaths) {
2301
+ async function statIfExists(targetPath) {
2302
+ try {
2303
+ return await fs.promises.stat(targetPath);
2304
+ } catch (error) {
2305
+ const code = error.code;
2306
+ if (code === "ENOENT" || code === "ENOTDIR") return null;
2307
+ throw error;
2308
+ }
2309
+ }
2310
+ async function readWorkerTextFile(branchPath, filePath, offset, limit, builtInPaths) {
2226
2311
  const resolved = resolveWorkerFilePath(branchPath, filePath, builtInPaths);
2227
- if (!fs.existsSync(resolved.absolutePath) || !fs.statSync(resolved.absolutePath).isFile()) {
2312
+ if (!(await statIfExists(resolved.absolutePath))?.isFile()) {
2228
2313
  throw new Error(`File not found: ${resolved.displayPath}`);
2229
2314
  }
2230
- const buffer = fs.readFileSync(resolved.absolutePath);
2315
+ const buffer = await fs.promises.readFile(resolved.absolutePath);
2231
2316
  const formatted = formatLineNumberedContent(buffer.toString("utf8"), offset, limit);
2232
2317
  return {
2233
2318
  type: "read",
@@ -2237,25 +2322,25 @@ function readWorkerTextFile(branchPath, filePath, offset, limit, builtInPaths) {
2237
2322
  ...formatted
2238
2323
  };
2239
2324
  }
2240
- function writeWorkerTextFile(branchPath, filePath, content, builtInPaths) {
2325
+ async function writeWorkerTextFile(branchPath, filePath, content, builtInPaths) {
2241
2326
  const resolved = resolveWorkerFilePath(branchPath, filePath, builtInPaths);
2242
- fs.mkdirSync(path.dirname(resolved.absolutePath), { recursive: true });
2243
- fs.writeFileSync(resolved.absolutePath, content, "utf8");
2327
+ await fs.promises.mkdir(path.dirname(resolved.absolutePath), { recursive: true });
2328
+ await fs.promises.writeFile(resolved.absolutePath, content, "utf8");
2244
2329
  return {
2245
2330
  type: "write",
2246
2331
  file: resolved.displayPath,
2247
2332
  gitBlobHash: getGitBlobHashForContent(content)
2248
2333
  };
2249
2334
  }
2250
- function editWorkerTextFile(branchPath, filePath, edits, builtInPaths) {
2335
+ async function editWorkerTextFile(branchPath, filePath, edits, builtInPaths) {
2251
2336
  const resolved = resolveWorkerFilePath(branchPath, filePath, builtInPaths);
2252
2337
  if (!Array.isArray(edits) || edits.length === 0) {
2253
2338
  throw new Error("edit requires at least one replacement");
2254
2339
  }
2255
- if (!fs.existsSync(resolved.absolutePath) || !fs.statSync(resolved.absolutePath).isFile()) {
2340
+ if (!(await statIfExists(resolved.absolutePath))?.isFile()) {
2256
2341
  throw new Error(`File not found: ${resolved.displayPath}`);
2257
2342
  }
2258
- const originalContent = fs.readFileSync(resolved.absolutePath, "utf8");
2343
+ const originalContent = await fs.promises.readFile(resolved.absolutePath, "utf8");
2259
2344
  const ranges = edits.map((edit, index) => {
2260
2345
  if (edit.oldText === edit.newText) {
2261
2346
  throw new Error(`edits[${index}].oldText and edits[${index}].newText must be different`);
@@ -2287,7 +2372,7 @@ function editWorkerTextFile(branchPath, filePath, edits, builtInPaths) {
2287
2372
  for (const range of [...sortedRanges].reverse()) {
2288
2373
  nextContent = nextContent.slice(0, range.start) + range.edit.newText + nextContent.slice(range.end);
2289
2374
  }
2290
- fs.writeFileSync(resolved.absolutePath, nextContent, "utf8");
2375
+ await fs.promises.writeFile(resolved.absolutePath, nextContent, "utf8");
2291
2376
  return {
2292
2377
  type: "edit",
2293
2378
  file: resolved.displayPath,
@@ -2320,7 +2405,7 @@ async function executePlanOperation(input) {
2320
2405
  let created = false;
2321
2406
  if (message.createIfMissing) {
2322
2407
  try {
2323
- fs.writeFileSync(resolved.absolutePath, createPlanMarkdown(message.planId), { encoding: "utf8", flag: "wx" });
2408
+ await fs.promises.writeFile(resolved.absolutePath, createPlanMarkdown(message.planId), { encoding: "utf8", flag: "wx" });
2324
2409
  created = true;
2325
2410
  } catch (error) {
2326
2411
  if (error.code !== "EEXIST") throw error;
@@ -2329,11 +2414,16 @@ async function executePlanOperation(input) {
2329
2414
  return {
2330
2415
  type: "read_plan",
2331
2416
  created,
2332
- read: readWorkerTextFile(resolvedTarget.rootPath, filePath, void 0, void 0, builtInPaths)
2417
+ read: await readWorkerTextFile(resolvedTarget.rootPath, filePath, void 0, void 0, builtInPaths)
2333
2418
  };
2334
2419
  }
2335
- const content = fs.readFileSync(resolved.absolutePath, "utf8");
2336
- writeWorkerTextFile(resolvedTarget.rootPath, filePath, updateTaskCheckbox(content, message.taskId, message.checked), builtInPaths);
2420
+ const content = await fs.promises.readFile(resolved.absolutePath, "utf8");
2421
+ await writeWorkerTextFile(
2422
+ resolvedTarget.rootPath,
2423
+ filePath,
2424
+ updateTaskCheckbox(content, message.taskId, message.checked),
2425
+ builtInPaths
2426
+ );
2337
2427
  return {
2338
2428
  type: "update_task_status",
2339
2429
  planId: message.planId,
@@ -2354,7 +2444,11 @@ async function executeReadFileOperation(input) {
2354
2444
  rootDir: input.rootDir,
2355
2445
  access: "read"
2356
2446
  });
2357
- return readWorkerTextFile(input.resolvedTarget.rootPath, input.message.filePath, input.message.offset, input.message.limit, builtInPaths);
2447
+ const resolved = resolveWorkerFilePath(input.resolvedTarget.rootPath, input.message.filePath, builtInPaths);
2448
+ return withFileMutationQueue(
2449
+ mutationQueueKey(input.resolvedTarget.target, resolved),
2450
+ () => readWorkerTextFile(input.resolvedTarget.rootPath, input.message.filePath, input.message.offset, input.message.limit, builtInPaths)
2451
+ );
2358
2452
  }
2359
2453
  async function executeWriteFileOperation(input) {
2360
2454
  const builtInPaths = await prepareBuiltInToolPathsForTarget({
@@ -2407,16 +2501,16 @@ function wildcardToRegex(pattern) {
2407
2501
  function normalizeFindPattern(pattern) {
2408
2502
  return /[*?]/.test(pattern) ? pattern : `*${pattern}*`;
2409
2503
  }
2410
- function walkWorkerEntries(branchPath, start) {
2504
+ async function walkWorkerEntries(branchPath, start) {
2411
2505
  const entries = [];
2412
2506
  const visitedDirectories = /* @__PURE__ */ new Set();
2413
- const realVirtualRoot = start.scope === "virtual" ? fs.realpathSync(start.virtualRootPath) : null;
2414
- const visit = (absolutePath, isRoot) => {
2507
+ const realVirtualRoot = start.scope === "virtual" ? await fs.promises.realpath(start.virtualRootPath) : null;
2508
+ const visit = async (absolutePath, isRoot) => {
2415
2509
  let stat;
2416
2510
  try {
2417
- stat = fs.statSync(absolutePath);
2511
+ stat = await fs.promises.stat(absolutePath);
2418
2512
  if (realVirtualRoot) {
2419
- assertInsideRoot(realVirtualRoot, fs.realpathSync(absolutePath), `${start.displayPath} path`);
2513
+ assertInsideRoot(realVirtualRoot, await fs.promises.realpath(absolutePath), `${start.displayPath} path`);
2420
2514
  }
2421
2515
  } catch (error) {
2422
2516
  if (isRoot) throw error;
@@ -2429,7 +2523,7 @@ function walkWorkerEntries(branchPath, start) {
2429
2523
  if (!stat.isDirectory()) return;
2430
2524
  let realDirectoryPath;
2431
2525
  try {
2432
- realDirectoryPath = fs.realpathSync(absolutePath);
2526
+ realDirectoryPath = await fs.promises.realpath(absolutePath);
2433
2527
  } catch (error) {
2434
2528
  if (isRoot) throw error;
2435
2529
  return;
@@ -2438,25 +2532,25 @@ function walkWorkerEntries(branchPath, start) {
2438
2532
  visitedDirectories.add(realDirectoryPath);
2439
2533
  let children;
2440
2534
  try {
2441
- children = fs.readdirSync(absolutePath, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name));
2535
+ children = (await fs.promises.readdir(absolutePath, { withFileTypes: true })).sort((a, b) => a.name.localeCompare(b.name));
2442
2536
  } catch (error) {
2443
2537
  if (isRoot) throw error;
2444
2538
  return;
2445
2539
  }
2446
2540
  for (const child of children) {
2447
2541
  if (IGNORED_ENTRY_NAMES.has(child.name)) continue;
2448
- visit(path.join(absolutePath, child.name), false);
2542
+ await visit(path.join(absolutePath, child.name), false);
2449
2543
  }
2450
2544
  };
2451
- visit(start.absolutePath, true);
2545
+ await visit(start.absolutePath, true);
2452
2546
  return entries;
2453
2547
  }
2454
2548
  function isProbablyText(bytes) {
2455
2549
  return !bytes.subarray(0, Math.min(bytes.length, 4096)).includes(0);
2456
2550
  }
2457
- function grepWorkerFiles(branchPath, input, builtInPaths) {
2551
+ async function grepWorkerFiles(branchPath, input, builtInPaths) {
2458
2552
  const start = resolveWorkerFilePath(branchPath, input.path ?? ".", builtInPaths);
2459
- if (!fs.existsSync(start.absolutePath)) {
2553
+ if (!await statIfExists(start.absolutePath)) {
2460
2554
  throw new Error(`Path not found: ${start.displayPath}`);
2461
2555
  }
2462
2556
  const limit = normalizeToolLimit(input.limit, 100, 1e3);
@@ -2466,12 +2560,12 @@ function grepWorkerFiles(branchPath, input, builtInPaths) {
2466
2560
  const lines = [];
2467
2561
  let matchCount = 0;
2468
2562
  let truncated = false;
2469
- for (const entry of walkWorkerEntries(branchPath, start)) {
2563
+ for (const entry of await walkWorkerEntries(branchPath, start)) {
2470
2564
  if (!entry.stat.isFile()) continue;
2471
2565
  if (globRegex && !globRegex.test(entry.matchPath)) continue;
2472
2566
  let bytes;
2473
2567
  try {
2474
- bytes = fs.readFileSync(entry.absolutePath);
2568
+ bytes = await fs.promises.readFile(entry.absolutePath);
2475
2569
  } catch {
2476
2570
  continue;
2477
2571
  }
@@ -2524,9 +2618,9 @@ async function executeGrepOperation(input) {
2524
2618
  builtInPaths
2525
2619
  );
2526
2620
  }
2527
- function findWorkerFiles(branchPath, input, builtInPaths) {
2621
+ async function findWorkerFiles(branchPath, input, builtInPaths) {
2528
2622
  const start = resolveWorkerFilePath(branchPath, input.path ?? ".", builtInPaths);
2529
- if (!fs.existsSync(start.absolutePath) || !fs.statSync(start.absolutePath).isDirectory()) {
2623
+ if (!(await statIfExists(start.absolutePath))?.isDirectory()) {
2530
2624
  throw new Error(`Directory not found: ${start.displayPath}`);
2531
2625
  }
2532
2626
  const pattern = normalizeFindPattern(input.pattern ?? "*");
@@ -2536,7 +2630,7 @@ function findWorkerFiles(branchPath, input, builtInPaths) {
2536
2630
  const matches = [];
2537
2631
  let count = 0;
2538
2632
  let truncated = false;
2539
- for (const entry of walkWorkerEntries(branchPath, start)) {
2633
+ for (const entry of await walkWorkerEntries(branchPath, start)) {
2540
2634
  if (entry.absolutePath === start.absolutePath) continue;
2541
2635
  const isDirectory = entry.stat.isDirectory();
2542
2636
  if (entryType === "file" && !entry.stat.isFile()) continue;
@@ -2581,13 +2675,13 @@ async function executeFindOperation(input) {
2581
2675
  builtInPaths
2582
2676
  );
2583
2677
  }
2584
- function listWorkerDirectory(branchPath, inputPath = ".", inputLimit, builtInPaths) {
2678
+ async function listWorkerDirectory(branchPath, inputPath = ".", inputLimit, builtInPaths) {
2585
2679
  const resolved = resolveWorkerFilePath(branchPath, inputPath, builtInPaths);
2586
- if (!fs.existsSync(resolved.absolutePath) || !fs.statSync(resolved.absolutePath).isDirectory()) {
2680
+ if (!(await statIfExists(resolved.absolutePath))?.isDirectory()) {
2587
2681
  throw new Error(`Directory not found: ${resolved.displayPath}`);
2588
2682
  }
2589
2683
  const limit = normalizeToolLimit(inputLimit, 200, 1e3);
2590
- const entries = fs.readdirSync(resolved.absolutePath, { withFileTypes: true }).filter((entry) => !IGNORED_ENTRY_NAMES.has(entry.name)).sort((a, b) => Number(b.isDirectory()) - Number(a.isDirectory()) || a.name.localeCompare(b.name));
2684
+ const entries = (await fs.promises.readdir(resolved.absolutePath, { withFileTypes: true })).filter((entry) => !IGNORED_ENTRY_NAMES.has(entry.name)).sort((a, b) => Number(b.isDirectory()) - Number(a.isDirectory()) || a.name.localeCompare(b.name));
2591
2685
  const displayed = entries.slice(0, limit).map((entry) => `${entry.name}${entry.isDirectory() ? "/" : ""}`);
2592
2686
  const truncated = entries.length > displayed.length;
2593
2687
  const hint = truncated ? "\n... truncated. Increase limit to continue." : "";
@@ -2638,36 +2732,40 @@ function resolveWorkerCodePath(branchPath, inputPath) {
2638
2732
  }
2639
2733
  return { absolutePath, displayPath };
2640
2734
  }
2641
- function listWorkerCodeDirectory(branchPath, inputPath) {
2735
+ async function listWorkerCodeDirectory(branchPath, inputPath) {
2642
2736
  const resolved = resolveWorkerCodePath(branchPath, inputPath);
2643
- if (!fs.statSync(resolved.absolutePath).isDirectory()) {
2737
+ if (!(await fs.promises.stat(resolved.absolutePath)).isDirectory()) {
2644
2738
  throw new Error(`Code directory not found: ${resolved.displayPath}`);
2645
2739
  }
2646
- const entries = fs.readdirSync(resolved.absolutePath, { withFileTypes: true }).filter((entry) => entry.name.toLowerCase() !== ".git").flatMap((entry) => {
2740
+ const listed = (await fs.promises.readdir(resolved.absolutePath, { withFileTypes: true })).filter(
2741
+ (entry) => entry.name.toLowerCase() !== ".git"
2742
+ );
2743
+ const entries = [];
2744
+ for (const entry of listed) {
2647
2745
  const entryPath = path.posix.join(resolved.displayPath, entry.name);
2648
2746
  try {
2649
2747
  const resolvedEntry = resolveWorkerCodePath(branchPath, entryPath);
2650
- const stats = fs.statSync(resolvedEntry.absolutePath);
2651
- if (!stats.isFile() && !stats.isDirectory()) return [];
2652
- return [
2653
- {
2654
- name: entry.name,
2655
- type: stats.isDirectory() ? "directory" : "file",
2656
- size: stats.isDirectory() ? null : stats.size,
2657
- mtime: stats.mtime.toISOString()
2658
- }
2659
- ];
2748
+ const stats = await fs.promises.stat(resolvedEntry.absolutePath);
2749
+ if (!stats.isFile() && !stats.isDirectory()) continue;
2750
+ entries.push({
2751
+ name: entry.name,
2752
+ type: stats.isDirectory() ? "directory" : "file",
2753
+ size: stats.isDirectory() ? null : stats.size,
2754
+ mtime: stats.mtime.toISOString()
2755
+ });
2660
2756
  } catch {
2661
- return [];
2662
2757
  }
2663
- }).sort((left, right) => Number(right.type === "directory") - Number(left.type === "directory") || left.name.localeCompare(right.name));
2758
+ }
2759
+ entries.sort(
2760
+ (left, right) => Number(right.type === "directory") - Number(left.type === "directory") || left.name.localeCompare(right.name)
2761
+ );
2664
2762
  return { type: "code_list", path: resolved.displayPath, entries };
2665
2763
  }
2666
- function readWorkerCodeFile(branchPath, inputPath) {
2764
+ async function readWorkerCodeFile(branchPath, inputPath) {
2667
2765
  const resolved = resolveWorkerCodePath(branchPath, inputPath);
2668
- const stats = fs.statSync(resolved.absolutePath);
2766
+ const stats = await fs.promises.stat(resolved.absolutePath);
2669
2767
  if (!stats.isFile()) throw new Error(`Code file not found: ${resolved.displayPath}`);
2670
- const bytes = fs.readFileSync(resolved.absolutePath);
2768
+ const bytes = await fs.promises.readFile(resolved.absolutePath);
2671
2769
  return {
2672
2770
  type: "code_read",
2673
2771
  path: resolved.displayPath,
@@ -2676,17 +2774,17 @@ function readWorkerCodeFile(branchPath, inputPath) {
2676
2774
  base64: bytes.toString("base64")
2677
2775
  };
2678
2776
  }
2679
- function executeCodeListOperation(input) {
2777
+ async function executeCodeListOperation(input) {
2680
2778
  if (input.resolvedTarget.target.type !== "project") throw new Error("Code listing requires a project branch target");
2681
2779
  return listWorkerCodeDirectory(input.resolvedTarget.rootPath, input.message.path);
2682
2780
  }
2683
- function executeCodeReadOperation(input) {
2781
+ async function executeCodeReadOperation(input) {
2684
2782
  if (input.resolvedTarget.target.type !== "project") throw new Error("Code reading requires a project branch target");
2685
2783
  return readWorkerCodeFile(input.resolvedTarget.rootPath, input.message.path);
2686
2784
  }
2687
- function readWorkerImageFile(branchPath, filePath, builtInPaths) {
2785
+ async function readWorkerImageFile(branchPath, filePath, builtInPaths) {
2688
2786
  const resolved = resolveWorkerFilePath(branchPath, filePath, builtInPaths);
2689
- if (!fs.existsSync(resolved.absolutePath) || !fs.statSync(resolved.absolutePath).isFile()) {
2787
+ if (!(await statIfExists(resolved.absolutePath))?.isFile()) {
2690
2788
  throw new Error(`File not found: ${resolved.displayPath}`);
2691
2789
  }
2692
2790
  const extension = path.extname(resolved.absolutePath).toLowerCase();
@@ -2694,7 +2792,7 @@ function readWorkerImageFile(branchPath, filePath, builtInPaths) {
2694
2792
  if (!mediaType) {
2695
2793
  throw new Error("read supports PNG and JPEG image inputs only");
2696
2794
  }
2697
- const bytes = fs.readFileSync(resolved.absolutePath);
2795
+ const bytes = await fs.promises.readFile(resolved.absolutePath);
2698
2796
  const dimensions = readImageDimensions(bytes, mediaType);
2699
2797
  return {
2700
2798
  type: "view_file_bytes",
@@ -2794,6 +2892,10 @@ function cancelWorkerCommandLaunch(resources) {
2794
2892
  });
2795
2893
  }
2796
2894
  async function acquireWorkerCommandLaunch(ws, message, assertAdmission) {
2895
+ if (message.type === "exec_start" && message.commandClass === "control") {
2896
+ assertControlCommandArgv(message.argv);
2897
+ if (message.interactive) throw new Error("Control commands cannot be interactive");
2898
+ }
2797
2899
  const id = message.type === "pty_open" ? `pty:${message.ptyId}` : message.runId;
2798
2900
  const operationId = message.type === "pty_open" ? id : `exec:${id}`;
2799
2901
  sendWorkerMessage(ws, { type: "heartbeat_lease", operationId, active: true });
@@ -2802,6 +2904,7 @@ async function acquireWorkerCommandLaunch(ws, message, assertAdmission) {
2802
2904
  id,
2803
2905
  kind: message.type,
2804
2906
  ..."workspaceEffect" in message && message.workspaceEffect === "none" ? { workspaceEffect: "none" } : {},
2907
+ ...message.type === "exec_start" && message.commandClass === "control" ? { commandClass: "control" } : {},
2805
2908
  ..."sessionId" in message && message.sessionId ? { sessionId: message.sessionId } : {},
2806
2909
  assertAdmission: () => {
2807
2910
  assertAdmission();
@@ -2949,8 +3052,17 @@ async function executeStreamingCommand(input) {
2949
3052
  });
2950
3053
  const cwd = resolveCommandCwd(input.resolvedTarget.rootPath, input.message.cwd);
2951
3054
  const interactive = input.message.interactive === true;
3055
+ const control = input.message.commandClass === "control";
3056
+ let argv = input.message.argv;
3057
+ let environment = workerChildProcessEnvironment([githubProcessEnv(), input.message.env ?? {}, targetProcessEnv2]);
3058
+ if (control) {
3059
+ assertControlCommandArgv(argv);
3060
+ if (interactive) throw new Error("Control commands cannot be interactive");
3061
+ argv = [resolveControlCommandExecutable(trustedControlPath, (program, options) => Bun.which(program, options)), ...argv.slice(1)];
3062
+ environment = controlCommandEnvironment(environment, trustedControlPath);
3063
+ }
2952
3064
  input.assertAdmission();
2953
- const subprocess = Bun.spawn(input.resources.wrap(input.message.argv), {
3065
+ const subprocess = Bun.spawn(input.resources.wrap(argv), {
2954
3066
  cwd,
2955
3067
  // Without an explicit stdin the process reads /dev/null and interactive
2956
3068
  // prompts see immediate EOF; "pipe" keeps stdin open for exec_stdin.
@@ -2958,7 +3070,7 @@ async function executeStreamingCommand(input) {
2958
3070
  stdout: "pipe",
2959
3071
  stderr: "pipe",
2960
3072
  detached: true,
2961
- env: workerChildProcessEnvironment([githubProcessEnv(), input.message.env ?? {}, targetProcessEnv2])
3073
+ env: environment
2962
3074
  });
2963
3075
  spawnedProcess = subprocess;
2964
3076
  credentialBearingProcessGroups.set(subprocess.pid, subprocess);
@@ -2973,12 +3085,13 @@ async function executeStreamingCommand(input) {
2973
3085
  pid: subprocess.pid,
2974
3086
  processGroupId: process.platform === "win32" ? void 0 : subprocess.pid,
2975
3087
  credentialId: input.message.credentialId,
2976
- argv: input.message.argv,
3088
+ argv,
2977
3089
  command: input.message.command,
2978
3090
  cwd,
2979
3091
  startedAt: (/* @__PURE__ */ new Date()).toISOString(),
2980
3092
  ...interactive ? { interactive: true, stdin: subprocess.stdin } : {},
2981
- ...input.message.workspaceEffect === "none" ? { workspaceEffect: "none" } : {}
3093
+ ...input.message.workspaceEffect === "none" ? { workspaceEffect: "none" } : {},
3094
+ ...control ? { commandClass: "control" } : {}
2982
3095
  });
2983
3096
  settlePreparation();
2984
3097
  started = true;
@@ -2990,7 +3103,8 @@ async function executeStreamingCommand(input) {
2990
3103
  pid: subprocess.pid,
2991
3104
  ...process.platform === "win32" ? {} : { processGroupId: subprocess.pid }
2992
3105
  });
2993
- if (input.message.timeoutMs) {
3106
+ const timeoutMs = control ? Math.min(input.message.timeoutMs ?? CONTROL_COMMAND_RUNTIME_MS, CONTROL_COMMAND_RUNTIME_MS) : input.message.timeoutMs;
3107
+ if (timeoutMs) {
2994
3108
  timeout = setTimeout(() => {
2995
3109
  timedOut = true;
2996
3110
  cancelWorkerCommandLaunch(input.resources);
@@ -3001,30 +3115,32 @@ async function executeStreamingCommand(input) {
3001
3115
  `
3002
3116
  );
3003
3117
  });
3004
- }, input.message.timeoutMs);
3118
+ }, timeoutMs);
3005
3119
  }
3006
3120
  const [exitCode] = await Promise.all([
3007
3121
  subprocess.exited,
3008
- streamCommandOutput(subprocess.stdout, (data) => {
3009
- sendWorkerMessage(input.ws, {
3122
+ streamCommandOutput(
3123
+ subprocess.stdout,
3124
+ (data) => sendWorkerOutput(input.ws, {
3010
3125
  type: "exec_output",
3011
3126
  runId: input.message.runId,
3012
3127
  stream: "stdout",
3013
3128
  data
3014
- });
3015
- }),
3016
- streamCommandOutput(subprocess.stderr, (data) => {
3017
- sendWorkerMessage(input.ws, {
3129
+ })
3130
+ ),
3131
+ streamCommandOutput(
3132
+ subprocess.stderr,
3133
+ (data) => sendWorkerOutput(input.ws, {
3018
3134
  type: "exec_output",
3019
3135
  runId: input.message.runId,
3020
3136
  stream: "stderr",
3021
3137
  data
3022
- });
3023
- })
3138
+ })
3139
+ )
3024
3140
  ]);
3025
3141
  const diagnostic = await input.resources.diagnose();
3026
3142
  if (diagnostic) {
3027
- sendWorkerMessage(input.ws, {
3143
+ await sendWorkerOutput(input.ws, {
3028
3144
  type: "exec_output",
3029
3145
  runId: input.message.runId,
3030
3146
  stream: "stderr",
@@ -3096,6 +3212,33 @@ async function reapCompletedCredentialBearingProcessGroup(runId, subprocess) {
3096
3212
  );
3097
3213
  }
3098
3214
  }
3215
+ async function cancelProcessRun(runId) {
3216
+ let resourceCancellationError;
3217
+ try {
3218
+ await workerCommandLauncher?.cancel(runId);
3219
+ } catch (error) {
3220
+ resourceCancellationError = error instanceof Error ? error.message : String(error);
3221
+ }
3222
+ const active = activeProcesses.get(runId);
3223
+ let cancelled = resourceCancellationError === void 0;
3224
+ let cancelMessage;
3225
+ if (active) {
3226
+ try {
3227
+ closeProcessStdin(active);
3228
+ await terminateProcessTree(active.process);
3229
+ cancelMessage = `Stopped ${active.command}`;
3230
+ } catch (error) {
3231
+ cancelled = false;
3232
+ cancelMessage = `Failed to stop ${active.command}: ${error instanceof Error ? error.message : String(error)}`;
3233
+ }
3234
+ } else {
3235
+ cancelledProcessRuns.add(runId);
3236
+ abortPendingReservation(`run:${runId}`, "Cancellation queued before command start");
3237
+ cancelMessage = "Cancellation queued before command start";
3238
+ }
3239
+ if (resourceCancellationError) cancelMessage += `; job resource termination failed: ${resourceCancellationError}`;
3240
+ return { cancelled, message: cancelMessage };
3241
+ }
3099
3242
  function closeProcessStdin(active) {
3100
3243
  if (!active?.stdin) {
3101
3244
  return;
@@ -3121,21 +3264,35 @@ function sendReplayWorkerMessage(ws, message) {
3121
3264
  }
3122
3265
  function sendWorkerMessage(ws, message) {
3123
3266
  if (launcherFailureInProgress) return;
3124
- if (recoveryStore && "requestId" in message && typeof message.requestId === "string" && recoveryStore.isUnknown(message.requestId))
3125
- return;
3126
3267
  const transportError = "error" in message && typeof message.error === "string" ? message.error : message.type === "workspace_sync_result" && message.result.outcome === "failed" ? message.result.error : void 0;
3127
- if (recoveryStore && transportError && isWorkerCommunicationFailure(transportError) && "requestId" in message && typeof message.requestId === "string") {
3128
- recoveryStore.unknown(message.requestId);
3268
+ if (recoveryJournal && transportError && isWorkerCommunicationFailure(transportError) && "requestId" in message && typeof message.requestId === "string") {
3269
+ recoveryJournal.unknown(message.requestId);
3129
3270
  currentWorkerSocket?.close(1012, "Communication recovery");
3130
3271
  return;
3131
3272
  }
3132
- const recorded = recoveryStore?.record(message) ?? message;
3133
- if ((message.type === "exec_exit" || message.type === "exec_error") && infrastructureStoppedRunIds.has(message.runId)) {
3134
- recoveryStore?.unknownRun(message.runId);
3273
+ if (!recoveryJournal) {
3274
+ sendReplayWorkerMessage(ws, message);
3135
3275
  return;
3136
3276
  }
3137
- if (message.type === "exec_output" && workerRecoveryId !== void 0) return;
3138
- sendReplayWorkerMessage(ws, recorded);
3277
+ recoveryJournal.send(message);
3278
+ }
3279
+ function sendWorkerOutput(ws, frame) {
3280
+ if (launcherFailureInProgress) return Promise.resolve();
3281
+ if (!recoveryJournal) {
3282
+ sendReplayWorkerMessage(ws, frame);
3283
+ return Promise.resolve();
3284
+ }
3285
+ return recoveryJournal.sendOutput(frame);
3286
+ }
3287
+ function deliverJournaledWorkerMessage(message) {
3288
+ const target = currentWorkerSocket;
3289
+ if ((message.type === "exec_exit" || message.type === "exec_error") && typeof message.runId === "string" && infrastructureStoppedRunIds.has(message.runId)) {
3290
+ recoveryJournal?.unknownRun(message.runId);
3291
+ return true;
3292
+ }
3293
+ if (!target || target.readyState !== WebSocket.OPEN) return false;
3294
+ if (message.type === "exec_output" && workerRecoveryId !== void 0) return false;
3295
+ return sendReplayWorkerMessage(target, message);
3139
3296
  }
3140
3297
  function sendSerializedWorkerMessage(ws, serialized) {
3141
3298
  sendWorkerMessage(ws, JSON.parse(serialized));
@@ -3164,7 +3321,8 @@ function buildActiveProcessReports() {
3164
3321
  ...active.cwd ? { cwd: active.cwd } : {},
3165
3322
  startedAt: active.startedAt,
3166
3323
  ...active.interactive ? { interactive: true } : {},
3167
- ...active.workspaceEffect === "none" ? { workspaceEffect: "none" } : {}
3324
+ ...active.workspaceEffect === "none" ? { workspaceEffect: "none" } : {},
3325
+ ...active.commandClass ? { commandClass: active.commandClass } : {}
3168
3326
  }));
3169
3327
  }
3170
3328
  function sendActiveProcessReport(ws) {
@@ -3686,6 +3844,7 @@ async function startWorker(options, projectRuntime = {
3686
3844
  validateLabel(label);
3687
3845
  if (!workerCommandLauncher) {
3688
3846
  const launcher = await createWorkerCommandLauncher({
3847
+ onCapacityChange: () => scheduleCapacityReport(),
3689
3848
  configuration: process.env.R5D_WORKER_COMMAND_LAUNCHER,
3690
3849
  env: commandLauncherEnvironment(),
3691
3850
  onFailure: handleWorkerCommandLauncherFailure
@@ -3696,9 +3855,19 @@ async function startWorker(options, projectRuntime = {
3696
3855
  `);
3697
3856
  }
3698
3857
  const rootDir = path.resolve(options.rootDir ?? process.env.R5D_ROOT ?? defaultRootDir());
3699
- recoveryStore ??= new WorkerRecoveryStore(
3700
- path.join(rootDir, ".worker-recovery", createHash("sha256").update(`${baseUrl}\0${label}`).digest("hex"), "protocol.sqlite")
3701
- );
3858
+ if (!recoveryJournal) {
3859
+ const journal = await WorkerRecoveryJournal.open({
3860
+ filename: path.join(
3861
+ rootDir,
3862
+ ".worker-recovery",
3863
+ createHash("sha256").update(`${baseUrl}\0${label}`).digest("hex"),
3864
+ "protocol.sqlite"
3865
+ )
3866
+ });
3867
+ journal.deliver = deliverJournaledWorkerMessage;
3868
+ journal.onFailure = handleRecoveryJournalFailure;
3869
+ recoveryJournal = journal;
3870
+ }
3702
3871
  const projectsRoot = path.join(rootDir, "projects");
3703
3872
  const syncRoot = path.join(rootDir, "sync");
3704
3873
  const workspaceShadowRoot = path.join(syncRoot, "workspace");
@@ -3801,7 +3970,6 @@ async function startWorker(options, projectRuntime = {
3801
3970
  const bearerAuthHeader = `Authorization: Bearer ${token}`;
3802
3971
  const workerCredentialUsername = credentialUsernameForAuthHeader(bearerAuthHeader);
3803
3972
  const projectBranchKey = (projectId, branchName) => `${projectId}\0${branchName}`;
3804
- const projectMountId = (projectId, branchName) => `project:${projectId}:${encodeURIComponent(branchName)}`;
3805
3973
  const projectPlanMountId = (projectId, branchName) => `plan:${projectId}:${encodeURIComponent(branchName)}`;
3806
3974
  const workspaceProjectRelativePath = (projectId, branchName) => path.posix.join("projects", projectId, "branches", encodeURIComponent(branchName));
3807
3975
  const workspacePlanRelativePath = (projectId, branchName) => path.posix.join("plans", projectId, encodeURIComponent(branchName));
@@ -3884,7 +4052,8 @@ async function startWorker(options, projectRuntime = {
3884
4052
  sourcePath: configuredProjectBranchPath(projectsRoot, project, branchName),
3885
4053
  planSourcePath: path.join(planRoot, project.projectId, ...branchName.split("/"))
3886
4054
  }));
3887
- removeProjectWorktrees({ projectRoot, branchNames: branches.map(({ branchName }) => branchName) });
4055
+ const removed = removeProjectWorktrees({ projectsRoot, projectRoot, branchNames: branches.map(({ branchName }) => branchName) });
4056
+ if (removed.stagedForCollectionPath) checkoutGarbageCollector.enqueue(removed.stagedForCollectionPath);
3888
4057
  for (const branch of branches) {
3889
4058
  fs.rmSync(branch.planSourcePath, { recursive: true, force: true });
3890
4059
  const key = projectBranchKey(project.projectId, branch.branchName);
@@ -3972,14 +4141,14 @@ async function startWorker(options, projectRuntime = {
3972
4141
  )
3973
4142
  };
3974
4143
  };
3975
- const stageDurableWorkspaceDeletions = () => {
4144
+ const stageDurableWorkspaceDeletions = async () => {
3976
4145
  for (const deletion of projectWorkspaceState.pendingTreeDeletions) {
3977
4146
  const cleanupConfig = cleanupConfigForDurableDeletion(deletion);
3978
4147
  if (deletion.kind === "project") {
3979
4148
  if (!deletion.projectDeleted) {
3980
4149
  const nextProject = projectConfigById.get(deletion.projectId);
3981
4150
  if (!nextProject) throw new Error(`Checkout-path move lost desired project ${deletion.projectId}`);
3982
- preserveProjectCheckoutPathMove({
4151
+ await workspaceFilesystemExecutor().run("project_checkout_path_move", {
3983
4152
  oldProjectRoot: deletion.managedPath,
3984
4153
  newProjectRoot: configuredProjectRoot(projectsRoot, nextProject),
3985
4154
  projectsDurabilityRoot: projectsRoot,
@@ -4346,7 +4515,7 @@ async function startWorker(options, projectRuntime = {
4346
4515
  }
4347
4516
  }
4348
4517
  if (transitionReseedRequired) {
4349
- projectWorkspaceState = projectWorkspaceStateStore.recordEnabledRepositoryTransition({
4518
+ projectWorkspaceState = await projectWorkspaceStateStore.recordEnabledRepositoryTransition({
4350
4519
  projectId: project.projectId,
4351
4520
  transitionId: project.repositoryTransitionId
4352
4521
  });
@@ -4515,7 +4684,7 @@ async function startWorker(options, projectRuntime = {
4515
4684
  )
4516
4685
  ];
4517
4686
  if (publishedTombstoneIds.length > 0) {
4518
- projectWorkspaceState = projectWorkspaceStateStore.recordTreePublication({
4687
+ projectWorkspaceState = await projectWorkspaceStateStore.recordTreePublication({
4519
4688
  tombstoneIds: publishedTombstoneIds,
4520
4689
  publishedHead
4521
4690
  });
@@ -4534,7 +4703,7 @@ async function startWorker(options, projectRuntime = {
4534
4703
  forgetLocalProjectMirrorRefsTokens(deletion.projectId, deletion.branchName);
4535
4704
  }
4536
4705
  if (deletion.tombstoneId && !deletion.projectDeleted) {
4537
- projectWorkspaceState = projectWorkspaceStateStore.recordMirrorRefDeletion({
4706
+ projectWorkspaceState = await projectWorkspaceStateStore.recordMirrorRefDeletion({
4538
4707
  tombstoneId: deletion.tombstoneId,
4539
4708
  branchName: deletion.branchName
4540
4709
  });
@@ -4731,7 +4900,10 @@ async function startWorker(options, projectRuntime = {
4731
4900
  };
4732
4901
  };
4733
4902
  const workspaceHydrationTransactionHooks = createWorkspaceHydrationLeaseHooks(
4734
- (message) => sendWorkerMessageFromCurrentSource(ws, message)
4903
+ (message) => sendWorkerMessageFromCurrentSource(ws, message),
4904
+ {
4905
+ fence: workspaceMountHoldFence
4906
+ }
4735
4907
  );
4736
4908
  const performWorkspaceSync = async (input) => {
4737
4909
  if (!workspaceRemoteUrl || !workspaceCredentialHelper || !workspaceCredentialUsername || !workspaceGitIdentity) {
@@ -5389,7 +5561,7 @@ async function startWorker(options, projectRuntime = {
5389
5561
  (project) => project.preserveOnlyBranches.map(({ branchId, branchName }) => ({ branchId, projectId: project.projectId, branchName }))
5390
5562
  );
5391
5563
  if (!incidentDeferral.incidentId) {
5392
- projectWorkspaceState = projectWorkspaceStateStore.reconcile({
5564
+ projectWorkspaceState = await projectWorkspaceStateStore.reconcile({
5393
5565
  desiredProjects: message.projects,
5394
5566
  preserveOnlyBranches
5395
5567
  });
@@ -5400,7 +5572,7 @@ async function startWorker(options, projectRuntime = {
5400
5572
  (candidate) => candidate.projectId === project.projectId && candidate.branchName === branch.branchName
5401
5573
  );
5402
5574
  if (pending2 && pending2.branchId === branch.branchId) {
5403
- projectWorkspaceState = projectWorkspaceStateStore.clearPendingCreatedBranch(pending2);
5575
+ projectWorkspaceState = await projectWorkspaceStateStore.clearPendingCreatedBranch(pending2);
5404
5576
  }
5405
5577
  }
5406
5578
  }
@@ -5568,7 +5740,7 @@ async function startWorker(options, projectRuntime = {
5568
5740
  for (const projectId of [...readyProjectIds]) {
5569
5741
  if (!projectConfigById.has(projectId)) readyProjectIds.delete(projectId);
5570
5742
  }
5571
- stageDurableWorkspaceDeletions();
5743
+ await stageDurableWorkspaceDeletions();
5572
5744
  const mountsBeforeGit = buildWorkspaceMounts();
5573
5745
  await hydrateWorkspaceGitMounts(
5574
5746
  workspaceShadowRoot,
@@ -5614,7 +5786,12 @@ async function startWorker(options, projectRuntime = {
5614
5786
  const connectSyncMs = Date.now() - connectSyncStartedAtMs;
5615
5787
  const result = syncResult.telemetry ? syncResult : {
5616
5788
  ...syncResult,
5617
- telemetry: { totalMs: connectSyncMs, queueMs: 0, prepareMs: lastWorkspaceSyncMirrorObservationMs, synchronizeMs: connectSyncMs }
5789
+ telemetry: {
5790
+ totalMs: connectSyncMs,
5791
+ queueMs: 0,
5792
+ prepareMs: lastWorkspaceSyncMirrorObservationMs,
5793
+ synchronizeMs: connectSyncMs
5794
+ }
5618
5795
  };
5619
5796
  assertConfigurationSyncStillAdmitted();
5620
5797
  if (receiptGeneration !== workspaceConfigurationReceiptGeneration) {
@@ -5633,7 +5810,7 @@ async function startWorker(options, projectRuntime = {
5633
5810
  ) ? [deletion.tombstoneId] : []
5634
5811
  );
5635
5812
  if (layoutCleanupIds.length > 0) {
5636
- projectWorkspaceState = projectWorkspaceStateStore.recordTreePublication({
5813
+ projectWorkspaceState = await projectWorkspaceStateStore.recordTreePublication({
5637
5814
  tombstoneIds: layoutCleanupIds,
5638
5815
  publishedHead
5639
5816
  });
@@ -5740,26 +5917,33 @@ async function startWorker(options, projectRuntime = {
5740
5917
  if (startupProjectSnapshotRecoveryCompleted) return;
5741
5918
  sendLifecycleProgress({ phase: "recovering", operationId, detail: "Checking interrupted workspace transactions" });
5742
5919
  const snapshotRecovery = await workspaceSyncSingleFlight.runExclusive(
5743
- async () => await recoverProjectSnapshotsInChild({
5744
- projectsRoot,
5745
- snapshotRoot: projectWorktreeSnapshotsRoot,
5746
- onProgress: (progress) => {
5747
- const lifecycleProgress = {
5748
- phase: "recovering",
5749
- operationId,
5750
- detail: describeSnapshotProgress(progress),
5751
- completedBytes: progress.completedBytes,
5752
- completedItems: progress.completedBranches,
5753
- totalItems: progress.totalBranches,
5754
- ...progress.totalBytes === void 0 ? {} : { totalBytes: progress.totalBytes }
5755
- };
5756
- sendLifecycleProgress(lifecycleProgress);
5757
- process.stdout.write(
5758
- `[r5d-worker] ${lifecycleProgress.detail}: ${progress.completedBranches}/${progress.totalBranches} branch(es), ${progress.completedBytes}/${progress.totalBytes ?? "?"} bytes
5920
+ async () => await workspaceFilesystemExecutor().run(
5921
+ "project_snapshots_recover",
5922
+ {
5923
+ projectsRoot,
5924
+ temporaryRoot: projectWorktreeSnapshotsRoot,
5925
+ currentProcessId: process.pid,
5926
+ currentOwnerSessionId: PROJECT_WORKTREE_SNAPSHOT_OWNER_SESSION_ID
5927
+ },
5928
+ {
5929
+ onProgress: (progress) => {
5930
+ const lifecycleProgress = {
5931
+ phase: "recovering",
5932
+ operationId,
5933
+ detail: describeSnapshotProgress(progress),
5934
+ completedBytes: progress.completedBytes,
5935
+ completedItems: progress.completedBranches,
5936
+ totalItems: progress.totalBranches,
5937
+ ...progress.totalBytes === void 0 ? {} : { totalBytes: progress.totalBytes }
5938
+ };
5939
+ sendLifecycleProgress(lifecycleProgress);
5940
+ process.stdout.write(
5941
+ `[r5d-worker] ${lifecycleProgress.detail}: ${progress.completedBranches}/${progress.totalBranches} branch(es), ${progress.completedBytes}/${progress.totalBytes ?? "?"} bytes
5759
5942
  `
5760
- );
5943
+ );
5944
+ }
5761
5945
  }
5762
- })
5946
+ )
5763
5947
  );
5764
5948
  if (snapshotRecovery.restored.length > 0) {
5765
5949
  process.stdout.write(`[r5d-worker] restored ${snapshotRecovery.restored.length} interrupted project snapshot(s)
@@ -5854,7 +6038,7 @@ async function startWorker(options, projectRuntime = {
5854
6038
  type: "hello",
5855
6039
  resumableProtocol: WORKER_RESUMABLE_PROTOCOL,
5856
6040
  runtimeId: workerRuntimeId,
5857
- ledgerId: recoveryStore.ledgerId,
6041
+ ledgerId: recoveryJournal.ledgerId,
5858
6042
  startupReady: startupRecoveryComplete,
5859
6043
  hostInfo: {
5860
6044
  hostname: hostname(),
@@ -5873,6 +6057,7 @@ async function startWorker(options, projectRuntime = {
5873
6057
  workspaceConfigResetToCanonicalV1: true,
5874
6058
  projectBranchDeletionFastAckV1: true,
5875
6059
  projectBranchDeletionStagedRemovalV1: true,
6060
+ commandControlLaneV1: workerCommandLauncher?.supportsControlLane === true,
5876
6061
  projectMirrorLeaseV1: true,
5877
6062
  projectMirrorRefsTokensV1: true,
5878
6063
  projectBranchWorkingTreeModeV1: true
@@ -5881,16 +6066,39 @@ async function startWorker(options, projectRuntime = {
5881
6066
  }
5882
6067
  };
5883
6068
  sendSerializedWorkerMessage(ws, JSON.stringify(hello));
6069
+ let outputPumpInFlight = false;
5884
6070
  outputReplayTimer = setInterval(() => {
5885
- if (workerRecoveryId !== void 0 || currentWorkerSocket !== ws) return;
5886
- for (const output of recoveryStore.output()) {
5887
- if (!sendReplayWorkerMessage(ws, output)) break;
5888
- }
6071
+ if (workerRecoveryId !== void 0 || currentWorkerSocket !== ws || outputPumpInFlight || !recoveryJournal) return;
6072
+ outputPumpInFlight = true;
6073
+ void (async () => {
6074
+ try {
6075
+ for await (const page of recoveryJournal.outputReplay(void 0)) {
6076
+ for (const output of page) {
6077
+ if (workerRecoveryId !== void 0 || currentWorkerSocket !== ws || !sendReplayWorkerMessage(ws, output)) return;
6078
+ }
6079
+ }
6080
+ } catch (error) {
6081
+ process.stderr.write(`[r5d-worker] output replay pump failed: ${error instanceof Error ? error.message : String(error)}
6082
+ `);
6083
+ } finally {
6084
+ outputPumpInFlight = false;
6085
+ }
6086
+ })();
5889
6087
  }, 250);
5890
6088
  outputReplayTimer.unref();
6089
+ let terminalPumpInFlight = false;
5891
6090
  terminalReplayTimer = setInterval(() => {
5892
- if (workerRecoveryId !== void 0) return;
5893
- for (const terminal of recoveryStore?.terminals() ?? []) sendReplayWorkerMessage(ws, terminal);
6091
+ if (workerRecoveryId !== void 0 || terminalPumpInFlight || !recoveryJournal) return;
6092
+ terminalPumpInFlight = true;
6093
+ void recoveryJournal.terminals().then((terminals) => {
6094
+ if (workerRecoveryId !== void 0 || currentWorkerSocket !== ws) return;
6095
+ for (const terminal of terminals) sendReplayWorkerMessage(ws, terminal);
6096
+ }).catch((error) => {
6097
+ process.stderr.write(`[r5d-worker] terminal replay pump failed: ${error instanceof Error ? error.message : String(error)}
6098
+ `);
6099
+ }).finally(() => {
6100
+ terminalPumpInFlight = false;
6101
+ });
5894
6102
  }, 5e3);
5895
6103
  terminalReplayTimer.unref();
5896
6104
  process.stdout.write("[r5d-worker] connected\n");
@@ -5903,28 +6111,37 @@ async function startWorker(options, projectRuntime = {
5903
6111
  if (message.type === "recovery_request") {
5904
6112
  workerRecoveryId = message.recoveryId;
5905
6113
  sendActiveProcessReport(ws);
5906
- sendWorkerMessage(ws, {
5907
- type: "recovery_state",
5908
- recoveryId: message.recoveryId,
5909
- runtimeId: workerRuntimeId,
5910
- ledgerId: recoveryStore.ledgerId,
5911
- operations: recoveryStore.operations()
5912
- });
5913
- const cursors = new Map(recoveryStore.outputCursors().map((cursor) => [cursor.runId, cursor]));
5914
- for (const cursor of message.outputOffsets) cursors.set(cursor.runId, cursor);
5915
- workerOutputWindow.reset([...cursors.values()]);
5916
- for (const response of recoveryStore.responses(message.requests)) sendReplayWorkerMessage(ws, response);
5917
- for (const output of recoveryStore.output(message.outputOffsets)) {
5918
- while (true) {
5919
- if (currentWorkerSocket !== ws || workerRecoveryId !== message.recoveryId || ws.readyState !== WebSocket.OPEN || workerLeaseExpired)
5920
- return;
5921
- if (sendReplayWorkerMessage(ws, output)) break;
5922
- await Bun.sleep(10);
6114
+ recoveryJournal.pauseDelivery();
6115
+ try {
6116
+ const state = await recoveryJournal.recoveryState(message.requests);
6117
+ if (currentWorkerSocket !== ws || workerRecoveryId !== message.recoveryId) return;
6118
+ sendWorkerMessage(ws, {
6119
+ type: "recovery_state",
6120
+ recoveryId: message.recoveryId,
6121
+ runtimeId: workerRuntimeId,
6122
+ ledgerId: recoveryJournal.ledgerId,
6123
+ operations: state.operations
6124
+ });
6125
+ const cursors = new Map(state.cursors.map((cursor) => [cursor.runId, cursor]));
6126
+ for (const cursor of message.outputOffsets) cursors.set(cursor.runId, cursor);
6127
+ workerOutputWindow.reset([...cursors.values()]);
6128
+ for (const response of state.responses) sendReplayWorkerMessage(ws, response);
6129
+ for await (const page of recoveryJournal.outputReplay(message.outputOffsets)) {
6130
+ for (const output of page) {
6131
+ while (true) {
6132
+ if (currentWorkerSocket !== ws || workerRecoveryId !== message.recoveryId || ws.readyState !== WebSocket.OPEN || workerLeaseExpired)
6133
+ return;
6134
+ if (sendReplayWorkerMessage(ws, output)) break;
6135
+ await Bun.sleep(10);
6136
+ }
6137
+ }
5923
6138
  }
6139
+ if (currentWorkerSocket !== ws || workerRecoveryId !== message.recoveryId) return;
6140
+ for (const terminal of state.terminals) sendReplayWorkerMessage(ws, terminal);
6141
+ sendWorkerMessage(ws, { type: "recovery_complete", recoveryId: message.recoveryId });
6142
+ } finally {
6143
+ recoveryJournal.resumeDelivery();
5924
6144
  }
5925
- if (currentWorkerSocket !== ws || workerRecoveryId !== message.recoveryId) return;
5926
- for (const terminal of recoveryStore.terminals()) sendReplayWorkerMessage(ws, terminal);
5927
- sendWorkerMessage(ws, { type: "recovery_complete", recoveryId: message.recoveryId });
5928
6145
  return;
5929
6146
  }
5930
6147
  if (message.type === "recovery_ready") {
@@ -5939,17 +6156,23 @@ async function startWorker(options, projectRuntime = {
5939
6156
  return;
5940
6157
  }
5941
6158
  if (message.type === "operation_result_ack") {
5942
- recoveryStore.acknowledgeResult(message.requestId);
6159
+ recoveryJournal.acknowledgeResult(message.requestId);
5943
6160
  return;
5944
6161
  }
5945
6162
  if (message.type === "exec_output_ack") {
5946
- recoveryStore.acknowledgeOutput(message.runId, message.stream, message.nextOffset);
6163
+ recoveryJournal.acknowledgeOutput(message.runId, message.stream, message.nextOffset);
5947
6164
  workerOutputWindow.acknowledge(message.runId, message.stream, message.nextOffset);
5948
6165
  return;
5949
6166
  }
5950
6167
  if (message.type === "cancel_session") {
5951
- recoveryStore.cancelSession(message.sessionId);
6168
+ const durableCancellation = recoveryJournal.cancelSession(message.sessionId);
6169
+ abortPendingReservations("Session cancelled before command start", message.sessionId);
5952
6170
  await Promise.all([
6171
+ durableCancellation,
6172
+ // File writes already in flight for the session settle before the
6173
+ // acknowledgement; queued ones re-check the cancellation before
6174
+ // their first write and never start.
6175
+ sessionFileMutations.settled(message.sessionId),
5953
6176
  workerCommandLauncher?.cancelSession(message.sessionId),
5954
6177
  ...[...activeProcesses].filter(([, active]) => active.sessionId === message.sessionId).map(async ([runId, active]) => {
5955
6178
  cancelledProcessRuns.add(runId);
@@ -5964,27 +6187,74 @@ async function startWorker(options, projectRuntime = {
5964
6187
  if (!startupRecoveryComplete && message.type !== "ping") return;
5965
6188
  if (currentWorkerSocket !== ws) return;
5966
6189
  const operationRequestId = "requestId" in message && typeof message.requestId === "string" && message.type !== "workspace_config" ? message.requestId : void 0;
6190
+ if (message.type === "cancel") {
6191
+ const admission = recoveryJournal.admit({ ...message, requestId: message.requestId });
6192
+ const cancelled = await cancelProcessRun(message.runId);
6193
+ let admitted;
6194
+ try {
6195
+ admitted = await admission;
6196
+ } catch (error) {
6197
+ if (!(error instanceof WorkerRecoveryJournalBusyError)) throw error;
6198
+ process.stderr.write(`[r5d-worker] cancel ${message.runId} stopped the run but was not durably admitted: ${error.message}
6199
+ `);
6200
+ return;
6201
+ }
6202
+ if (admitted.admission === "conflict") {
6203
+ ws.close(1008, "Operation identity conflict");
6204
+ return;
6205
+ }
6206
+ sendWorkerMessage(ws, { type: "operation_received", requestId: message.requestId });
6207
+ if (admitted.admission !== "new") {
6208
+ if (admitted.response && admitted.admission !== "unknown") sendReplayWorkerMessage(ws, admitted.response);
6209
+ return;
6210
+ }
6211
+ sendSerializedWorkerMessage(
6212
+ ws,
6213
+ JSON.stringify({
6214
+ type: "cancel_result",
6215
+ requestId: message.requestId,
6216
+ runId: message.runId,
6217
+ cancelled: cancelled.cancelled,
6218
+ message: cancelled.message
6219
+ })
6220
+ );
6221
+ return;
6222
+ }
5967
6223
  if (operationRequestId) {
5968
- const admission = recoveryStore.admit({ ...message, requestId: operationRequestId });
5969
- if (admission === "conflict") {
6224
+ let admitted;
6225
+ try {
6226
+ admitted = await recoveryJournal.admit({ ...message, requestId: operationRequestId });
6227
+ } catch (error) {
6228
+ if (!(error instanceof WorkerRecoveryJournalBusyError)) throw error;
6229
+ process.stderr.write(`[r5d-worker] ${message.type} ${operationRequestId} was not admitted: ${error.message}
6230
+ `);
6231
+ return;
6232
+ }
6233
+ if (admitted.admission === "conflict") {
5970
6234
  ws.close(1008, "Operation identity conflict");
5971
6235
  return;
5972
6236
  }
5973
6237
  sendWorkerMessage(ws, { type: "operation_received", requestId: operationRequestId });
5974
- if (admission !== "new") {
5975
- const response = recoveryStore.response(operationRequestId);
5976
- if (response && admission !== "unknown") sendReplayWorkerMessage(ws, response);
5977
- if (!(admission === "unknown" && message.type === "delete_project_branch" && recoveryStore.readmitUnknownBranchDeletion(operationRequestId))) {
6238
+ if (admitted.admission !== "new") {
6239
+ if (admitted.response && admitted.admission !== "unknown") sendReplayWorkerMessage(ws, admitted.response);
6240
+ if (!(admitted.admission === "unknown" && message.type === "delete_project_branch" && await recoveryJournal.readmitUnknownBranchDeletion(
6241
+ operationRequestId,
6242
+ "sessionId" in message && typeof message.sessionId === "string" ? message.sessionId : void 0
6243
+ ))) {
5978
6244
  return;
5979
6245
  }
5980
6246
  }
6247
+ if (recoveryJournal.isRequestCancelled(operationRequestId)) {
6248
+ recoveryJournal.unknown(operationRequestId);
6249
+ return;
6250
+ }
5981
6251
  }
5982
6252
  const messageAdmissionGeneration = workerAdmissionGeneration;
5983
6253
  const admissionCredentials = configuredCredentialGenerationFingerprint;
5984
6254
  const assertMessageAdmission = () => {
5985
6255
  if (operationRequestId) {
5986
- if (workerLeaseExpired || recoveryStore.isRequestCancelled(operationRequestId) || admissionCredentials !== configuredCredentialGenerationFingerprint || shutdownAfterClose || !workspaceConfigured) {
5987
- recoveryStore.unknown(operationRequestId);
6256
+ if (workerLeaseExpired || recoveryJournal.isRequestCancelled(operationRequestId) || admissionCredentials !== configuredCredentialGenerationFingerprint || shutdownAfterClose || !workspaceConfigured) {
6257
+ recoveryJournal.unknown(operationRequestId);
5988
6258
  throw new StaleWorkerAdmissionError();
5989
6259
  }
5990
6260
  return;
@@ -5999,6 +6269,8 @@ async function startWorker(options, projectRuntime = {
5999
6269
  });
6000
6270
  };
6001
6271
  if (message.type === "connected") {
6272
+ capacityReportSocket = ws;
6273
+ scheduleCapacityReport(true);
6002
6274
  return;
6003
6275
  }
6004
6276
  if (message.type === "project_mirror_refs_tokens") {
@@ -6119,7 +6391,7 @@ async function startWorker(options, projectRuntime = {
6119
6391
  }
6120
6392
  recoveryDeadline ??= Date.now() + WORKER_RECOVERY_TIMEOUT_MS;
6121
6393
  if (workerLeaseExpired || Date.now() >= recoveryDeadline) {
6122
- recoveryStore.unknown(message.requestId);
6394
+ recoveryJournal.unknown(message.requestId);
6123
6395
  return;
6124
6396
  }
6125
6397
  currentWorkerSocket?.close(1012, "Communication recovery");
@@ -6149,7 +6421,7 @@ async function startWorker(options, projectRuntime = {
6149
6421
  projectId: message.projectId,
6150
6422
  branchName: message.targetBranch
6151
6423
  };
6152
- const created = await workspaceSyncSingleFlight.runBranchMutation(() => {
6424
+ const created = await workspaceSyncSingleFlight.runBranchMutation(async () => {
6153
6425
  if (workspaceOperationsAreFenced(deferredWorkspaceConfiguration, activeWorkspaceIncidentId)) {
6154
6426
  throw new Error(
6155
6427
  `Project branch creation was fenced while queued by workspace incident ${deferredWorkspaceConfiguration?.incidentId ?? activeWorkspaceIncidentId}`
@@ -6164,7 +6436,7 @@ async function startWorker(options, projectRuntime = {
6164
6436
  const pendingWasRecorded = projectWorkspaceState.locallyPendingCreatedBranches.some(
6165
6437
  ({ projectId, branchName }) => projectId === message.projectId && branchName === message.targetBranch
6166
6438
  );
6167
- projectWorkspaceState = projectWorkspaceStateStore.recordPendingCreatedBranch(pendingBranch);
6439
+ projectWorkspaceState = await projectWorkspaceStateStore.recordPendingCreatedBranch(pendingBranch);
6168
6440
  let gitBranchCreated = false;
6169
6441
  try {
6170
6442
  if (!readyProjectIds.has(project.projectId)) {
@@ -6185,7 +6457,7 @@ async function startWorker(options, projectRuntime = {
6185
6457
  }
6186
6458
  return { branchPath: existingPath, baseCommitHash: existingBranch.baseCommitHash };
6187
6459
  }
6188
- const createdBranch = createOrRetryLinkedProjectBranch({
6460
+ const createdBranch = await createOrRetryLinkedProjectBranch({
6189
6461
  projectRoot: configuredProjectRoot(projectsRoot, project),
6190
6462
  primaryBranchName: primaryProjectBranch(project),
6191
6463
  sourceBranchName: message.sourceBranch,
@@ -6210,7 +6482,7 @@ async function startWorker(options, projectRuntime = {
6210
6482
  return createdBranch;
6211
6483
  } catch (error) {
6212
6484
  if (!pendingWasRecorded && !gitBranchCreated && !projectBranchMayExistAfterCreateFailure(error)) {
6213
- projectWorkspaceState = projectWorkspaceStateStore.rollbackPendingCreatedBranch(pendingBranch);
6485
+ projectWorkspaceState = await projectWorkspaceStateStore.rollbackPendingCreatedBranch(pendingBranch);
6214
6486
  }
6215
6487
  throw error;
6216
6488
  }
@@ -6254,7 +6526,7 @@ async function startWorker(options, projectRuntime = {
6254
6526
  });
6255
6527
  try {
6256
6528
  const deletionKey = projectBranchKey(message.projectId, message.branchName);
6257
- const deletionNeedsSync = await workspaceSyncSingleFlight.runBranchMutation(() => {
6529
+ const deletionNeedsSync = await workspaceSyncSingleFlight.runBranchMutation(async () => {
6258
6530
  if (workspaceOperationsAreFenced(deferredWorkspaceConfiguration, activeWorkspaceIncidentId)) {
6259
6531
  throw new Error(
6260
6532
  `Project branch deletion was fenced while queued by workspace incident ${deferredWorkspaceConfiguration?.incidentId ?? activeWorkspaceIncidentId}`
@@ -6281,12 +6553,12 @@ async function startWorker(options, projectRuntime = {
6281
6553
  if (!configuredBranch && !pendingLocalBranch && !hasDurableTombstone && !pendingMirrorDeletes.has(deletionKey)) {
6282
6554
  return false;
6283
6555
  }
6284
- projectWorkspaceState = projectWorkspaceStateStore.beginBranchDeletion({
6556
+ projectWorkspaceState = await projectWorkspaceStateStore.beginBranchDeletion({
6285
6557
  projectId: message.projectId,
6286
6558
  branchName: message.branchName
6287
6559
  });
6288
6560
  if (pendingLocalBranch) {
6289
- projectWorkspaceState = projectWorkspaceStateStore.clearPendingCreatedBranch(pendingLocalBranch);
6561
+ projectWorkspaceState = await projectWorkspaceStateStore.clearPendingCreatedBranch(pendingLocalBranch);
6290
6562
  }
6291
6563
  pendingCreatedBranchPublicationNotBefore.delete(pendingCreatedBranchKey(message.projectId, message.branchName));
6292
6564
  if (pendingMirrorDeletes.has(deletionKey)) return true;
@@ -6390,7 +6662,7 @@ async function startWorker(options, projectRuntime = {
6390
6662
  }
6391
6663
  if (message.type === "exec_terminal_ack") {
6392
6664
  pendingProcessTerminals.delete(message.runId);
6393
- recoveryStore.acknowledgeTerminal(message.runId);
6665
+ recoveryJournal.acknowledgeTerminal(message.runId);
6394
6666
  return;
6395
6667
  }
6396
6668
  if (message.type === "port_forward_connect") {
@@ -6445,42 +6717,6 @@ async function startWorker(options, projectRuntime = {
6445
6717
  sendSerializedWorkerMessage(ws, JSON.stringify({ type: "pong" }));
6446
6718
  return;
6447
6719
  }
6448
- if (message.type === "cancel") {
6449
- let resourceCancellationError;
6450
- try {
6451
- await workerCommandLauncher?.cancel(message.runId);
6452
- } catch (error) {
6453
- resourceCancellationError = error instanceof Error ? error.message : String(error);
6454
- }
6455
- const active = activeProcesses.get(message.runId);
6456
- let cancelled = resourceCancellationError === void 0;
6457
- let cancelMessage;
6458
- if (active) {
6459
- try {
6460
- closeProcessStdin(active);
6461
- await terminateProcessTree(active.process);
6462
- cancelMessage = `Stopped ${active.command}`;
6463
- } catch (error) {
6464
- cancelled = false;
6465
- cancelMessage = `Failed to stop ${active.command}: ${error instanceof Error ? error.message : String(error)}`;
6466
- }
6467
- } else {
6468
- cancelledProcessRuns.add(message.runId);
6469
- cancelMessage = "Cancellation queued before command start";
6470
- }
6471
- if (resourceCancellationError) cancelMessage += `; job resource termination failed: ${resourceCancellationError}`;
6472
- sendSerializedWorkerMessage(
6473
- ws,
6474
- JSON.stringify({
6475
- type: "cancel_result",
6476
- requestId: message.requestId,
6477
- runId: message.runId,
6478
- cancelled,
6479
- message: cancelMessage
6480
- })
6481
- );
6482
- return;
6483
- }
6484
6720
  if (message.type === "exec_stdin") {
6485
6721
  const sendAck = (payload) => sendSerializedWorkerMessage(
6486
6722
  ws,
@@ -6561,10 +6797,20 @@ async function startWorker(options, projectRuntime = {
6561
6797
  let mutationLeaseTransferred = false;
6562
6798
  try {
6563
6799
  resources = await acquireWorkerCommandLaunch(ws, message, assertMessageAdmission);
6564
- await reserveWorkspaceCommandAfterCurrentSync(message.target, workspaceSyncSingleFlight, () => {
6565
- workspaceSyncPriorityPtyTargets.set(message.ptyId, message.target);
6566
- if (targetMayMutateVisibleWorkspace(message.target)) recordVisibleWorkspaceMutation(message.target);
6567
- });
6800
+ const reservationAbort = registerPendingReservation(`pty:${message.ptyId}`, void 0);
6801
+ try {
6802
+ await reserveWorkspaceCommandAfterCurrentSync(
6803
+ message.target,
6804
+ workspaceSyncSingleFlight,
6805
+ () => {
6806
+ workspaceSyncPriorityPtyTargets.set(message.ptyId, message.target);
6807
+ if (targetMayMutateVisibleWorkspace(message.target)) recordVisibleWorkspaceMutation(message.target);
6808
+ },
6809
+ { signal: reservationAbort.signal }
6810
+ );
6811
+ } finally {
6812
+ unregisterPendingReservation(`pty:${message.ptyId}`, reservationAbort);
6813
+ }
6568
6814
  releaseWorkspaceMutation = await acquireWorkspaceCommandMutation(message.target, workspaceSyncSingleFlight);
6569
6815
  if (workspaceOperationsAreFenced(deferredWorkspaceConfiguration, activeWorkspaceIncidentId)) {
6570
6816
  throw new Error(
@@ -6629,6 +6875,7 @@ async function startWorker(options, projectRuntime = {
6629
6875
  }
6630
6876
  if (message.type === "pty_close") {
6631
6877
  const activePty = activePtys.get(message.ptyId);
6878
+ abortPendingReservation(`pty:${message.ptyId}`, "Shell closed before it started");
6632
6879
  await closePty(message);
6633
6880
  if (activePty && targetMayMutateVisibleWorkspace(activePty.target)) {
6634
6881
  markWorkspaceDirty({ type: "process_terminal", detail: `pty ${message.ptyId} closed` }, true, activePty.target);
@@ -6665,11 +6912,21 @@ async function startWorker(options, projectRuntime = {
6665
6912
  try {
6666
6913
  resources = await acquireWorkerCommandLaunch(ws, message, assertMessageAdmission);
6667
6914
  if (hasWorkspaceEffect) {
6668
- await reserveWorkspaceCommandAfterCurrentSync(message.target, workspaceSyncSingleFlight, () => {
6669
- workspaceSyncPriorityProcessTargets.set(message.runId, message.target);
6670
- targetReserved = true;
6671
- if (targetMayMutateVisibleWorkspace(message.target)) recordVisibleWorkspaceMutation(message.target);
6672
- });
6915
+ const reservationAbort = registerPendingReservation(`run:${message.runId}`, message.sessionId);
6916
+ try {
6917
+ await reserveWorkspaceCommandAfterCurrentSync(
6918
+ message.target,
6919
+ workspaceSyncSingleFlight,
6920
+ () => {
6921
+ workspaceSyncPriorityProcessTargets.set(message.runId, message.target);
6922
+ targetReserved = true;
6923
+ if (targetMayMutateVisibleWorkspace(message.target)) recordVisibleWorkspaceMutation(message.target);
6924
+ },
6925
+ { signal: reservationAbort.signal }
6926
+ );
6927
+ } finally {
6928
+ unregisterPendingReservation(`run:${message.runId}`, reservationAbort);
6929
+ }
6673
6930
  }
6674
6931
  const runCommand = async () => {
6675
6932
  if (!workspaceCommandTransportIsAllowed(deferredWorkspaceConfiguration, activeWorkspaceIncidentId, "exec")) {
@@ -6721,7 +6978,7 @@ async function startWorker(options, projectRuntime = {
6721
6978
  cancelledProcessRuns.delete(message.runId);
6722
6979
  }
6723
6980
  if (infrastructureStoppedRunIds.has(message.runId)) {
6724
- recoveryStore.unknown(message.requestId);
6981
+ recoveryJournal.unknown(message.requestId);
6725
6982
  return;
6726
6983
  }
6727
6984
  sendSerializedWorkerMessage(ws, JSON.stringify(result));
@@ -6754,8 +7011,10 @@ async function startWorker(options, projectRuntime = {
6754
7011
  const runCommand = async () => {
6755
7012
  try {
6756
7013
  const resolvedTarget = resolveMessageTarget(message.target);
6757
- process.stdout.write(`[r5d-worker] exec_start ${message.runId}: ${message.argv.join(" ")}
6758
- `);
7014
+ process.stdout.write(
7015
+ `[r5d-worker] exec_start ${message.runId}${message.commandClass === "control" ? " (control)" : ""}: ${message.argv.join(" ")}
7016
+ `
7017
+ );
6759
7018
  await executeStreamingCommand({
6760
7019
  resources,
6761
7020
  ws,
@@ -6793,11 +7052,21 @@ async function startWorker(options, projectRuntime = {
6793
7052
  try {
6794
7053
  resources = await acquireWorkerCommandLaunch(ws, message, assertMessageAdmission);
6795
7054
  if (hasWorkspaceEffect) {
6796
- await reserveWorkspaceCommandAfterCurrentSync(message.target, workspaceSyncSingleFlight, () => {
6797
- workspaceSyncPriorityProcessTargets.set(message.runId, message.target);
6798
- targetReserved = true;
6799
- if (targetMayMutateVisibleWorkspace(message.target)) recordVisibleWorkspaceMutation(message.target);
6800
- });
7055
+ const reservationAbort = registerPendingReservation(`run:${message.runId}`, message.sessionId);
7056
+ try {
7057
+ await reserveWorkspaceCommandAfterCurrentSync(
7058
+ message.target,
7059
+ workspaceSyncSingleFlight,
7060
+ () => {
7061
+ workspaceSyncPriorityProcessTargets.set(message.runId, message.target);
7062
+ targetReserved = true;
7063
+ if (targetMayMutateVisibleWorkspace(message.target)) recordVisibleWorkspaceMutation(message.target);
7064
+ },
7065
+ { signal: reservationAbort.signal }
7066
+ );
7067
+ } finally {
7068
+ unregisterPendingReservation(`run:${message.runId}`, reservationAbort);
7069
+ }
6801
7070
  }
6802
7071
  await runReservedWorkspaceCommand(message.target, workspaceSyncSingleFlight, runCommand, () => {
6803
7072
  if (targetReserved) {
@@ -6832,16 +7101,40 @@ async function startWorker(options, projectRuntime = {
6832
7101
  return;
6833
7102
  }
6834
7103
  const reservesVisibleWorkspace = targetMayMutateVisibleWorkspace(message.target);
6835
- const mutatesVisibleWorkspace = (message.type === "write" || message.type === "edit" || message.type === "update_task_status" || message.type === "read_plan" && message.createIfMissing) && targetMayMutateVisibleWorkspace(message.target);
7104
+ const mutatesFiles = message.type === "write" || message.type === "edit" || message.type === "update_task_status" || message.type === "read_plan" && message.createIfMissing;
7105
+ const mutatesVisibleWorkspace = mutatesFiles && targetMayMutateVisibleWorkspace(message.target);
6836
7106
  if (reservesVisibleWorkspace) {
6837
- await reserveWorkspaceCommandAfterCurrentSync(message.target, workspaceSyncSingleFlight, () => {
6838
- workspaceSyncPriorityOperationTargets.set(message.requestId, message.target);
6839
- if (mutatesVisibleWorkspace) recordVisibleWorkspaceMutation(message.target);
6840
- });
7107
+ const reservationAbort = registerPendingReservation(
7108
+ `op:${message.requestId}`,
7109
+ "sessionId" in message && typeof message.sessionId === "string" ? message.sessionId : void 0
7110
+ );
7111
+ try {
7112
+ await reserveWorkspaceCommandAfterCurrentSync(
7113
+ message.target,
7114
+ workspaceSyncSingleFlight,
7115
+ () => {
7116
+ workspaceSyncPriorityOperationTargets.set(message.requestId, message.target);
7117
+ if (mutatesVisibleWorkspace) recordVisibleWorkspaceMutation(message.target);
7118
+ },
7119
+ { signal: reservationAbort.signal }
7120
+ );
7121
+ } catch (error) {
7122
+ sendSerializedWorkerMessage(
7123
+ ws,
7124
+ JSON.stringify({
7125
+ type: "operation_result",
7126
+ requestId: message.requestId,
7127
+ error: error instanceof Error ? error.message : String(error)
7128
+ })
7129
+ );
7130
+ return;
7131
+ } finally {
7132
+ unregisterPendingReservation(`op:${message.requestId}`, reservationAbort);
7133
+ }
6841
7134
  }
6842
7135
  let dirtyTrigger;
6843
7136
  try {
6844
- const result = await runWorkspaceCommand(message.target, workspaceSyncSingleFlight, async () => {
7137
+ const operation = runWorkspaceCommand(message.target, workspaceSyncSingleFlight, async () => {
6845
7138
  const resolvedTarget = resolveMessageTarget(message.target);
6846
7139
  return await executeOperation({
6847
7140
  message,
@@ -6852,6 +7145,8 @@ async function startWorker(options, projectRuntime = {
6852
7145
  assertAdmission: assertMessageAdmission
6853
7146
  });
6854
7147
  });
7148
+ if (mutatesFiles && typeof message.sessionId === "string") sessionFileMutations.track(message.sessionId, operation);
7149
+ const result = await operation;
6855
7150
  sendSerializedWorkerMessage(
6856
7151
  ws,
6857
7152
  JSON.stringify({
@@ -6935,6 +7230,7 @@ async function startWorker(options, projectRuntime = {
6935
7230
  projectRuntime.configuration.deferred = deferredWorkspaceConfiguration;
6936
7231
  }
6937
7232
  if (currentWorkerSocket === ws) currentWorkerSocket = null;
7233
+ abortPendingReservations("Worker connection closed before command start");
6938
7234
  workerRecoveryId = "reconnecting";
6939
7235
  const reason = event.reason ? `: ${event.reason}` : "";
6940
7236
  process.stderr.write(`[r5d-worker] disconnected (${event.code}${reason})
@@ -6968,6 +7264,11 @@ async function startWorker(options, projectRuntime = {
6968
7264
  });
6969
7265
  }
6970
7266
  function installWorkerRuntimeFailureLogging() {
7267
+ installWorkspaceFilesystemExecutorFailureHandler((error) => {
7268
+ process.stderr.write(`[r5d-worker] ${error.message}; restarting the worker runtime
7269
+ `);
7270
+ void exitWorkerRuntime(WORKER_RECONNECT_EXIT_CODE);
7271
+ });
6971
7272
  process.on("uncaughtException", (error) => {
6972
7273
  process.stderr.write(`[r5d-worker] uncaught exception: ${error instanceof Error ? error.stack ?? error.message : String(error)}
6973
7274
  `);
@@ -6980,11 +7281,6 @@ function installWorkerRuntimeFailureLogging() {
6980
7281
  });
6981
7282
  }
6982
7283
  async function main() {
6983
- if (process.argv[2] === PROJECT_SNAPSHOT_RECOVERY_HELPER_COMMAND) {
6984
- const exitCode2 = runProjectSnapshotRecoveryHelper(process.argv.slice(3));
6985
- if (exitCode2 !== 0) process.exit(exitCode2);
6986
- return;
6987
- }
6988
7284
  const parsed = parseArgs(process.argv.slice(2));
6989
7285
  if (parsed.command === "help") {
6990
7286
  printHelp();