@ricsam/r5d-worker 0.0.107 → 0.0.109

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cjs/main.cjs CHANGED
@@ -56,7 +56,13 @@ __export(main_exports, {
56
56
  writeWorkerTextFile: () => writeWorkerTextFile
57
57
  });
58
58
  module.exports = __toCommonJS(main_exports);
59
+ var import_plan_paths = require("./plan-paths.cjs");
60
+ var import_plan_parser = require("./plan-parser.cjs");
61
+ var import_plan_lint = require("./plan-lint.cjs");
59
62
  var import_node_fs = __toESM(require("node:fs"), 1);
63
+ var import_recovery_store = require("./recovery-store.cjs");
64
+ var import_recovery_output_window = require("./recovery-output-window.cjs");
65
+ var import_recovery_protocol = require("./recovery-protocol.cjs");
60
66
  var import_node_path = __toESM(require("node:path"), 1);
61
67
  var import_node_crypto = require("node:crypto");
62
68
  var import_node_os = __toESM(require("node:os"), 1);
@@ -158,6 +164,63 @@ const cancelledProcessRuns = /* @__PURE__ */ new Set();
158
164
  const activePtys = /* @__PURE__ */ new Map();
159
165
  const workspaceSyncPriorityPtyTargets = /* @__PURE__ */ new Map();
160
166
  let currentWorkerSocket = null;
167
+ const workerRuntimeId = crypto.randomUUID();
168
+ const workerOutputWindow = new import_recovery_output_window.WorkerOutputWindow();
169
+ let recoveryStore;
170
+ let workerReconnectAttempt = 0;
171
+ let workerRecoveryId;
172
+ let workerExecutionLeaseDeadline = 0;
173
+ let workerExecutionLeaseTimer;
174
+ let workerLeaseExpired = false;
175
+ const infrastructureStoppedRunIds = /* @__PURE__ */ new Set();
176
+ function isWorkerCommunicationFailure(message) {
177
+ return /Could not resolve host|Couldn't connect|Failed to connect|Connection reset|Connection timed out|remote end hung up|Empty reply|requested URL returned error: (?:408|429|5\d\d)|fetch failed|socket connection|ECONNRESET|ECONNREFUSED|ETIMEDOUT|Unable to connect|network connection was lost|Worker communication recovery expired|operation was aborted due to timeout|operation timed out|Failed to (?:fetch|download).*\((?:408|429|5\d\d)\)/i.test(
178
+ message
179
+ );
180
+ }
181
+ async function retryWorkerServerRead(read) {
182
+ let deadline;
183
+ let retry = 0;
184
+ while (true) {
185
+ try {
186
+ return await read();
187
+ } catch (error) {
188
+ if (!recoveryStore || !isWorkerCommunicationFailure(error instanceof Error ? error.message : String(error))) throw error;
189
+ deadline ??= Date.now() + import_recovery_protocol.WORKER_RECOVERY_TIMEOUT_MS;
190
+ if (workerLeaseExpired || Date.now() >= deadline) throw new WorkerServerUnavailableError("Worker communication recovery expired");
191
+ currentWorkerSocket?.close(1012, "Communication recovery");
192
+ await Bun.sleep((0, import_supervisor.workerReconnectDelayMs)(Math.random, retry++));
193
+ }
194
+ }
195
+ }
196
+ function renewWorkerExecutionLease(leaseMs) {
197
+ if (!Number.isFinite(leaseMs) || leaseMs <= 0) return;
198
+ workerLeaseExpired = false;
199
+ workerExecutionLeaseDeadline = Date.now() + Math.min(leaseMs, import_recovery_protocol.WORKER_RECOVERY_TIMEOUT_MS);
200
+ if (workerExecutionLeaseTimer) clearTimeout(workerExecutionLeaseTimer);
201
+ workerExecutionLeaseTimer = setTimeout(
202
+ () => {
203
+ if (Date.now() < workerExecutionLeaseDeadline) return;
204
+ workerLeaseExpired = true;
205
+ for (const operation of recoveryStore?.operations() ?? []) {
206
+ if (operation.sessionId && operation.state !== "completed") recoveryStore?.cancelSession(operation.sessionId);
207
+ }
208
+ for (const [runId, active] of activeProcesses) {
209
+ if (!active.sessionId) continue;
210
+ infrastructureStoppedRunIds.add(runId);
211
+ recoveryStore?.unknownRun(runId);
212
+ cancelledProcessRuns.add(runId);
213
+ closeProcessStdin(active);
214
+ void (0, import_process_tree.terminateProcessTree)(active.process).catch(
215
+ (error) => process.stderr.write(`[r5d-worker] execution lease termination failed: ${error}
216
+ `)
217
+ );
218
+ }
219
+ },
220
+ Math.min(leaseMs, import_recovery_protocol.WORKER_RECOVERY_TIMEOUT_MS)
221
+ );
222
+ workerExecutionLeaseTimer.unref();
223
+ }
161
224
  let workerAdmissionGeneration = 0;
162
225
  const workspaceMutationGate = new import_workspace_mutation_gate.WorkspaceMutationGate();
163
226
  let workspaceSyncQueue = Promise.resolve();
@@ -565,7 +628,7 @@ function targetPlanDir(planRoot, target) {
565
628
  function planEnv(activePlanId) {
566
629
  const env = {};
567
630
  if (activePlanId) {
568
- validatePlanId(activePlanId);
631
+ (0, import_plan_paths.validatePlanId)(activePlanId);
569
632
  env.R5D_ACTIVE_PLAN_ID = activePlanId;
570
633
  }
571
634
  return env;
@@ -616,40 +679,46 @@ function parseRemoteFileManifest(value, collectionKey, label) {
616
679
  });
617
680
  }
618
681
  async function fetchRemoteFileManifest(input) {
619
- const response = await fetch(input.url, {
620
- headers: {
621
- Authorization: `Bearer ${input.token}`
682
+ return retryWorkerServerRead(async () => {
683
+ const response = await fetch(input.url, {
684
+ signal: AbortSignal.timeout(3e4),
685
+ headers: {
686
+ Authorization: `Bearer ${input.token}`
687
+ }
688
+ });
689
+ if (!response.ok) {
690
+ const detail = (await readResponseText(response)).trim();
691
+ throw new Error(`Failed to fetch ${input.label} manifest (${response.status})${detail ? `: ${detail}` : ""}`);
622
692
  }
693
+ return parseRemoteFileManifest(await response.json(), input.collectionKey, input.label);
623
694
  });
624
- if (!response.ok) {
625
- const detail = (await readResponseText(response)).trim();
626
- throw new Error(`Failed to fetch ${input.label} manifest (${response.status})${detail ? `: ${detail}` : ""}`);
627
- }
628
- return parseRemoteFileManifest(await response.json(), input.collectionKey, input.label);
629
695
  }
630
696
  async function downloadRemoteSyncedFile(input) {
631
- const targetPath = import_node_path.default.join(input.targetDir, input.file.filename);
632
- assertInsideRoot(input.targetDir, targetPath, input.label);
633
- const response = await fetch(input.url, {
634
- headers: {
635
- Authorization: `Bearer ${input.token}`
697
+ return retryWorkerServerRead(async () => {
698
+ const targetPath = import_node_path.default.join(input.targetDir, input.file.filename);
699
+ assertInsideRoot(input.targetDir, targetPath, input.label);
700
+ const response = await fetch(input.url, {
701
+ signal: AbortSignal.timeout(3e4),
702
+ headers: {
703
+ Authorization: `Bearer ${input.token}`
704
+ }
705
+ });
706
+ if (!response.ok) {
707
+ const detail = (await readResponseText(response)).trim();
708
+ throw new Error(`Failed to download ${input.label} ${input.file.filename} (${response.status})${detail ? `: ${detail}` : ""}`);
636
709
  }
710
+ const buffer = Buffer.from(await response.arrayBuffer());
711
+ const receivedHash = (0, import_node_crypto.createHash)("sha256").update(buffer).digest("hex");
712
+ if (receivedHash !== input.file.sha256) {
713
+ throw new Error(`Downloaded ${input.label} ${input.file.filename} failed sha256 verification`);
714
+ }
715
+ if (buffer.length !== input.file.size) {
716
+ throw new Error(`Downloaded ${input.label} ${input.file.filename} has unexpected size`);
717
+ }
718
+ const tempPath = `${targetPath}.${process.pid}.${Date.now()}.tmp`;
719
+ import_node_fs.default.writeFileSync(tempPath, buffer);
720
+ import_node_fs.default.renameSync(tempPath, targetPath);
637
721
  });
638
- if (!response.ok) {
639
- const detail = (await readResponseText(response)).trim();
640
- throw new Error(`Failed to download ${input.label} ${input.file.filename} (${response.status})${detail ? `: ${detail}` : ""}`);
641
- }
642
- const buffer = Buffer.from(await response.arrayBuffer());
643
- const receivedHash = (0, import_node_crypto.createHash)("sha256").update(buffer).digest("hex");
644
- if (receivedHash !== input.file.sha256) {
645
- throw new Error(`Downloaded ${input.label} ${input.file.filename} failed sha256 verification`);
646
- }
647
- if (buffer.length !== input.file.size) {
648
- throw new Error(`Downloaded ${input.label} ${input.file.filename} has unexpected size`);
649
- }
650
- const tempPath = `${targetPath}.${process.pid}.${Date.now()}.tmp`;
651
- import_node_fs.default.writeFileSync(tempPath, buffer);
652
- import_node_fs.default.renameSync(tempPath, targetPath);
653
722
  }
654
723
  async function syncRemoteFileSet(input) {
655
724
  import_node_fs.default.mkdirSync(input.targetDir, { recursive: true });
@@ -776,6 +845,7 @@ async function verifyR5dctlAuth(baseUrl, token, workerLabel) {
776
845
  let response;
777
846
  try {
778
847
  response = await fetch(statusUrl, {
848
+ signal: AbortSignal.timeout(3e4),
779
849
  headers: {
780
850
  Authorization: `Bearer ${token}`
781
851
  }
@@ -834,11 +904,6 @@ function validateProjectId(projectId) {
834
904
  function validateBranchName(branchName) {
835
905
  (0, import_managed_paths.validateManagedBranchName)(branchName);
836
906
  }
837
- function validatePlanId(planId) {
838
- if (!/^[a-z0-9][a-z0-9-]*$/.test(planId)) {
839
- throw new Error(`Invalid plan id from server: ${planId}`);
840
- }
841
- }
842
907
  const NON_RECURSIVE_GIT_CONFIG_ARGS = [
843
908
  "-c",
844
909
  "submodule.recurse=false",
@@ -1745,6 +1810,7 @@ const workerGitSecurityTestHarness = {
1745
1810
  workspaceSyncFailureHydrationIsSafe,
1746
1811
  assertWorkerChildAdmission,
1747
1812
  executeWriteFileOperation,
1813
+ executeOperation,
1748
1814
  executeEditFileOperation,
1749
1815
  terminateCredentialBearingChildren,
1750
1816
  terminateCredentialBearingChildrenWithRetention,
@@ -2183,6 +2249,55 @@ function editWorkerTextFile(branchPath, filePath, edits, builtInPaths) {
2183
2249
  edits: edits.length
2184
2250
  };
2185
2251
  }
2252
+ async function executePlanOperation(input) {
2253
+ const { message, resolvedTarget } = input;
2254
+ const filePath = message.type === "lint_plans" ? (0, import_plan_paths.getWorkerPlansPath)(resolvedTarget.target) : (0, import_plan_paths.getWorkerPlanPath)(resolvedTarget.target, message.planId);
2255
+ const builtInPaths = await prepareBuiltInToolPathsForTarget({
2256
+ target: resolvedTarget.target,
2257
+ filePath,
2258
+ baseUrl: input.baseUrl,
2259
+ token: input.token,
2260
+ rootDir: input.rootDir,
2261
+ sessionId: message.sessionId,
2262
+ access: message.type === "update_task_status" || message.type === "read_plan" && message.createIfMissing ? "write" : "read"
2263
+ });
2264
+ const resolved = resolveWorkerFilePath(resolvedTarget.rootPath, filePath, builtInPaths);
2265
+ if (message.type === "lint_plans") {
2266
+ const result = await (0, import_plan_lint.lintPlans)(resolved.absolutePath, message.glob);
2267
+ return {
2268
+ type: "lint_plans",
2269
+ content: result.lintedPlanCount === 0 ? message.glob ? `No plan files matched glob "${message.glob}".` : "No plan files found." : (0, import_plan_lint.formatPlanLintErrors)(result.errors) || "No plan lint issues found."
2270
+ };
2271
+ }
2272
+ return withFileMutationQueue(mutationQueueKey(resolvedTarget.target, resolved), async () => {
2273
+ input.assertAdmission();
2274
+ if (message.type === "read_plan") {
2275
+ let created = false;
2276
+ if (message.createIfMissing) {
2277
+ try {
2278
+ import_node_fs.default.writeFileSync(resolved.absolutePath, (0, import_plan_parser.createPlanMarkdown)(message.planId), { encoding: "utf8", flag: "wx" });
2279
+ created = true;
2280
+ } catch (error) {
2281
+ if (error.code !== "EEXIST") throw error;
2282
+ }
2283
+ }
2284
+ return {
2285
+ type: "read_plan",
2286
+ created,
2287
+ read: readWorkerTextFile(resolvedTarget.rootPath, filePath, void 0, void 0, builtInPaths)
2288
+ };
2289
+ }
2290
+ const content = import_node_fs.default.readFileSync(resolved.absolutePath, "utf8");
2291
+ writeWorkerTextFile(resolvedTarget.rootPath, filePath, (0, import_plan_parser.updateTaskCheckbox)(content, message.taskId, message.checked), builtInPaths);
2292
+ return {
2293
+ type: "update_task_status",
2294
+ planId: message.planId,
2295
+ taskId: message.taskId,
2296
+ checked: message.checked,
2297
+ message: `Task ${message.taskId} marked ${message.checked ? "complete" : "incomplete"}.`
2298
+ };
2299
+ });
2300
+ }
2186
2301
  async function executeReadFileOperation(input) {
2187
2302
  const builtInPaths = await prepareBuiltInToolPathsForTarget({
2188
2303
  target: input.resolvedTarget.target,
@@ -2561,6 +2676,10 @@ async function executeViewFileBytesOperation(input) {
2561
2676
  }
2562
2677
  async function executeOperation(input) {
2563
2678
  switch (input.message.type) {
2679
+ case "read_plan":
2680
+ case "lint_plans":
2681
+ case "update_task_status":
2682
+ return executePlanOperation({ ...input, message: input.message });
2564
2683
  case "read":
2565
2684
  return executeReadFileOperation({ ...input, message: input.message });
2566
2685
  case "write":
@@ -2890,23 +3009,46 @@ function closeProcessStdin(active) {
2890
3009
  }
2891
3010
  active.stdin = void 0;
2892
3011
  }
2893
- function sendWorkerMessage(ws, message) {
3012
+ function sendReplayWorkerMessage(ws, message) {
2894
3013
  const target = currentWorkerSocket ?? ws;
2895
- try {
2896
- target.send(JSON.stringify(message));
2897
- } catch (error) {
2898
- process.stderr.write(`[r5d-worker] failed to send ${message.type}: ${error instanceof Error ? error.message : String(error)}
2899
- `);
3014
+ if (target.readyState !== WebSocket.OPEN) return false;
3015
+ const send = (payload) => {
3016
+ try {
3017
+ target.send(JSON.stringify(payload));
3018
+ return true;
3019
+ } catch {
3020
+ return false;
3021
+ }
3022
+ };
3023
+ return message.type === "exec_output" ? workerOutputWindow.send(message, send, target.bufferedAmount) : send(message);
3024
+ }
3025
+ function sendWorkerMessage(ws, message) {
3026
+ if (recoveryStore && "requestId" in message && typeof message.requestId === "string" && recoveryStore.isUnknown(message.requestId))
3027
+ return;
3028
+ const transportError = "error" in message && typeof message.error === "string" ? message.error : message.type === "workspace_sync_result" && message.result.outcome === "failed" ? message.result.error : void 0;
3029
+ if (recoveryStore && transportError && isWorkerCommunicationFailure(transportError) && "requestId" in message && typeof message.requestId === "string") {
3030
+ recoveryStore.unknown(message.requestId);
3031
+ currentWorkerSocket?.close(1012, "Communication recovery");
3032
+ return;
2900
3033
  }
3034
+ const recorded = recoveryStore?.record(message) ?? message;
3035
+ if ((message.type === "exec_exit" || message.type === "exec_error") && infrastructureStoppedRunIds.has(message.runId)) {
3036
+ recoveryStore?.unknownRun(message.runId);
3037
+ return;
3038
+ }
3039
+ if (message.type === "exec_output" && workerRecoveryId !== void 0) return;
3040
+ sendReplayWorkerMessage(ws, recorded);
3041
+ }
3042
+ function sendSerializedWorkerMessage(ws, serialized) {
3043
+ sendWorkerMessage(ws, JSON.parse(serialized));
2901
3044
  }
2902
3045
  function sendWorkerMessageFromCurrentSource(ws, message) {
2903
- if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) return;
2904
- try {
2905
- ws.send(JSON.stringify(message));
2906
- } catch (error) {
2907
- process.stderr.write(`[r5d-worker] failed to send ${message.type}: ${error instanceof Error ? error.message : String(error)}
2908
- `);
3046
+ if ("requestId" in message && typeof message.requestId === "string" && message.type !== "workspace_configured") {
3047
+ sendWorkerMessage(ws, message);
3048
+ return;
2909
3049
  }
3050
+ if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) return;
3051
+ sendWorkerMessage(ws, message);
2910
3052
  }
2911
3053
  function buildActiveProcessReports() {
2912
3054
  return Array.from(activeProcesses.entries()).filter(([, active]) => active.sessionId.length > 0).map(([runId, active]) => ({
@@ -3420,14 +3562,21 @@ function websocketUrl(baseUrl, label) {
3420
3562
  url.searchParams.set("version", getWorkerVersion());
3421
3563
  return url.toString();
3422
3564
  }
3423
- async function startWorker(options) {
3424
- advanceWorkerAdmissionGeneration();
3565
+ async function startWorker(options, projectRuntime = {
3566
+ projectConfigById: /* @__PURE__ */ new Map(),
3567
+ readyProjectIds: /* @__PURE__ */ new Set(),
3568
+ reconciledProjectConfigFingerprints: /* @__PURE__ */ new Map(),
3569
+ gitIdentity: null
3570
+ }) {
3425
3571
  const config = readConfig(options.configPath);
3426
3572
  const { baseUrl, token } = resolveCredentials(options, config);
3427
3573
  initializeCredentialStoreDirectoryOnce();
3428
3574
  const label = requireValue(options.label, "Missing required --label <label>");
3429
3575
  validateLabel(label);
3430
3576
  const rootDir = import_node_path.default.resolve(options.rootDir ?? process.env.R5D_ROOT ?? defaultRootDir());
3577
+ recoveryStore ??= new import_recovery_store.WorkerRecoveryStore(
3578
+ import_node_path.default.join(rootDir, ".worker-recovery", (0, import_node_crypto.createHash)("sha256").update(`${baseUrl}\0${label}`).digest("hex"), "protocol.sqlite")
3579
+ );
3431
3580
  const projectsRoot = import_node_path.default.join(rootDir, "projects");
3432
3581
  const syncRoot = import_node_path.default.join(rootDir, "sync");
3433
3582
  const workspaceShadowRoot = import_node_path.default.join(syncRoot, "workspace");
@@ -3448,7 +3597,7 @@ async function startWorker(options) {
3448
3597
  `);
3449
3598
  runGit(["--version"]);
3450
3599
  await verifyR5dctlAuth(baseUrl, token, label);
3451
- const initializedWorkspaceState = await workspaceSyncSingleFlight.runExclusive(() => {
3600
+ const initializedWorkspaceState = projectRuntime.initializedWorkspaceState ?? await workspaceSyncSingleFlight.runExclusive(() => {
3452
3601
  import_node_fs.default.mkdirSync(projectsRoot, { recursive: true });
3453
3602
  import_node_fs.default.mkdirSync(syncRoot, { recursive: true });
3454
3603
  import_node_fs.default.mkdirSync(artifactRoot, { recursive: true });
@@ -3459,12 +3608,11 @@ async function startWorker(options) {
3459
3608
  });
3460
3609
  return { store, state: store.read() };
3461
3610
  });
3611
+ projectRuntime.initializedWorkspaceState = initializedWorkspaceState;
3462
3612
  const projectWorkspaceStateStore = initializedWorkspaceState.store;
3463
- let projectWorkspaceState = initializedWorkspaceState.state;
3464
- const projectConfigById = /* @__PURE__ */ new Map();
3613
+ let projectWorkspaceState = initializedWorkspaceState.store.read();
3614
+ const { projectConfigById, readyProjectIds, reconciledProjectConfigFingerprints } = projectRuntime;
3465
3615
  const pendingCheckouts = /* @__PURE__ */ new Map();
3466
- const readyProjectIds = /* @__PURE__ */ new Set();
3467
- const reconciledProjectConfigFingerprints = /* @__PURE__ */ new Map();
3468
3616
  const lastObservedProjectHeads = /* @__PURE__ */ new Map();
3469
3617
  const advertisedProjectMirrorRefsTokens = /* @__PURE__ */ new Map();
3470
3618
  const advertisedProjectMirrorMountTokens = /* @__PURE__ */ new Map();
@@ -3494,7 +3642,7 @@ async function startWorker(options) {
3494
3642
  let workspaceRemoteUrl = null;
3495
3643
  let workspaceCredentialHelper = null;
3496
3644
  let workspaceCredentialUsername = null;
3497
- let workspaceGitIdentity = null;
3645
+ let workspaceGitIdentity = projectRuntime.gitIdentity;
3498
3646
  let workspaceConfigured = false;
3499
3647
  let activeWorkspaceIncidentId = null;
3500
3648
  let deferredWorkspaceConfiguration = null;
@@ -3502,11 +3650,12 @@ async function startWorker(options) {
3502
3650
  let workspaceConfigurationReceiptGeneration = 0;
3503
3651
  let workspaceAutomaticTimer;
3504
3652
  let workspacePeriodicTimer;
3505
- let pendingAutomaticTrigger;
3653
+ let pendingAutomaticTrigger = projectRuntime.pendingAutomaticTrigger;
3506
3654
  const pendingCreatedBranchPublicationNotBefore = /* @__PURE__ */ new Map();
3507
3655
  const workspaceProjectionLedger = new import_workspace_projection_ledger.WorkspaceProjectionLedger();
3508
3656
  let automaticSyncInFlight = false;
3509
3657
  let terminalReplayTimer;
3658
+ let outputReplayTimer;
3510
3659
  let workspaceSyncRequestsInFlight = 0;
3511
3660
  let lastWorkspaceSyncMirrorObservationMs = 0;
3512
3661
  let lastInventoriedMissingCheckouts;
@@ -3855,7 +4004,9 @@ async function startWorker(options) {
3855
4004
  const reportedMissingCheckouts = (inventory) => inventory.missingCheckouts.map(({ projectId, branchName, reason }) => ({ projectId, branchName, reason }));
3856
4005
  const logMissingCheckouts = (inventory) => {
3857
4006
  if (inventory.missingCheckouts.length === 0) return;
3858
- const described = inventory.missingCheckouts.map(({ checkoutPath, reason }) => `${checkoutPath} (${reason === "missing" ? "directory missing" : "not a linked worktree of the project"})`).join(", ");
4007
+ const described = inventory.missingCheckouts.map(
4008
+ ({ checkoutPath, reason }) => `${checkoutPath} (${reason === "missing" ? "directory missing" : "not a linked worktree of the project"})`
4009
+ ).join(", ");
3859
4010
  process.stderr.write(
3860
4011
  `[r5d-worker] skipping ${inventory.missingCheckouts.length} configured project checkout(s) this cycle: ${described}; delete the branch through the platform or reconnect this worker to recreate the checkout
3861
4012
  `
@@ -4426,7 +4577,9 @@ async function startWorker(options) {
4426
4577
  error: error instanceof Error ? error.message : String(error)
4427
4578
  };
4428
4579
  };
4429
- const workspaceHydrationTransactionHooks = createWorkspaceHydrationLeaseHooks((message) => sendWorkerMessageFromCurrentSource(ws, message));
4580
+ const workspaceHydrationTransactionHooks = createWorkspaceHydrationLeaseHooks(
4581
+ (message) => sendWorkerMessageFromCurrentSource(ws, message)
4582
+ );
4430
4583
  const performWorkspaceSync = async (input) => {
4431
4584
  if (!workspaceRemoteUrl || !workspaceCredentialHelper || !workspaceCredentialUsername || !workspaceGitIdentity) {
4432
4585
  throw new Error("Worker workspace configuration has not been received");
@@ -4483,7 +4636,11 @@ async function startWorker(options) {
4483
4636
  const outerRemediation = input.trigger.type === "remediation" || input.trigger.type === "remediation_confirm";
4484
4637
  const mounts = outerRemediation ? (0, import_workspace_git_sync.workspaceGitMountsForRemediation)(configuredMounts) : configuredMounts;
4485
4638
  if (outerRemediation) workspaceProjectionLedger.reset();
4486
- const observedProjectHeads = outerRemediation ? [] : await observeProjectHeadsBeforeOuterWorkspace({ allowNonFastForward: false, failClosed: false, projects: checkoutInventory.projects });
4639
+ const observedProjectHeads = outerRemediation ? [] : await observeProjectHeadsBeforeOuterWorkspace({
4640
+ allowNonFastForward: false,
4641
+ failClosed: false,
4642
+ projects: checkoutInventory.projects
4643
+ });
4487
4644
  const expectedMirrorHeads = /* @__PURE__ */ new Map();
4488
4645
  for (const bundle of observedProjectHeads) {
4489
4646
  for (const observation of bundle.observations) {
@@ -4590,7 +4747,7 @@ async function startWorker(options) {
4590
4747
  });
4591
4748
  const workspaceSyncAdmissionSnapshotsMatch = (left, right) => left.generation === right.generation && left.activeIncidentId === right.activeIncidentId && left.deferredIncidentId === right.deferredIncidentId;
4592
4749
  const assertWorkspaceSyncStillAdmitted = (admitted, trigger) => {
4593
- if (!workspaceSyncAdmissionSnapshotsMatch(admitted, captureWorkspaceSyncAdmission()) || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN || !deferredWorkspaceSyncTriggerIsAllowed(deferredWorkspaceConfiguration, activeWorkspaceIncidentId, trigger)) {
4750
+ if (!workspaceSyncAdmissionSnapshotsMatch(admitted, captureWorkspaceSyncAdmission()) || !deferredWorkspaceSyncTriggerIsAllowed(deferredWorkspaceConfiguration, activeWorkspaceIncidentId, trigger)) {
4594
4751
  throw new WorkspaceSyncAdmissionChangedError("Workspace synchronization admission changed while asynchronous work was in flight");
4595
4752
  }
4596
4753
  };
@@ -4644,6 +4801,7 @@ async function startWorker(options) {
4644
4801
  bouncePending: unsafeWorkspaceSyncBouncePending,
4645
4802
  invalidateExecution: () => {
4646
4803
  workspaceConfigured = false;
4804
+ projectRuntime.configuration = void 0;
4647
4805
  advanceWorkerAdmissionGeneration();
4648
4806
  },
4649
4807
  scheduleRecoveryBounce: () => {
@@ -4672,7 +4830,11 @@ async function startWorker(options) {
4672
4830
  };
4673
4831
  }
4674
4832
  };
4675
- const REMEDIATION_TRIGGER_TYPES = /* @__PURE__ */ new Set(["remediation", "remediation_confirm", "remediation_reset"]);
4833
+ const REMEDIATION_TRIGGER_TYPES = /* @__PURE__ */ new Set([
4834
+ "remediation",
4835
+ "remediation_confirm",
4836
+ "remediation_reset"
4837
+ ]);
4676
4838
  const isOrdinaryWorkspaceSyncRequest = (input) => input.confirmedLargeDiff !== true && input.resetToCanonical !== true && !(input.requiredAncestorHeads && input.requiredAncestorHeads.length > 0) && !REMEDIATION_TRIGGER_TYPES.has(input.trigger.type);
4677
4839
  const LOCAL_ACTIVITY_TRIGGER_TYPES = /* @__PURE__ */ new Set([
4678
4840
  "write",
@@ -4713,7 +4875,14 @@ async function startWorker(options) {
4713
4875
  trigger: leader.trigger,
4714
4876
  assertStillAdmitted: () => assertWorkspaceSyncStillAdmitted(admission, leader.trigger)
4715
4877
  });
4716
- return { result, ranCycle: true, startedAtMs, finishedAtMs: Date.now(), prepareMs: lastWorkspaceSyncMirrorObservationMs, staleAttemptIds };
4878
+ return {
4879
+ result,
4880
+ ranCycle: true,
4881
+ startedAtMs,
4882
+ finishedAtMs: Date.now(),
4883
+ prepareMs: lastWorkspaceSyncMirrorObservationMs,
4884
+ staleAttemptIds
4885
+ };
4717
4886
  }
4718
4887
  });
4719
4888
  const runWorkspaceSync = async (input) => {
@@ -4743,7 +4912,7 @@ async function startWorker(options) {
4743
4912
  if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
4744
4913
  throw new Error("Cannot start autonomous workspace synchronization without an open worker control socket");
4745
4914
  }
4746
- ws.send(JSON.stringify({ type: "workspace_sync_started", attemptId }));
4915
+ sendSerializedWorkerMessage(ws, JSON.stringify({ type: "workspace_sync_started", attemptId }));
4747
4916
  }
4748
4917
  let queueMs = 0;
4749
4918
  let prepareMs = 0;
@@ -4795,6 +4964,7 @@ async function startWorker(options) {
4795
4964
  };
4796
4965
  const scheduleAutomaticWorkspaceSync = (trigger, delayMs = WORKSPACE_GIT_QUIET_MS) => {
4797
4966
  pendingAutomaticTrigger = trigger;
4967
+ projectRuntime.pendingAutomaticTrigger = trigger;
4798
4968
  const pendingBranchDeferral = () => (0, import_workspace_automatic_sync_policy.pendingCreatedBranchAutomaticSyncDeferral)(
4799
4969
  projectWorkspaceState.locallyPendingCreatedBranches,
4800
4970
  activeWorkspaceMutationTargets(),
@@ -4817,6 +4987,7 @@ async function startWorker(options) {
4817
4987
  return;
4818
4988
  }
4819
4989
  pendingAutomaticTrigger = void 0;
4990
+ projectRuntime.pendingAutomaticTrigger = void 0;
4820
4991
  automaticSyncInFlight = true;
4821
4992
  void runWorkspaceSync({ trigger: scheduledTrigger }).then((result) => {
4822
4993
  if (result.outcome === "failed") {
@@ -4840,9 +5011,14 @@ async function startWorker(options) {
4840
5011
  workspaceAutomaticTimer.unref();
4841
5012
  };
4842
5013
  const markWorkspaceDirty = (trigger, force = false, target) => {
5014
+ if (projectRuntime.markWorkspaceDirty && projectRuntime.markWorkspaceDirty !== markWorkspaceDirty) {
5015
+ projectRuntime.markWorkspaceDirty(trigger, force, target);
5016
+ return;
5017
+ }
4843
5018
  if (target) recordVisibleWorkspaceMutation(target);
4844
5019
  scheduleAutomaticWorkspaceSync(trigger, force ? 0 : WORKSPACE_GIT_QUIET_MS);
4845
5020
  };
5021
+ projectRuntime.markWorkspaceDirty = markWorkspaceDirty;
4846
5022
  const targetMayMutateVisibleWorkspace = (target) => target.type === "project" || target.rootProfile === "visible_projects";
4847
5023
  const resolveMessageTarget = (target) => {
4848
5024
  if (!workspaceConfigured) throw new Error("Worker workspace configuration has not completed successfully");
@@ -5145,6 +5321,7 @@ async function startWorker(options) {
5145
5321
  bootstrapCredentialGenerationFingerprintAtProcessStart = null;
5146
5322
  visibleGitIdentity = message.gitIdentity;
5147
5323
  workspaceGitIdentity = message.gitIdentity;
5324
+ projectRuntime.gitIdentity = message.gitIdentity;
5148
5325
  workspaceRemoteUrl = message.workspaceRemoteUrl;
5149
5326
  workspaceCredentialHelper = nextWorkspaceCredentialHelper;
5150
5327
  workspaceCredentialUsername = workerCredentialUsername;
@@ -5359,6 +5536,10 @@ async function startWorker(options) {
5359
5536
  Authorization: `Bearer ${token}`
5360
5537
  }
5361
5538
  });
5539
+ const connectionAttemptTimer = setTimeout(() => {
5540
+ if (ws.readyState === WebSocket.CONNECTING) ws.terminate();
5541
+ }, 3e4);
5542
+ connectionAttemptTimer.unref();
5362
5543
  let lastServerControlMessageAtMs = performance.now();
5363
5544
  let serverHeartbeatProbeStartedAtMs;
5364
5545
  let lastServerHeartbeatSweepAtMs = performance.now();
@@ -5454,7 +5635,9 @@ async function startWorker(options) {
5454
5635
  process.on("SIGTERM", handleGracefulShutdown);
5455
5636
  process.on("SIGINT", handleGracefulShutdown);
5456
5637
  ws.addEventListener("open", () => {
5638
+ clearTimeout(connectionAttemptTimer);
5457
5639
  currentWorkerSocket = ws;
5640
+ workerRecoveryId = "connecting";
5458
5641
  noteServerControlActivity();
5459
5642
  lastServerHeartbeatSweepAtMs = performance.now();
5460
5643
  serverHeartbeatWatchdog = setInterval(() => {
@@ -5498,6 +5681,9 @@ async function startWorker(options) {
5498
5681
  });
5499
5682
  const hello = {
5500
5683
  type: "hello",
5684
+ resumableProtocol: import_recovery_protocol.WORKER_RESUMABLE_PROTOCOL,
5685
+ runtimeId: workerRuntimeId,
5686
+ ledgerId: recoveryStore.ledgerId,
5501
5687
  startupReady: startupRecoveryComplete,
5502
5688
  hostInfo: {
5503
5689
  hostname: (0, import_node_os.hostname)(),
@@ -5522,15 +5708,17 @@ async function startWorker(options) {
5522
5708
  root: rootDir
5523
5709
  }
5524
5710
  };
5525
- ws.send(JSON.stringify(hello));
5526
- sendActiveProcessReport(ws);
5527
- for (const terminal of pendingProcessTerminals.values()) {
5528
- sendWorkerMessage(ws, terminal);
5529
- }
5530
- terminalReplayTimer = setInterval(() => {
5531
- for (const terminal of pendingProcessTerminals.values()) {
5532
- sendWorkerMessage(ws, terminal);
5711
+ sendSerializedWorkerMessage(ws, JSON.stringify(hello));
5712
+ outputReplayTimer = setInterval(() => {
5713
+ if (workerRecoveryId !== void 0 || currentWorkerSocket !== ws) return;
5714
+ for (const output of recoveryStore.output()) {
5715
+ if (!sendReplayWorkerMessage(ws, output)) break;
5533
5716
  }
5717
+ }, 250);
5718
+ outputReplayTimer.unref();
5719
+ terminalReplayTimer = setInterval(() => {
5720
+ if (workerRecoveryId !== void 0) return;
5721
+ for (const terminal of recoveryStore?.terminals() ?? []) sendReplayWorkerMessage(ws, terminal);
5534
5722
  }, 5e3);
5535
5723
  terminalReplayTimer.unref();
5536
5724
  process.stdout.write("[r5d-worker] connected\n");
@@ -5538,18 +5726,103 @@ async function startWorker(options) {
5538
5726
  ws.addEventListener("message", (event) => {
5539
5727
  noteServerControlActivity();
5540
5728
  void (async () => {
5729
+ if (currentWorkerSocket !== ws) return;
5541
5730
  const message = JSON.parse(String(event.data));
5731
+ if (message.type === "recovery_request") {
5732
+ workerRecoveryId = message.recoveryId;
5733
+ sendActiveProcessReport(ws);
5734
+ sendWorkerMessage(ws, {
5735
+ type: "recovery_state",
5736
+ recoveryId: message.recoveryId,
5737
+ runtimeId: workerRuntimeId,
5738
+ ledgerId: recoveryStore.ledgerId,
5739
+ operations: recoveryStore.operations()
5740
+ });
5741
+ const cursors = new Map(recoveryStore.outputCursors().map((cursor) => [cursor.runId, cursor]));
5742
+ for (const cursor of message.outputOffsets) cursors.set(cursor.runId, cursor);
5743
+ workerOutputWindow.reset([...cursors.values()]);
5744
+ for (const response of recoveryStore.responses(message.requests)) sendReplayWorkerMessage(ws, response);
5745
+ for (const output of recoveryStore.output(message.outputOffsets)) {
5746
+ while (true) {
5747
+ if (currentWorkerSocket !== ws || workerRecoveryId !== message.recoveryId || ws.readyState !== WebSocket.OPEN || workerLeaseExpired)
5748
+ return;
5749
+ if (sendReplayWorkerMessage(ws, output)) break;
5750
+ await Bun.sleep(10);
5751
+ }
5752
+ }
5753
+ if (currentWorkerSocket !== ws || workerRecoveryId !== message.recoveryId) return;
5754
+ for (const terminal of recoveryStore.terminals()) sendReplayWorkerMessage(ws, terminal);
5755
+ sendWorkerMessage(ws, { type: "recovery_complete", recoveryId: message.recoveryId });
5756
+ return;
5757
+ }
5758
+ if (message.type === "recovery_ready") {
5759
+ if (workerRecoveryId !== message.recoveryId) return;
5760
+ workerRecoveryId = void 0;
5761
+ workerReconnectAttempt = 0;
5762
+ renewWorkerExecutionLease(message.leaseMs);
5763
+ return;
5764
+ }
5765
+ if (message.type === "worker_execution_lease") {
5766
+ renewWorkerExecutionLease(message.leaseMs);
5767
+ return;
5768
+ }
5769
+ if (message.type === "operation_result_ack") {
5770
+ recoveryStore.acknowledgeResult(message.requestId);
5771
+ return;
5772
+ }
5773
+ if (message.type === "exec_output_ack") {
5774
+ recoveryStore.acknowledgeOutput(message.runId, message.stream, message.nextOffset);
5775
+ workerOutputWindow.acknowledge(message.runId, message.stream, message.nextOffset);
5776
+ return;
5777
+ }
5778
+ if (message.type === "cancel_session") {
5779
+ recoveryStore.cancelSession(message.sessionId);
5780
+ await Promise.all(
5781
+ [...activeProcesses].filter(([, active]) => active.sessionId === message.sessionId).map(async ([runId, active]) => {
5782
+ cancelledProcessRuns.add(runId);
5783
+ closeProcessStdin(active);
5784
+ await (0, import_process_tree.terminateProcessTree)(active.process);
5785
+ })
5786
+ );
5787
+ sendWorkerMessage(ws, { type: "cancel_session_ack", cancellationId: message.cancellationId });
5788
+ return;
5789
+ }
5542
5790
  if (!startupRecoveryComplete && message.type !== "ping") await startupRecoveryPromise;
5543
5791
  if (!startupRecoveryComplete && message.type !== "ping") return;
5792
+ if (currentWorkerSocket !== ws) return;
5793
+ const operationRequestId = "requestId" in message && typeof message.requestId === "string" && message.type !== "workspace_config" ? message.requestId : void 0;
5794
+ if (operationRequestId) {
5795
+ const admission = recoveryStore.admit({ ...message, requestId: operationRequestId });
5796
+ if (admission === "conflict") {
5797
+ ws.close(1008, "Operation identity conflict");
5798
+ return;
5799
+ }
5800
+ sendWorkerMessage(ws, { type: "operation_received", requestId: operationRequestId });
5801
+ if (admission !== "new") {
5802
+ const response = recoveryStore.response(operationRequestId);
5803
+ if (response && admission !== "unknown") sendReplayWorkerMessage(ws, response);
5804
+ return;
5805
+ }
5806
+ }
5544
5807
  const messageAdmissionGeneration = workerAdmissionGeneration;
5545
- const assertMessageAdmission = () => assertWorkerChildAdmission({
5546
- capturedGeneration: messageAdmissionGeneration,
5547
- currentGeneration: workerAdmissionGeneration,
5548
- sourceSocketIsCurrent: currentWorkerSocket === ws,
5549
- sourceSocketOpen: ws.readyState === WebSocket.OPEN,
5550
- workspaceConfigured,
5551
- runtimeHealthy: !shutdownAfterClose
5552
- });
5808
+ const admissionCredentials = configuredCredentialGenerationFingerprint;
5809
+ const assertMessageAdmission = () => {
5810
+ if (operationRequestId) {
5811
+ if (workerLeaseExpired || recoveryStore.isRequestCancelled(operationRequestId) || admissionCredentials !== configuredCredentialGenerationFingerprint || shutdownAfterClose || !workspaceConfigured) {
5812
+ recoveryStore.unknown(operationRequestId);
5813
+ throw new StaleWorkerAdmissionError();
5814
+ }
5815
+ return;
5816
+ }
5817
+ assertWorkerChildAdmission({
5818
+ capturedGeneration: messageAdmissionGeneration,
5819
+ currentGeneration: workerAdmissionGeneration,
5820
+ sourceSocketIsCurrent: currentWorkerSocket === ws,
5821
+ sourceSocketOpen: ws.readyState === WebSocket.OPEN,
5822
+ workspaceConfigured,
5823
+ runtimeHealthy: !shutdownAfterClose
5824
+ });
5825
+ };
5553
5826
  if (message.type === "connected") {
5554
5827
  return;
5555
5828
  }
@@ -5565,8 +5838,17 @@ async function startWorker(options) {
5565
5838
  if (message.type === "workspace_config") {
5566
5839
  const receiptGeneration = ++workspaceConfigurationReceiptGeneration;
5567
5840
  const refreshesDeferredConfiguration = deferredWorkspaceConfiguration !== null && activeWorkspaceIncidentId === null && message.deferWorkspaceSyncForIncidentId === void 0;
5568
- advanceWorkerAdmissionGeneration();
5569
- const configured = await configureWorkerWorkspace(message, receiptGeneration);
5841
+ const digest = (0, import_recovery_store.workerOperationDigest)(message);
5842
+ const cached = projectRuntime.configuration;
5843
+ if (cached?.digest !== digest) advanceWorkerAdmissionGeneration();
5844
+ const canReuseConfiguration = cached?.digest === digest && !unsafeWorkspaceSyncBouncePending && credentialGenerationFingerprint({ ...message, workerBaseUrl: baseUrl, workerAuthHeader: bearerAuthHeader }) === configuredCredentialGenerationFingerprint;
5845
+ const configured = canReuseConfiguration ? cached.configured : await configureWorkerWorkspace(message, receiptGeneration);
5846
+ if (canReuseConfiguration && configured.result.outcome !== "failed") {
5847
+ workspaceConfigured = true;
5848
+ activeWorkspaceIncidentId = cached.incidentId;
5849
+ deferredWorkspaceConfiguration = cached.deferred;
5850
+ }
5851
+ projectRuntime.configuration = configured.result.outcome !== "failed" ? { digest, configured, incidentId: activeWorkspaceIncidentId, deferred: deferredWorkspaceConfiguration } : void 0;
5570
5852
  unsafeWorkspaceSyncBouncePending = false;
5571
5853
  if (configured.result.outcome !== "failed") unsafeWorkspaceSyncRecoveryAttempts = 0;
5572
5854
  sendWorkerMessageFromCurrentSource(ws, {
@@ -5612,16 +5894,31 @@ async function startWorker(options) {
5612
5894
  return;
5613
5895
  }
5614
5896
  if (message.type === "sync_workspace") {
5615
- await runWorkspaceSync({
5616
- requestId: message.requestId,
5617
- attemptId: message.attemptId,
5618
- trigger: message.trigger,
5619
- confirmedLargeDiff: message.confirmedLargeDiff,
5620
- confirmationReason: message.confirmationReason,
5621
- resetToCanonical: message.resetToCanonical,
5622
- requiredAncestorHeads: message.requiredAncestorHeads
5623
- });
5624
- return;
5897
+ let retry = 0;
5898
+ let recoveryDeadline;
5899
+ while (true) {
5900
+ const result = await runWorkspaceSync({
5901
+ requestId: message.requestId,
5902
+ attemptId: message.attemptId,
5903
+ trigger: message.trigger,
5904
+ confirmedLargeDiff: message.confirmedLargeDiff,
5905
+ confirmationReason: message.confirmationReason,
5906
+ resetToCanonical: message.resetToCanonical,
5907
+ requiredAncestorHeads: message.requiredAncestorHeads,
5908
+ sendResult: false
5909
+ });
5910
+ if (result.outcome !== "failed" || !isWorkerCommunicationFailure(result.error ?? "")) {
5911
+ sendWorkspaceSyncResult(message.requestId, result);
5912
+ return;
5913
+ }
5914
+ recoveryDeadline ??= Date.now() + import_recovery_protocol.WORKER_RECOVERY_TIMEOUT_MS;
5915
+ if (workerLeaseExpired || Date.now() >= recoveryDeadline) {
5916
+ recoveryStore.unknown(message.requestId);
5917
+ return;
5918
+ }
5919
+ currentWorkerSocket?.close(1012, "Communication recovery");
5920
+ await Bun.sleep((0, import_supervisor.workerReconnectDelayMs)(Math.random, retry++));
5921
+ }
5625
5922
  }
5626
5923
  if (message.type === "create_project_branch") {
5627
5924
  if (workspaceOperationsAreFenced(deferredWorkspaceConfiguration, activeWorkspaceIncidentId)) {
@@ -5887,6 +6184,7 @@ async function startWorker(options) {
5887
6184
  }
5888
6185
  if (message.type === "exec_terminal_ack") {
5889
6186
  pendingProcessTerminals.delete(message.runId);
6187
+ recoveryStore.acknowledgeTerminal(message.runId);
5890
6188
  return;
5891
6189
  }
5892
6190
  if (message.type === "port_forward_connect") {
@@ -5938,7 +6236,7 @@ async function startWorker(options) {
5938
6236
  return;
5939
6237
  }
5940
6238
  if (message.type === "ping") {
5941
- ws.send(JSON.stringify({ type: "pong" }));
6239
+ sendSerializedWorkerMessage(ws, JSON.stringify({ type: "pong" }));
5942
6240
  return;
5943
6241
  }
5944
6242
  if (message.type === "cancel") {
@@ -5958,7 +6256,8 @@ async function startWorker(options) {
5958
6256
  cancelledProcessRuns.add(message.runId);
5959
6257
  cancelMessage = "Cancellation queued before command start";
5960
6258
  }
5961
- ws.send(
6259
+ sendSerializedWorkerMessage(
6260
+ ws,
5962
6261
  JSON.stringify({
5963
6262
  type: "cancel_result",
5964
6263
  requestId: message.requestId,
@@ -5970,7 +6269,8 @@ async function startWorker(options) {
5970
6269
  return;
5971
6270
  }
5972
6271
  if (message.type === "exec_stdin") {
5973
- const sendAck = (payload) => ws.send(
6272
+ const sendAck = (payload) => sendSerializedWorkerMessage(
6273
+ ws,
5974
6274
  JSON.stringify({
5975
6275
  type: "operation_result",
5976
6276
  requestId: message.requestId,
@@ -6093,16 +6393,19 @@ async function startWorker(options) {
6093
6393
  ptyId: message.ptyId,
6094
6394
  error: `Shell input is deferred for workspace incident ${deferredWorkspaceConfiguration?.incidentId ?? activeWorkspaceIncidentId ?? "unknown"}`
6095
6395
  });
6396
+ sendWorkerMessage(ws, { type: "operation_completed", requestId: message.requestId });
6096
6397
  return;
6097
6398
  }
6098
6399
  if (activePty && targetMayMutateVisibleWorkspace(activePty.target)) {
6099
6400
  markWorkspaceDirty({ type: "shell_inline", detail: `pty ${message.ptyId}` }, false, activePty.target);
6100
6401
  }
6101
6402
  writePty(ws, message);
6403
+ sendWorkerMessage(ws, { type: "operation_completed", requestId: message.requestId });
6102
6404
  return;
6103
6405
  }
6104
6406
  if (message.type === "pty_resize") {
6105
6407
  resizePty(message);
6408
+ sendWorkerMessage(ws, { type: "operation_completed", requestId: message.requestId });
6106
6409
  return;
6107
6410
  }
6108
6411
  if (message.type === "pty_close") {
@@ -6111,6 +6414,7 @@ async function startWorker(options) {
6111
6414
  if (activePty && targetMayMutateVisibleWorkspace(activePty.target)) {
6112
6415
  markWorkspaceDirty({ type: "process_terminal", detail: `pty ${message.ptyId} closed` }, true, activePty.target);
6113
6416
  }
6417
+ sendWorkerMessage(ws, { type: "operation_completed", requestId: message.requestId });
6114
6418
  return;
6115
6419
  }
6116
6420
  if (message.type === "exec") {
@@ -6192,7 +6496,11 @@ async function startWorker(options) {
6192
6496
  } finally {
6193
6497
  if (targetReserved) workspaceSyncPriorityProcessTargets.delete(message.runId);
6194
6498
  }
6195
- ws.send(JSON.stringify(result));
6499
+ if (infrastructureStoppedRunIds.has(message.runId)) {
6500
+ recoveryStore.unknown(message.requestId);
6501
+ return;
6502
+ }
6503
+ sendSerializedWorkerMessage(ws, JSON.stringify(result));
6196
6504
  if (hasWorkspaceEffect && targetMayMutateVisibleWorkspace(message.target)) {
6197
6505
  markWorkspaceDirty(
6198
6506
  { type: "shell_inline", sessionId: message.sessionId, processRunId: message.runId, detail: "foreground command completed" },
@@ -6285,7 +6593,7 @@ async function startWorker(options) {
6285
6593
  });
6286
6594
  return;
6287
6595
  }
6288
- if (message.type === "read" || message.type === "write" || message.type === "edit" || message.type === "grep" || message.type === "find" || message.type === "ls" || message.type === "view_file_bytes" || message.type === "code_list" || message.type === "code_read") {
6596
+ if (message.type === "read_plan" || message.type === "lint_plans" || message.type === "update_task_status" || message.type === "read" || message.type === "write" || message.type === "edit" || message.type === "grep" || message.type === "find" || message.type === "ls" || message.type === "view_file_bytes" || message.type === "code_list" || message.type === "code_read") {
6289
6597
  if (workspaceOperationsAreFenced(deferredWorkspaceConfiguration, activeWorkspaceIncidentId)) {
6290
6598
  sendWorkerMessage(ws, {
6291
6599
  type: "operation_result",
@@ -6295,7 +6603,7 @@ async function startWorker(options) {
6295
6603
  return;
6296
6604
  }
6297
6605
  const reservesVisibleWorkspace = targetMayMutateVisibleWorkspace(message.target);
6298
- const mutatesVisibleWorkspace = (message.type === "write" || message.type === "edit") && targetMayMutateVisibleWorkspace(message.target);
6606
+ const mutatesVisibleWorkspace = (message.type === "write" || message.type === "edit" || message.type === "update_task_status" || message.type === "read_plan" && message.createIfMissing) && targetMayMutateVisibleWorkspace(message.target);
6299
6607
  if (reservesVisibleWorkspace) {
6300
6608
  await (0, import_workspace_command_sync_policy.reserveWorkspaceCommandAfterCurrentSync)(message.target, workspaceSyncSingleFlight, () => {
6301
6609
  workspaceSyncPriorityOperationTargets.set(message.requestId, message.target);
@@ -6315,16 +6623,17 @@ async function startWorker(options) {
6315
6623
  assertAdmission: assertMessageAdmission
6316
6624
  });
6317
6625
  });
6318
- ws.send(
6626
+ sendSerializedWorkerMessage(
6627
+ ws,
6319
6628
  JSON.stringify({
6320
6629
  type: "operation_result",
6321
6630
  requestId: message.requestId,
6322
6631
  result
6323
6632
  })
6324
6633
  );
6325
- if (mutatesVisibleWorkspace) {
6634
+ if (mutatesVisibleWorkspace && (result.type !== "read_plan" || result.created)) {
6326
6635
  dirtyTrigger = {
6327
- type: message.type,
6636
+ type: message.type === "edit" || message.type === "update_task_status" ? "edit" : "write",
6328
6637
  sessionId: message.sessionId,
6329
6638
  toolCallId: message.requestId,
6330
6639
  ...message.target.type === "project" ? { projectId: message.target.projectId, branchName: message.target.branchName } : {}
@@ -6332,7 +6641,8 @@ async function startWorker(options) {
6332
6641
  markWorkspaceDirty(dirtyTrigger, false, message.target);
6333
6642
  }
6334
6643
  } catch (error) {
6335
- ws.send(
6644
+ sendSerializedWorkerMessage(
6645
+ ws,
6336
6646
  JSON.stringify({
6337
6647
  type: "operation_result",
6338
6648
  requestId: message.requestId,
@@ -6351,7 +6661,8 @@ async function startWorker(options) {
6351
6661
  }
6352
6662
  const unhandledMessage = message;
6353
6663
  if (typeof unhandledMessage.requestId === "string") {
6354
- ws.send(
6664
+ sendSerializedWorkerMessage(
6665
+ ws,
6355
6666
  JSON.stringify({
6356
6667
  type: "operation_result",
6357
6668
  requestId: unhandledMessage.requestId,
@@ -6365,6 +6676,7 @@ async function startWorker(options) {
6365
6676
  });
6366
6677
  });
6367
6678
  ws.addEventListener("close", (event) => {
6679
+ clearTimeout(connectionAttemptTimer);
6368
6680
  const processEvents = process;
6369
6681
  processEvents.off("SIGTERM", handleGracefulShutdown);
6370
6682
  processEvents.off("SIGINT", handleGracefulShutdown);
@@ -6381,15 +6693,20 @@ async function startWorker(options) {
6381
6693
  clearTimeout(deferredWorkspaceConfigurationRefreshTimer);
6382
6694
  deferredWorkspaceConfigurationRefreshTimer = void 0;
6383
6695
  }
6696
+ if (outputReplayTimer) {
6697
+ clearInterval(outputReplayTimer);
6698
+ outputReplayTimer = void 0;
6699
+ }
6384
6700
  if (terminalReplayTimer) {
6385
6701
  clearInterval(terminalReplayTimer);
6386
6702
  terminalReplayTimer = void 0;
6387
6703
  }
6388
- if (currentWorkerSocket === ws) {
6389
- advanceWorkerAdmissionGeneration();
6390
- currentWorkerSocket = null;
6704
+ if (projectRuntime.configuration) {
6705
+ projectRuntime.configuration.incidentId = activeWorkspaceIncidentId;
6706
+ projectRuntime.configuration.deferred = deferredWorkspaceConfiguration;
6391
6707
  }
6392
- closeAllPtys();
6708
+ if (currentWorkerSocket === ws) currentWorkerSocket = null;
6709
+ workerRecoveryId = "reconnecting";
6393
6710
  const reason = event.reason ? `: ${event.reason}` : "";
6394
6711
  process.stderr.write(`[r5d-worker] disconnected (${event.code}${reason})
6395
6712
  `);
@@ -6399,50 +6716,19 @@ async function startWorker(options) {
6399
6716
  if (shutdownAfterClose) {
6400
6717
  process.exit(0);
6401
6718
  }
6402
- if (!startupRecoveryComplete) {
6403
- process.stderr.write("[r5d-worker] control connection closed during workspace recovery; finishing recovery before reconnecting\n");
6404
- void startupRecoveryPromise.then(() => {
6405
- if (!startupRecoveryComplete) {
6406
- process.exit(1);
6407
- return;
6408
- }
6409
- const reconnectDelayMs = (0, import_supervisor.workerReconnectDelayMs)();
6410
- process.stderr.write(`[r5d-worker] workspace recovery complete; reconnecting in ${reconnectDelayMs}ms
6719
+ const reconnect = () => {
6720
+ const delay = (0, import_supervisor.workerReconnectDelayMs)(Math.random, workerReconnectAttempt++);
6721
+ process.stderr.write(`[r5d-worker] reconnecting in ${delay}ms
6411
6722
  `);
6412
- setTimeout(() => {
6413
- void startWorker(options).catch((error) => {
6414
- process.stderr.write(`[r5d-worker] reconnect failed: ${error instanceof Error ? error.message : String(error)}
6415
- `);
6416
- process.exit(error instanceof WorkerServerUnavailableError ? import_supervisor.WORKER_RECONNECT_EXIT_CODE : 1);
6417
- });
6418
- }, reconnectDelayMs);
6419
- });
6420
- return;
6421
- }
6422
- if (activeProcesses.size > 0 || credentialBearingProcessGroups.size > 0 || activePtys.size > 0 || pendingProcessTerminals.size > 0) {
6423
- const scheduleReconnect = (reconnect2) => {
6424
- const reconnectDelayMs = (0, import_supervisor.workerReconnectDelayMs)();
6425
- process.stderr.write(
6426
- `[r5d-worker] ${activeProcesses.size} process(es), ${credentialBearingProcessGroups.size} owned process group(s), and ${activePtys.size} PTY(s) active with ${pendingProcessTerminals.size} terminal report(s) pending; reconnecting in ${reconnectDelayMs}ms
6427
- `
6428
- );
6429
- setTimeout(reconnect2, reconnectDelayMs);
6430
- };
6431
- const reconnect = () => {
6432
- void startWorker(options).catch((error) => {
6723
+ setTimeout(() => {
6724
+ void startupRecoveryPromise.then(() => startWorker(options, projectRuntime)).catch((error) => {
6433
6725
  process.stderr.write(`[r5d-worker] reconnect failed: ${error instanceof Error ? error.message : String(error)}
6434
6726
  `);
6435
- if (activeProcesses.size > 0 || credentialBearingProcessGroups.size > 0 || activePtys.size > 0 || pendingProcessTerminals.size > 0) {
6436
- scheduleReconnect(reconnect);
6437
- return;
6438
- }
6439
- process.exit(error instanceof WorkerServerUnavailableError ? import_supervisor.WORKER_RECONNECT_EXIT_CODE : 1);
6727
+ reconnect();
6440
6728
  });
6441
- };
6442
- scheduleReconnect(reconnect);
6443
- return;
6444
- }
6445
- process.exit(import_supervisor.WORKER_RECONNECT_EXIT_CODE);
6729
+ }, delay);
6730
+ };
6731
+ reconnect();
6446
6732
  });
6447
6733
  ws.addEventListener("error", (event) => {
6448
6734
  const message = "message" in event && typeof event.message === "string" && event.message.trim() ? `: ${event.message.trim()}` : "";