@ricsam/r5d-worker 0.0.91 → 0.0.93

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -43,7 +43,7 @@ The supervisor also reconnects automatically after server rollouts and transient
43
43
 
44
44
  The runtime opens its control connection before recovering interrupted project snapshots. During recovery and initial project configuration it remains connected but non-dispatchable, reports its current phase plus branch/byte progress to the server, and prints the same progress locally. Recovery runs in a credential-sanitized helper process so the control loop can continue answering heartbeats. A durable `restoring` transaction state permits same-filesystem snapshots to be restored with one atomic directory move; cross-filesystem layouts retain the copy-and-fsync fallback.
45
45
 
46
- The server registry is the sole heartbeat timeout authority. The worker answers server pings but does not run an independent connection watchdog, because synchronous bounded workspace operations can temporarily occupy its event loop. The server keeps those operations leased for their bounded lifetime and closes genuinely stale connections after the lease ends.
46
+ Both sides use monotonic, two-stage heartbeat recovery. The server sends pings, grants bounded leases to synchronous workspace and command-preparation work, and requires a fresh missed probe window before force-evicting a stale socket. The worker answers pings and uses a deliberately slower backup watchdog only for a half-open connection; reconnect delays are jittered across workers. A delayed watchdog sweep renews the probe instead of disconnecting, so synchronous local work or a stalled server event loop cannot turn one overdue timer callback into a mass disconnect.
47
47
 
48
48
  Credential-generation restart fencing protects against stale or accidentally detached descendants. It is not a hostile same-UID sandbox: code running as the worker's OS user can read or tamper with that user's config and auth files, install a persistent user service, or return after reboot. Run untrusted agents in a disposable VM that is reset at the boundary, or use a separately privileged credential broker/service account. When rotating the worker's own local `r5dctl` credential, stop the worker and reboot/reset the host before writing the replacement credential so an old same-UID descendant cannot read the new config.
49
49
 
package/dist/cjs/main.cjs CHANGED
@@ -50,6 +50,7 @@ __export(main_exports, {
50
50
  syncSessionArtifacts: () => syncSessionArtifacts,
51
51
  workerChildProcessEnvironment: () => workerChildProcessEnvironment,
52
52
  workerGitSecurityTestHarness: () => workerGitSecurityTestHarness,
53
+ workerLifecycleTestHarness: () => workerLifecycleTestHarness,
53
54
  workerPtyBridgeTestHarness: () => workerPtyBridgeTestHarness,
54
55
  workerPtyTestHarness: () => workerPtyTestHarness,
55
56
  writeWorkerTextFile: () => writeWorkerTextFile
@@ -84,6 +85,21 @@ var import_project_snapshot_recovery_runner = require("./project-snapshot-recove
84
85
  var import_project_workspace_state = require("./project-workspace-state.cjs");
85
86
  var import_workspace_git_sync = require("./workspace-git-sync.cjs");
86
87
  const import_meta = {};
88
+ const WORKER_LIFECYCLE_DETAIL_MAX_LENGTH = 500;
89
+ const WORKER_SERVER_HEARTBEAT_INTERVAL_MS = 15e3;
90
+ const WORKER_SERVER_HEARTBEAT_TIMEOUT_MS = 2 * 60 * 1e3;
91
+ function boundedWorkerLifecycleProgress(progress) {
92
+ return progress.detail === void 0 ? progress : { ...progress, detail: progress.detail.slice(0, WORKER_LIFECYCLE_DETAIL_MAX_LENGTH) };
93
+ }
94
+ function workerServerHeartbeatAction(input) {
95
+ if (input.elapsedMs < input.timeoutMs) return "wait";
96
+ if (input.watchdogSweepDelayed || input.probeElapsedMs === void 0) return "start_probe";
97
+ return input.probeElapsedMs >= input.timeoutMs ? "terminate" : "wait";
98
+ }
99
+ const workerLifecycleTestHarness = {
100
+ boundedWorkerLifecycleProgress,
101
+ workerServerHeartbeatAction
102
+ };
87
103
  class ProjectWorkspaceConfigurationDeferredError extends Error {
88
104
  }
89
105
  class WorkerServerUnavailableError extends Error {
@@ -2597,6 +2613,13 @@ async function executeCommand(input) {
2597
2613
  const startedAt = Date.now();
2598
2614
  let timeout;
2599
2615
  let spawnedProcess;
2616
+ let preparationSettled = false;
2617
+ const settlePreparation = () => {
2618
+ if (preparationSettled) return;
2619
+ preparationSettled = true;
2620
+ input.setPreparationHeartbeatLease(false);
2621
+ };
2622
+ input.setPreparationHeartbeatLease(true);
2600
2623
  try {
2601
2624
  const targetProcessEnv = await prepareShellEnvForTarget({
2602
2625
  target: input.resolvedTarget.target,
@@ -2634,6 +2657,7 @@ async function executeCommand(input) {
2634
2657
  startedAt: (/* @__PURE__ */ new Date()).toISOString(),
2635
2658
  ...input.message.workspaceEffect === "none" ? { workspaceEffect: "none" } : {}
2636
2659
  });
2660
+ settlePreparation();
2637
2661
  if (input.message.timeoutMs) {
2638
2662
  timeout = setTimeout(() => {
2639
2663
  void (0, import_process_tree.terminateProcessTree)(subprocess).catch((error) => {
@@ -2669,6 +2693,7 @@ async function executeCommand(input) {
2669
2693
  error: error instanceof Error ? error.message : String(error)
2670
2694
  };
2671
2695
  } finally {
2696
+ settlePreparation();
2672
2697
  if (timeout) {
2673
2698
  clearTimeout(timeout);
2674
2699
  }
@@ -2684,6 +2709,13 @@ async function executeStreamingCommand(input) {
2684
2709
  let timedOut = false;
2685
2710
  let timeout;
2686
2711
  let spawnedProcess;
2712
+ let preparationSettled = false;
2713
+ const settlePreparation = () => {
2714
+ if (preparationSettled) return;
2715
+ preparationSettled = true;
2716
+ input.setPreparationHeartbeatLease(false);
2717
+ };
2718
+ input.setPreparationHeartbeatLease(true);
2687
2719
  try {
2688
2720
  if (cancelledProcessRuns.delete(input.message.runId)) {
2689
2721
  sendWorkerMessage(input.ws, {
@@ -2736,6 +2768,7 @@ async function executeStreamingCommand(input) {
2736
2768
  ...interactive ? { interactive: true, stdin: subprocess.stdin } : {},
2737
2769
  ...input.message.workspaceEffect === "none" ? { workspaceEffect: "none" } : {}
2738
2770
  });
2771
+ settlePreparation();
2739
2772
  started = true;
2740
2773
  sendWorkerMessage(input.ws, {
2741
2774
  type: "exec_started",
@@ -2812,6 +2845,7 @@ async function executeStreamingCommand(input) {
2812
2845
  });
2813
2846
  }
2814
2847
  } finally {
2848
+ settlePreparation();
2815
2849
  if (timeout) {
2816
2850
  clearTimeout(timeout);
2817
2851
  }
@@ -3153,7 +3187,8 @@ function createNodePtyBridge(options) {
3153
3187
  };
3154
3188
  }
3155
3189
  const workerPtyBridgeTestHarness = {
3156
- create: createNodePtyBridge
3190
+ create: createNodePtyBridge,
3191
+ closeAllBestEffort: closePtysBestEffort
3157
3192
  };
3158
3193
  function resolveHostShell(command, platform = process.platform) {
3159
3194
  if (platform === "win32") {
@@ -3302,11 +3337,23 @@ function closePty(message) {
3302
3337
  }
3303
3338
  ptyProcess.kill();
3304
3339
  }
3305
- function closeAllPtys() {
3306
- for (const ptyProcess of activePtys.values()) {
3307
- ptyProcess.kill();
3340
+ function closePtysBestEffort(ptys, reportFailure) {
3341
+ for (const [ptyId, ptyProcess] of ptys) {
3342
+ try {
3343
+ ptyProcess.kill();
3344
+ } catch (error) {
3345
+ reportFailure(ptyId, error);
3346
+ }
3308
3347
  }
3309
3348
  }
3349
+ function closeAllPtys() {
3350
+ closePtysBestEffort(activePtys, (ptyId, error) => {
3351
+ process.stderr.write(
3352
+ `[r5d-worker] failed to close PTY ${ptyId} after control disconnect: ${error instanceof Error ? error.message : String(error)}
3353
+ `
3354
+ );
3355
+ });
3356
+ }
3310
3357
  async function terminateCredentialBearingChildren(children) {
3311
3358
  const results = await Promise.allSettled(children.map(async ({ terminate }) => await terminate()));
3312
3359
  const failures = results.flatMap(
@@ -3366,16 +3413,6 @@ function websocketUrl(baseUrl, label) {
3366
3413
  }
3367
3414
  async function startWorker(options) {
3368
3415
  advanceWorkerAdmissionGeneration();
3369
- process.on("uncaughtException", (error) => {
3370
- process.stderr.write(`[r5d-worker] uncaught exception: ${error instanceof Error ? error.stack ?? error.message : String(error)}
3371
- `);
3372
- });
3373
- process.on("unhandledRejection", (reason) => {
3374
- process.stderr.write(
3375
- `[r5d-worker] unhandled rejection: ${reason instanceof Error ? reason.stack ?? reason.message : String(reason)}
3376
- `
3377
- );
3378
- });
3379
3416
  const config = readConfig(options.configPath);
3380
3417
  const { baseUrl, token } = resolveCredentials(options, config);
3381
3418
  initializeCredentialStoreDirectoryOnce();
@@ -4388,11 +4425,12 @@ async function startWorker(options) {
4388
4425
  }
4389
4426
  try {
4390
4427
  assertStillAdmitted();
4428
+ const hasDurableBranchDeletion = projectWorkspaceState.tombstones.some((tombstone) => tombstone.kind === "branch");
4391
4429
  return await performWorkspaceSync({
4392
4430
  attemptId,
4393
4431
  trigger: input.trigger,
4394
- confirmedLargeDiff: input.confirmedLargeDiff,
4395
- confirmationReason: input.confirmationReason,
4432
+ confirmedLargeDiff: input.confirmedLargeDiff === true || hasDurableBranchDeletion,
4433
+ confirmationReason: input.confirmationReason ?? (hasDurableBranchDeletion ? "durable project branch deletion" : void 0),
4396
4434
  resetToCanonical: input.resetToCanonical,
4397
4435
  requiredAncestorHeads: input.requiredAncestorHeads,
4398
4436
  assertStillAdmitted
@@ -4929,9 +4967,12 @@ async function startWorker(options) {
4929
4967
  );
4930
4968
  }
4931
4969
  };
4970
+ const hasDurableBranchDeletion = projectWorkspaceState.tombstones.some((tombstone) => tombstone.kind === "branch");
4932
4971
  const result = await performWorkspaceSync({
4933
4972
  attemptId: crypto.randomUUID(),
4934
4973
  trigger: { type: "connect" },
4974
+ confirmedLargeDiff: hasDurableBranchDeletion,
4975
+ confirmationReason: hasDurableBranchDeletion ? "durable project branch deletion" : void 0,
4935
4976
  resetToCanonical: workspaceConfigurationResetToCanonicalIsAllowed(message.resetToCanonical, incidentDeferral.incidentId),
4936
4977
  assertStillAdmitted: assertConfigurationSyncStillAdmitted
4937
4978
  });
@@ -5019,9 +5060,22 @@ async function startWorker(options) {
5019
5060
  Authorization: `Bearer ${token}`
5020
5061
  }
5021
5062
  });
5063
+ let lastServerControlMessageAtMs = performance.now();
5064
+ let serverHeartbeatProbeStartedAtMs;
5065
+ let lastServerHeartbeatSweepAtMs = performance.now();
5066
+ let serverHeartbeatWatchdog;
5067
+ const noteServerControlActivity = () => {
5068
+ lastServerControlMessageAtMs = performance.now();
5069
+ serverHeartbeatProbeStartedAtMs = void 0;
5070
+ };
5071
+ const stopServerHeartbeatWatchdog = () => {
5072
+ if (!serverHeartbeatWatchdog) return;
5073
+ clearInterval(serverHeartbeatWatchdog);
5074
+ serverHeartbeatWatchdog = void 0;
5075
+ };
5022
5076
  const sendLifecycleProgress = (progress) => {
5023
5077
  if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) return;
5024
- sendWorkerMessage(ws, { type: "worker_progress", progress });
5078
+ sendWorkerMessage(ws, { type: "worker_progress", progress: boundedWorkerLifecycleProgress(progress) });
5025
5079
  };
5026
5080
  const describeSnapshotProgress = (progress) => {
5027
5081
  const project = import_node_path.default.relative(projectsRoot, progress.projectRoot) || import_node_path.default.basename(progress.projectRoot);
@@ -5102,6 +5156,29 @@ async function startWorker(options) {
5102
5156
  process.on("SIGINT", handleGracefulShutdown);
5103
5157
  ws.addEventListener("open", () => {
5104
5158
  currentWorkerSocket = ws;
5159
+ noteServerControlActivity();
5160
+ lastServerHeartbeatSweepAtMs = performance.now();
5161
+ serverHeartbeatWatchdog = setInterval(() => {
5162
+ if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) return;
5163
+ const nowMs = performance.now();
5164
+ const heartbeatAction = workerServerHeartbeatAction({
5165
+ elapsedMs: Math.max(0, nowMs - lastServerControlMessageAtMs),
5166
+ ...serverHeartbeatProbeStartedAtMs === void 0 ? {} : { probeElapsedMs: Math.max(0, nowMs - serverHeartbeatProbeStartedAtMs) },
5167
+ timeoutMs: WORKER_SERVER_HEARTBEAT_TIMEOUT_MS,
5168
+ watchdogSweepDelayed: nowMs - lastServerHeartbeatSweepAtMs >= WORKER_SERVER_HEARTBEAT_INTERVAL_MS * 2
5169
+ });
5170
+ lastServerHeartbeatSweepAtMs = nowMs;
5171
+ if (heartbeatAction === "start_probe") {
5172
+ serverHeartbeatProbeStartedAtMs = nowMs;
5173
+ return;
5174
+ }
5175
+ if (heartbeatAction === "terminate") {
5176
+ stopServerHeartbeatWatchdog();
5177
+ process.stderr.write("[r5d-worker] server control heartbeat timed out; reconnecting\n");
5178
+ ws.terminate();
5179
+ }
5180
+ }, WORKER_SERVER_HEARTBEAT_INTERVAL_MS);
5181
+ serverHeartbeatWatchdog.unref();
5105
5182
  const startupOperationId = crypto.randomUUID();
5106
5183
  startupRecoveryPromise = Promise.resolve().then(async () => await performStartupRecovery(startupOperationId)).then(() => {
5107
5184
  startupRecoveryComplete = true;
@@ -5137,7 +5214,8 @@ async function startWorker(options) {
5137
5214
  ptyEnvFilesV1: true,
5138
5215
  workspaceRemediationAncestorGuardV1: true,
5139
5216
  workspaceIncidentConfigDeferralV1: true,
5140
- workspaceConfigResetToCanonicalV1: true
5217
+ workspaceConfigResetToCanonicalV1: true,
5218
+ projectBranchDeletionFastAckV1: true
5141
5219
  },
5142
5220
  projectRoot: projectsRoot,
5143
5221
  artifactRoot,
@@ -5158,6 +5236,7 @@ async function startWorker(options) {
5158
5236
  process.stdout.write("[r5d-worker] connected\n");
5159
5237
  });
5160
5238
  ws.addEventListener("message", (event) => {
5239
+ noteServerControlActivity();
5161
5240
  void (async () => {
5162
5241
  const message = JSON.parse(String(event.data));
5163
5242
  if (!startupRecoveryComplete && message.type !== "ping") await startupRecoveryPromise;
@@ -5390,23 +5469,15 @@ async function startWorker(options) {
5390
5469
  return true;
5391
5470
  });
5392
5471
  if (deletionNeedsSync) {
5393
- const result = await runWorkspaceSync({
5394
- trigger: {
5472
+ scheduleAutomaticWorkspaceSync(
5473
+ {
5395
5474
  type: "manual",
5396
5475
  projectId: message.projectId,
5397
5476
  branchName: message.branchName,
5398
5477
  detail: "deleted project branch"
5399
5478
  },
5400
- confirmedLargeDiff: true,
5401
- confirmationReason: "explicit project branch deletion",
5402
- sendResult: false
5403
- });
5404
- if (result.outcome === "failed" || result.outcome === "conflict_blocked" || result.outcome === "large_diff_blocked") {
5405
- throw new Error(result.error ?? `Project branch deletion workspace sync ended with ${result.outcome}`);
5406
- }
5407
- if (pendingMirrorDeletes.has(deletionKey)) {
5408
- throw new Error("Project branch deletion did not remove the hidden mirror ref");
5409
- }
5479
+ 0
5480
+ );
5410
5481
  }
5411
5482
  sendWorkerMessage(ws, {
5412
5483
  type: "operation_result",
@@ -5765,6 +5836,13 @@ async function startWorker(options) {
5765
5836
  token,
5766
5837
  artifactRoot,
5767
5838
  planRoot,
5839
+ setPreparationHeartbeatLease: (active) => {
5840
+ sendWorkerMessageFromCurrentSource(ws, {
5841
+ type: "heartbeat_lease",
5842
+ operationId: `exec:${message.runId}`,
5843
+ active
5844
+ });
5845
+ },
5768
5846
  assertAdmission: () => {
5769
5847
  if (!workspaceCommandTransportIsAllowed(deferredWorkspaceConfiguration, activeWorkspaceIncidentId, "exec")) {
5770
5848
  throw new Error(
@@ -5828,6 +5906,13 @@ async function startWorker(options) {
5828
5906
  token,
5829
5907
  artifactRoot,
5830
5908
  planRoot,
5909
+ setPreparationHeartbeatLease: (active) => {
5910
+ sendWorkerMessageFromCurrentSource(ws, {
5911
+ type: "heartbeat_lease",
5912
+ operationId: `exec:${message.runId}`,
5913
+ active
5914
+ });
5915
+ },
5831
5916
  assertAdmission: assertMessageAdmission
5832
5917
  });
5833
5918
  } finally {
@@ -5960,6 +6045,7 @@ async function startWorker(options) {
5960
6045
  const processEvents = process;
5961
6046
  processEvents.off("SIGTERM", handleGracefulShutdown);
5962
6047
  processEvents.off("SIGINT", handleGracefulShutdown);
6048
+ stopServerHeartbeatWatchdog();
5963
6049
  if (workspaceAutomaticTimer) {
5964
6050
  clearTimeout(workspaceAutomaticTimer);
5965
6051
  workspaceAutomaticTimer = void 0;
@@ -5997,31 +6083,40 @@ async function startWorker(options) {
5997
6083
  process.exit(1);
5998
6084
  return;
5999
6085
  }
6000
- void startWorker(options).catch((error) => {
6001
- process.stderr.write(`[r5d-worker] reconnect failed: ${error instanceof Error ? error.message : String(error)}
6086
+ const reconnectDelayMs = (0, import_supervisor.workerReconnectDelayMs)();
6087
+ process.stderr.write(`[r5d-worker] workspace recovery complete; reconnecting in ${reconnectDelayMs}ms
6002
6088
  `);
6003
- process.exit(error instanceof WorkerServerUnavailableError ? import_supervisor.WORKER_RECONNECT_EXIT_CODE : 1);
6004
- });
6089
+ setTimeout(() => {
6090
+ void startWorker(options).catch((error) => {
6091
+ process.stderr.write(`[r5d-worker] reconnect failed: ${error instanceof Error ? error.message : String(error)}
6092
+ `);
6093
+ process.exit(error instanceof WorkerServerUnavailableError ? import_supervisor.WORKER_RECONNECT_EXIT_CODE : 1);
6094
+ });
6095
+ }, reconnectDelayMs);
6005
6096
  });
6006
6097
  return;
6007
6098
  }
6008
6099
  if (activeProcesses.size > 0 || credentialBearingProcessGroups.size > 0 || activePtys.size > 0 || pendingProcessTerminals.size > 0) {
6009
- process.stderr.write(
6010
- `[r5d-worker] ${activeProcesses.size} process(es), ${credentialBearingProcessGroups.size} owned process group(s), and ${activePtys.size} PTY(s) active with ${pendingProcessTerminals.size} terminal report(s) pending; reconnecting in ${import_supervisor.WORKER_RECONNECT_DELAY_MS}ms
6100
+ const scheduleReconnect = (reconnect2) => {
6101
+ const reconnectDelayMs = (0, import_supervisor.workerReconnectDelayMs)();
6102
+ process.stderr.write(
6103
+ `[r5d-worker] ${activeProcesses.size} process(es), ${credentialBearingProcessGroups.size} owned process group(s), and ${activePtys.size} PTY(s) active with ${pendingProcessTerminals.size} terminal report(s) pending; reconnecting in ${reconnectDelayMs}ms
6011
6104
  `
6012
- );
6105
+ );
6106
+ setTimeout(reconnect2, reconnectDelayMs);
6107
+ };
6013
6108
  const reconnect = () => {
6014
6109
  void startWorker(options).catch((error) => {
6015
6110
  process.stderr.write(`[r5d-worker] reconnect failed: ${error instanceof Error ? error.message : String(error)}
6016
6111
  `);
6017
6112
  if (activeProcesses.size > 0 || credentialBearingProcessGroups.size > 0 || activePtys.size > 0 || pendingProcessTerminals.size > 0) {
6018
- setTimeout(reconnect, import_supervisor.WORKER_RECONNECT_DELAY_MS);
6113
+ scheduleReconnect(reconnect);
6019
6114
  return;
6020
6115
  }
6021
6116
  process.exit(error instanceof WorkerServerUnavailableError ? import_supervisor.WORKER_RECONNECT_EXIT_CODE : 1);
6022
6117
  });
6023
6118
  };
6024
- setTimeout(reconnect, import_supervisor.WORKER_RECONNECT_DELAY_MS);
6119
+ scheduleReconnect(reconnect);
6025
6120
  return;
6026
6121
  }
6027
6122
  process.exit(import_supervisor.WORKER_RECONNECT_EXIT_CODE);
@@ -6031,7 +6126,20 @@ async function startWorker(options) {
6031
6126
  process.stderr.write(`[r5d-worker] websocket error${message}
6032
6127
  `);
6033
6128
  });
6034
- await new Promise(() => {
6129
+ await new Promise((resolve) => {
6130
+ ws.addEventListener("close", () => resolve(), { once: true });
6131
+ });
6132
+ }
6133
+ function installWorkerRuntimeFailureLogging() {
6134
+ process.on("uncaughtException", (error) => {
6135
+ process.stderr.write(`[r5d-worker] uncaught exception: ${error instanceof Error ? error.stack ?? error.message : String(error)}
6136
+ `);
6137
+ });
6138
+ process.on("unhandledRejection", (reason) => {
6139
+ process.stderr.write(
6140
+ `[r5d-worker] unhandled rejection: ${reason instanceof Error ? reason.stack ?? reason.message : String(reason)}
6141
+ `
6142
+ );
6035
6143
  });
6036
6144
  }
6037
6145
  async function main() {
@@ -6051,6 +6159,7 @@ async function main() {
6051
6159
  }
6052
6160
  if (process.env[import_supervisor.WORKER_RUNTIME_ENV] === "1") {
6053
6161
  delete process.env[import_supervisor.WORKER_RUNTIME_ENV];
6162
+ installWorkerRuntimeFailureLogging();
6054
6163
  try {
6055
6164
  await startWorker(parsed.options);
6056
6165
  } catch (error) {
@@ -6104,6 +6213,7 @@ if (isCliEntrypoint()) {
6104
6213
  syncSessionArtifacts,
6105
6214
  workerChildProcessEnvironment,
6106
6215
  workerGitSecurityTestHarness,
6216
+ workerLifecycleTestHarness,
6107
6217
  workerPtyBridgeTestHarness,
6108
6218
  workerPtyTestHarness,
6109
6219
  writeWorkerTextFile
@@ -1,5 +1,5 @@
1
1
  {
2
2
  "name": "@ricsam/r5d-worker",
3
- "version": "0.0.91",
3
+ "version": "0.0.93",
4
4
  "type": "commonjs"
5
5
  }
@@ -21,10 +21,12 @@ __export(supervisor_exports, {
21
21
  WORKER_CREDENTIAL_RESTART_EXIT_CODE: () => WORKER_CREDENTIAL_RESTART_EXIT_CODE,
22
22
  WORKER_RECONNECT_DELAY_MS: () => WORKER_RECONNECT_DELAY_MS,
23
23
  WORKER_RECONNECT_EXIT_CODE: () => WORKER_RECONNECT_EXIT_CODE,
24
+ WORKER_RECONNECT_JITTER_MS: () => WORKER_RECONNECT_JITTER_MS,
24
25
  WORKER_RELOAD_EXIT_CODE: () => WORKER_RELOAD_EXIT_CODE,
25
26
  WORKER_RUNTIME_ENV: () => WORKER_RUNTIME_ENV,
26
27
  isRetryableWorkerServerStatus: () => isRetryableWorkerServerStatus,
27
- superviseWorkerRuntime: () => superviseWorkerRuntime
28
+ superviseWorkerRuntime: () => superviseWorkerRuntime,
29
+ workerReconnectDelayMs: () => workerReconnectDelayMs
28
30
  });
29
31
  module.exports = __toCommonJS(supervisor_exports);
30
32
  var import_node_os = require("node:os");
@@ -34,6 +36,7 @@ const WORKER_RELOAD_EXIT_CODE = 75;
34
36
  const WORKER_RECONNECT_EXIT_CODE = 76;
35
37
  const WORKER_CREDENTIAL_RESTART_EXIT_CODE = 77;
36
38
  const WORKER_RECONNECT_DELAY_MS = 2e3;
39
+ const WORKER_RECONNECT_JITTER_MS = 3e3;
37
40
  const SUPERVISOR_SIGNALS = ["SIGINT", "SIGTERM", "SIGHUP"];
38
41
  function defaultRuntimeSpawner(argv, options) {
39
42
  return Bun.spawn(argv, options);
@@ -41,6 +44,10 @@ function defaultRuntimeSpawner(argv, options) {
41
44
  function defaultRuntimeDelay(delayMs) {
42
45
  return new Promise((resolve) => setTimeout(resolve, delayMs));
43
46
  }
47
+ function workerReconnectDelayMs(random = Math.random) {
48
+ const sample = Math.max(0, Math.min(1, random()));
49
+ return WORKER_RECONNECT_DELAY_MS + Math.floor(sample * WORKER_RECONNECT_JITTER_MS);
50
+ }
44
51
  async function defaultRuntimeReaper(runtime) {
45
52
  await (0, import_process_tree.terminateProcessTree)(runtime);
46
53
  }
@@ -53,7 +60,7 @@ function signalExitCode(signal) {
53
60
  function osSignalNumber(signal) {
54
61
  return import_node_os.constants.signals[signal];
55
62
  }
56
- async function superviseWorkerRuntime(argv, env = process.env, spawnRuntime = defaultRuntimeSpawner, waitBeforeReconnect = defaultRuntimeDelay, reapCredentialRuntime = defaultRuntimeReaper) {
63
+ async function superviseWorkerRuntime(argv, env = process.env, spawnRuntime = defaultRuntimeSpawner, waitBeforeReconnect = defaultRuntimeDelay, reapCredentialRuntime = defaultRuntimeReaper, reconnectDelay = workerReconnectDelayMs) {
57
64
  if (!process.argv[1]) {
58
65
  throw new Error("Cannot locate the r5d-worker entrypoint");
59
66
  }
@@ -105,9 +112,10 @@ async function superviseWorkerRuntime(argv, env = process.env, spawnRuntime = de
105
112
  continue;
106
113
  }
107
114
  if (exitCode === WORKER_RECONNECT_EXIT_CODE) {
108
- process.stderr.write(`[r5d-worker] reconnecting in ${WORKER_RECONNECT_DELAY_MS}ms
115
+ const reconnectDelayMs = reconnectDelay();
116
+ process.stderr.write(`[r5d-worker] reconnecting in ${reconnectDelayMs}ms
109
117
  `);
110
- await waitBeforeReconnect(WORKER_RECONNECT_DELAY_MS);
118
+ await waitBeforeReconnect(reconnectDelayMs);
111
119
  if (receivedSignal) return signalExitCode(receivedSignal);
112
120
  continue;
113
121
  }
@@ -123,8 +131,10 @@ async function superviseWorkerRuntime(argv, env = process.env, spawnRuntime = de
123
131
  WORKER_CREDENTIAL_RESTART_EXIT_CODE,
124
132
  WORKER_RECONNECT_DELAY_MS,
125
133
  WORKER_RECONNECT_EXIT_CODE,
134
+ WORKER_RECONNECT_JITTER_MS,
126
135
  WORKER_RELOAD_EXIT_CODE,
127
136
  WORKER_RUNTIME_ENV,
128
137
  isRetryableWorkerServerStatus,
129
- superviseWorkerRuntime
138
+ superviseWorkerRuntime,
139
+ workerReconnectDelayMs
130
140
  });
package/dist/mjs/main.mjs CHANGED
@@ -49,10 +49,10 @@ import {
49
49
  isRetryableWorkerServerStatus,
50
50
  superviseWorkerRuntime,
51
51
  WORKER_CREDENTIAL_RESTART_EXIT_CODE,
52
- WORKER_RECONNECT_DELAY_MS,
53
52
  WORKER_RECONNECT_EXIT_CODE,
54
53
  WORKER_RELOAD_EXIT_CODE,
55
- WORKER_RUNTIME_ENV
54
+ WORKER_RUNTIME_ENV,
55
+ workerReconnectDelayMs
56
56
  } from "./supervisor.mjs";
57
57
  import { assertDisjointManagedRoots, managedProjectRoot, validateManagedBranchName } from "./managed-paths.mjs";
58
58
  import {
@@ -91,6 +91,21 @@ import {
91
91
  workspaceGitMountsForRemediation,
92
92
  WorkspaceRemediationAncestryError
93
93
  } from "./workspace-git-sync.mjs";
94
+ const WORKER_LIFECYCLE_DETAIL_MAX_LENGTH = 500;
95
+ const WORKER_SERVER_HEARTBEAT_INTERVAL_MS = 15e3;
96
+ const WORKER_SERVER_HEARTBEAT_TIMEOUT_MS = 2 * 60 * 1e3;
97
+ function boundedWorkerLifecycleProgress(progress) {
98
+ return progress.detail === void 0 ? progress : { ...progress, detail: progress.detail.slice(0, WORKER_LIFECYCLE_DETAIL_MAX_LENGTH) };
99
+ }
100
+ function workerServerHeartbeatAction(input) {
101
+ if (input.elapsedMs < input.timeoutMs) return "wait";
102
+ if (input.watchdogSweepDelayed || input.probeElapsedMs === void 0) return "start_probe";
103
+ return input.probeElapsedMs >= input.timeoutMs ? "terminate" : "wait";
104
+ }
105
+ const workerLifecycleTestHarness = {
106
+ boundedWorkerLifecycleProgress,
107
+ workerServerHeartbeatAction
108
+ };
94
109
  class ProjectWorkspaceConfigurationDeferredError extends Error {
95
110
  }
96
111
  class WorkerServerUnavailableError extends Error {
@@ -2604,6 +2619,13 @@ async function executeCommand(input) {
2604
2619
  const startedAt = Date.now();
2605
2620
  let timeout;
2606
2621
  let spawnedProcess;
2622
+ let preparationSettled = false;
2623
+ const settlePreparation = () => {
2624
+ if (preparationSettled) return;
2625
+ preparationSettled = true;
2626
+ input.setPreparationHeartbeatLease(false);
2627
+ };
2628
+ input.setPreparationHeartbeatLease(true);
2607
2629
  try {
2608
2630
  const targetProcessEnv = await prepareShellEnvForTarget({
2609
2631
  target: input.resolvedTarget.target,
@@ -2641,6 +2663,7 @@ async function executeCommand(input) {
2641
2663
  startedAt: (/* @__PURE__ */ new Date()).toISOString(),
2642
2664
  ...input.message.workspaceEffect === "none" ? { workspaceEffect: "none" } : {}
2643
2665
  });
2666
+ settlePreparation();
2644
2667
  if (input.message.timeoutMs) {
2645
2668
  timeout = setTimeout(() => {
2646
2669
  void terminateProcessTree(subprocess).catch((error) => {
@@ -2676,6 +2699,7 @@ async function executeCommand(input) {
2676
2699
  error: error instanceof Error ? error.message : String(error)
2677
2700
  };
2678
2701
  } finally {
2702
+ settlePreparation();
2679
2703
  if (timeout) {
2680
2704
  clearTimeout(timeout);
2681
2705
  }
@@ -2691,6 +2715,13 @@ async function executeStreamingCommand(input) {
2691
2715
  let timedOut = false;
2692
2716
  let timeout;
2693
2717
  let spawnedProcess;
2718
+ let preparationSettled = false;
2719
+ const settlePreparation = () => {
2720
+ if (preparationSettled) return;
2721
+ preparationSettled = true;
2722
+ input.setPreparationHeartbeatLease(false);
2723
+ };
2724
+ input.setPreparationHeartbeatLease(true);
2694
2725
  try {
2695
2726
  if (cancelledProcessRuns.delete(input.message.runId)) {
2696
2727
  sendWorkerMessage(input.ws, {
@@ -2743,6 +2774,7 @@ async function executeStreamingCommand(input) {
2743
2774
  ...interactive ? { interactive: true, stdin: subprocess.stdin } : {},
2744
2775
  ...input.message.workspaceEffect === "none" ? { workspaceEffect: "none" } : {}
2745
2776
  });
2777
+ settlePreparation();
2746
2778
  started = true;
2747
2779
  sendWorkerMessage(input.ws, {
2748
2780
  type: "exec_started",
@@ -2819,6 +2851,7 @@ async function executeStreamingCommand(input) {
2819
2851
  });
2820
2852
  }
2821
2853
  } finally {
2854
+ settlePreparation();
2822
2855
  if (timeout) {
2823
2856
  clearTimeout(timeout);
2824
2857
  }
@@ -3160,7 +3193,8 @@ function createNodePtyBridge(options) {
3160
3193
  };
3161
3194
  }
3162
3195
  const workerPtyBridgeTestHarness = {
3163
- create: createNodePtyBridge
3196
+ create: createNodePtyBridge,
3197
+ closeAllBestEffort: closePtysBestEffort
3164
3198
  };
3165
3199
  function resolveHostShell(command, platform = process.platform) {
3166
3200
  if (platform === "win32") {
@@ -3309,11 +3343,23 @@ function closePty(message) {
3309
3343
  }
3310
3344
  ptyProcess.kill();
3311
3345
  }
3312
- function closeAllPtys() {
3313
- for (const ptyProcess of activePtys.values()) {
3314
- ptyProcess.kill();
3346
+ function closePtysBestEffort(ptys, reportFailure) {
3347
+ for (const [ptyId, ptyProcess] of ptys) {
3348
+ try {
3349
+ ptyProcess.kill();
3350
+ } catch (error) {
3351
+ reportFailure(ptyId, error);
3352
+ }
3315
3353
  }
3316
3354
  }
3355
+ function closeAllPtys() {
3356
+ closePtysBestEffort(activePtys, (ptyId, error) => {
3357
+ process.stderr.write(
3358
+ `[r5d-worker] failed to close PTY ${ptyId} after control disconnect: ${error instanceof Error ? error.message : String(error)}
3359
+ `
3360
+ );
3361
+ });
3362
+ }
3317
3363
  async function terminateCredentialBearingChildren(children) {
3318
3364
  const results = await Promise.allSettled(children.map(async ({ terminate }) => await terminate()));
3319
3365
  const failures = results.flatMap(
@@ -3373,16 +3419,6 @@ function websocketUrl(baseUrl, label) {
3373
3419
  }
3374
3420
  async function startWorker(options) {
3375
3421
  advanceWorkerAdmissionGeneration();
3376
- process.on("uncaughtException", (error) => {
3377
- process.stderr.write(`[r5d-worker] uncaught exception: ${error instanceof Error ? error.stack ?? error.message : String(error)}
3378
- `);
3379
- });
3380
- process.on("unhandledRejection", (reason) => {
3381
- process.stderr.write(
3382
- `[r5d-worker] unhandled rejection: ${reason instanceof Error ? reason.stack ?? reason.message : String(reason)}
3383
- `
3384
- );
3385
- });
3386
3422
  const config = readConfig(options.configPath);
3387
3423
  const { baseUrl, token } = resolveCredentials(options, config);
3388
3424
  initializeCredentialStoreDirectoryOnce();
@@ -4395,11 +4431,12 @@ async function startWorker(options) {
4395
4431
  }
4396
4432
  try {
4397
4433
  assertStillAdmitted();
4434
+ const hasDurableBranchDeletion = projectWorkspaceState.tombstones.some((tombstone) => tombstone.kind === "branch");
4398
4435
  return await performWorkspaceSync({
4399
4436
  attemptId,
4400
4437
  trigger: input.trigger,
4401
- confirmedLargeDiff: input.confirmedLargeDiff,
4402
- confirmationReason: input.confirmationReason,
4438
+ confirmedLargeDiff: input.confirmedLargeDiff === true || hasDurableBranchDeletion,
4439
+ confirmationReason: input.confirmationReason ?? (hasDurableBranchDeletion ? "durable project branch deletion" : void 0),
4403
4440
  resetToCanonical: input.resetToCanonical,
4404
4441
  requiredAncestorHeads: input.requiredAncestorHeads,
4405
4442
  assertStillAdmitted
@@ -4936,9 +4973,12 @@ async function startWorker(options) {
4936
4973
  );
4937
4974
  }
4938
4975
  };
4976
+ const hasDurableBranchDeletion = projectWorkspaceState.tombstones.some((tombstone) => tombstone.kind === "branch");
4939
4977
  const result = await performWorkspaceSync({
4940
4978
  attemptId: crypto.randomUUID(),
4941
4979
  trigger: { type: "connect" },
4980
+ confirmedLargeDiff: hasDurableBranchDeletion,
4981
+ confirmationReason: hasDurableBranchDeletion ? "durable project branch deletion" : void 0,
4942
4982
  resetToCanonical: workspaceConfigurationResetToCanonicalIsAllowed(message.resetToCanonical, incidentDeferral.incidentId),
4943
4983
  assertStillAdmitted: assertConfigurationSyncStillAdmitted
4944
4984
  });
@@ -5026,9 +5066,22 @@ async function startWorker(options) {
5026
5066
  Authorization: `Bearer ${token}`
5027
5067
  }
5028
5068
  });
5069
+ let lastServerControlMessageAtMs = performance.now();
5070
+ let serverHeartbeatProbeStartedAtMs;
5071
+ let lastServerHeartbeatSweepAtMs = performance.now();
5072
+ let serverHeartbeatWatchdog;
5073
+ const noteServerControlActivity = () => {
5074
+ lastServerControlMessageAtMs = performance.now();
5075
+ serverHeartbeatProbeStartedAtMs = void 0;
5076
+ };
5077
+ const stopServerHeartbeatWatchdog = () => {
5078
+ if (!serverHeartbeatWatchdog) return;
5079
+ clearInterval(serverHeartbeatWatchdog);
5080
+ serverHeartbeatWatchdog = void 0;
5081
+ };
5029
5082
  const sendLifecycleProgress = (progress) => {
5030
5083
  if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) return;
5031
- sendWorkerMessage(ws, { type: "worker_progress", progress });
5084
+ sendWorkerMessage(ws, { type: "worker_progress", progress: boundedWorkerLifecycleProgress(progress) });
5032
5085
  };
5033
5086
  const describeSnapshotProgress = (progress) => {
5034
5087
  const project = path.relative(projectsRoot, progress.projectRoot) || path.basename(progress.projectRoot);
@@ -5109,6 +5162,29 @@ async function startWorker(options) {
5109
5162
  process.on("SIGINT", handleGracefulShutdown);
5110
5163
  ws.addEventListener("open", () => {
5111
5164
  currentWorkerSocket = ws;
5165
+ noteServerControlActivity();
5166
+ lastServerHeartbeatSweepAtMs = performance.now();
5167
+ serverHeartbeatWatchdog = setInterval(() => {
5168
+ if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) return;
5169
+ const nowMs = performance.now();
5170
+ const heartbeatAction = workerServerHeartbeatAction({
5171
+ elapsedMs: Math.max(0, nowMs - lastServerControlMessageAtMs),
5172
+ ...serverHeartbeatProbeStartedAtMs === void 0 ? {} : { probeElapsedMs: Math.max(0, nowMs - serverHeartbeatProbeStartedAtMs) },
5173
+ timeoutMs: WORKER_SERVER_HEARTBEAT_TIMEOUT_MS,
5174
+ watchdogSweepDelayed: nowMs - lastServerHeartbeatSweepAtMs >= WORKER_SERVER_HEARTBEAT_INTERVAL_MS * 2
5175
+ });
5176
+ lastServerHeartbeatSweepAtMs = nowMs;
5177
+ if (heartbeatAction === "start_probe") {
5178
+ serverHeartbeatProbeStartedAtMs = nowMs;
5179
+ return;
5180
+ }
5181
+ if (heartbeatAction === "terminate") {
5182
+ stopServerHeartbeatWatchdog();
5183
+ process.stderr.write("[r5d-worker] server control heartbeat timed out; reconnecting\n");
5184
+ ws.terminate();
5185
+ }
5186
+ }, WORKER_SERVER_HEARTBEAT_INTERVAL_MS);
5187
+ serverHeartbeatWatchdog.unref();
5112
5188
  const startupOperationId = crypto.randomUUID();
5113
5189
  startupRecoveryPromise = Promise.resolve().then(async () => await performStartupRecovery(startupOperationId)).then(() => {
5114
5190
  startupRecoveryComplete = true;
@@ -5144,7 +5220,8 @@ async function startWorker(options) {
5144
5220
  ptyEnvFilesV1: true,
5145
5221
  workspaceRemediationAncestorGuardV1: true,
5146
5222
  workspaceIncidentConfigDeferralV1: true,
5147
- workspaceConfigResetToCanonicalV1: true
5223
+ workspaceConfigResetToCanonicalV1: true,
5224
+ projectBranchDeletionFastAckV1: true
5148
5225
  },
5149
5226
  projectRoot: projectsRoot,
5150
5227
  artifactRoot,
@@ -5165,6 +5242,7 @@ async function startWorker(options) {
5165
5242
  process.stdout.write("[r5d-worker] connected\n");
5166
5243
  });
5167
5244
  ws.addEventListener("message", (event) => {
5245
+ noteServerControlActivity();
5168
5246
  void (async () => {
5169
5247
  const message = JSON.parse(String(event.data));
5170
5248
  if (!startupRecoveryComplete && message.type !== "ping") await startupRecoveryPromise;
@@ -5397,23 +5475,15 @@ async function startWorker(options) {
5397
5475
  return true;
5398
5476
  });
5399
5477
  if (deletionNeedsSync) {
5400
- const result = await runWorkspaceSync({
5401
- trigger: {
5478
+ scheduleAutomaticWorkspaceSync(
5479
+ {
5402
5480
  type: "manual",
5403
5481
  projectId: message.projectId,
5404
5482
  branchName: message.branchName,
5405
5483
  detail: "deleted project branch"
5406
5484
  },
5407
- confirmedLargeDiff: true,
5408
- confirmationReason: "explicit project branch deletion",
5409
- sendResult: false
5410
- });
5411
- if (result.outcome === "failed" || result.outcome === "conflict_blocked" || result.outcome === "large_diff_blocked") {
5412
- throw new Error(result.error ?? `Project branch deletion workspace sync ended with ${result.outcome}`);
5413
- }
5414
- if (pendingMirrorDeletes.has(deletionKey)) {
5415
- throw new Error("Project branch deletion did not remove the hidden mirror ref");
5416
- }
5485
+ 0
5486
+ );
5417
5487
  }
5418
5488
  sendWorkerMessage(ws, {
5419
5489
  type: "operation_result",
@@ -5772,6 +5842,13 @@ async function startWorker(options) {
5772
5842
  token,
5773
5843
  artifactRoot,
5774
5844
  planRoot,
5845
+ setPreparationHeartbeatLease: (active) => {
5846
+ sendWorkerMessageFromCurrentSource(ws, {
5847
+ type: "heartbeat_lease",
5848
+ operationId: `exec:${message.runId}`,
5849
+ active
5850
+ });
5851
+ },
5775
5852
  assertAdmission: () => {
5776
5853
  if (!workspaceCommandTransportIsAllowed(deferredWorkspaceConfiguration, activeWorkspaceIncidentId, "exec")) {
5777
5854
  throw new Error(
@@ -5835,6 +5912,13 @@ async function startWorker(options) {
5835
5912
  token,
5836
5913
  artifactRoot,
5837
5914
  planRoot,
5915
+ setPreparationHeartbeatLease: (active) => {
5916
+ sendWorkerMessageFromCurrentSource(ws, {
5917
+ type: "heartbeat_lease",
5918
+ operationId: `exec:${message.runId}`,
5919
+ active
5920
+ });
5921
+ },
5838
5922
  assertAdmission: assertMessageAdmission
5839
5923
  });
5840
5924
  } finally {
@@ -5967,6 +6051,7 @@ async function startWorker(options) {
5967
6051
  const processEvents = process;
5968
6052
  processEvents.off("SIGTERM", handleGracefulShutdown);
5969
6053
  processEvents.off("SIGINT", handleGracefulShutdown);
6054
+ stopServerHeartbeatWatchdog();
5970
6055
  if (workspaceAutomaticTimer) {
5971
6056
  clearTimeout(workspaceAutomaticTimer);
5972
6057
  workspaceAutomaticTimer = void 0;
@@ -6004,31 +6089,40 @@ async function startWorker(options) {
6004
6089
  process.exit(1);
6005
6090
  return;
6006
6091
  }
6007
- void startWorker(options).catch((error) => {
6008
- process.stderr.write(`[r5d-worker] reconnect failed: ${error instanceof Error ? error.message : String(error)}
6092
+ const reconnectDelayMs = workerReconnectDelayMs();
6093
+ process.stderr.write(`[r5d-worker] workspace recovery complete; reconnecting in ${reconnectDelayMs}ms
6009
6094
  `);
6010
- process.exit(error instanceof WorkerServerUnavailableError ? WORKER_RECONNECT_EXIT_CODE : 1);
6011
- });
6095
+ setTimeout(() => {
6096
+ void startWorker(options).catch((error) => {
6097
+ process.stderr.write(`[r5d-worker] reconnect failed: ${error instanceof Error ? error.message : String(error)}
6098
+ `);
6099
+ process.exit(error instanceof WorkerServerUnavailableError ? WORKER_RECONNECT_EXIT_CODE : 1);
6100
+ });
6101
+ }, reconnectDelayMs);
6012
6102
  });
6013
6103
  return;
6014
6104
  }
6015
6105
  if (activeProcesses.size > 0 || credentialBearingProcessGroups.size > 0 || activePtys.size > 0 || pendingProcessTerminals.size > 0) {
6016
- process.stderr.write(
6017
- `[r5d-worker] ${activeProcesses.size} process(es), ${credentialBearingProcessGroups.size} owned process group(s), and ${activePtys.size} PTY(s) active with ${pendingProcessTerminals.size} terminal report(s) pending; reconnecting in ${WORKER_RECONNECT_DELAY_MS}ms
6106
+ const scheduleReconnect = (reconnect2) => {
6107
+ const reconnectDelayMs = workerReconnectDelayMs();
6108
+ process.stderr.write(
6109
+ `[r5d-worker] ${activeProcesses.size} process(es), ${credentialBearingProcessGroups.size} owned process group(s), and ${activePtys.size} PTY(s) active with ${pendingProcessTerminals.size} terminal report(s) pending; reconnecting in ${reconnectDelayMs}ms
6018
6110
  `
6019
- );
6111
+ );
6112
+ setTimeout(reconnect2, reconnectDelayMs);
6113
+ };
6020
6114
  const reconnect = () => {
6021
6115
  void startWorker(options).catch((error) => {
6022
6116
  process.stderr.write(`[r5d-worker] reconnect failed: ${error instanceof Error ? error.message : String(error)}
6023
6117
  `);
6024
6118
  if (activeProcesses.size > 0 || credentialBearingProcessGroups.size > 0 || activePtys.size > 0 || pendingProcessTerminals.size > 0) {
6025
- setTimeout(reconnect, WORKER_RECONNECT_DELAY_MS);
6119
+ scheduleReconnect(reconnect);
6026
6120
  return;
6027
6121
  }
6028
6122
  process.exit(error instanceof WorkerServerUnavailableError ? WORKER_RECONNECT_EXIT_CODE : 1);
6029
6123
  });
6030
6124
  };
6031
- setTimeout(reconnect, WORKER_RECONNECT_DELAY_MS);
6125
+ scheduleReconnect(reconnect);
6032
6126
  return;
6033
6127
  }
6034
6128
  process.exit(WORKER_RECONNECT_EXIT_CODE);
@@ -6038,7 +6132,20 @@ async function startWorker(options) {
6038
6132
  process.stderr.write(`[r5d-worker] websocket error${message}
6039
6133
  `);
6040
6134
  });
6041
- await new Promise(() => {
6135
+ await new Promise((resolve) => {
6136
+ ws.addEventListener("close", () => resolve(), { once: true });
6137
+ });
6138
+ }
6139
+ function installWorkerRuntimeFailureLogging() {
6140
+ process.on("uncaughtException", (error) => {
6141
+ process.stderr.write(`[r5d-worker] uncaught exception: ${error instanceof Error ? error.stack ?? error.message : String(error)}
6142
+ `);
6143
+ });
6144
+ process.on("unhandledRejection", (reason) => {
6145
+ process.stderr.write(
6146
+ `[r5d-worker] unhandled rejection: ${reason instanceof Error ? reason.stack ?? reason.message : String(reason)}
6147
+ `
6148
+ );
6042
6149
  });
6043
6150
  }
6044
6151
  async function main() {
@@ -6058,6 +6165,7 @@ async function main() {
6058
6165
  }
6059
6166
  if (process.env[WORKER_RUNTIME_ENV] === "1") {
6060
6167
  delete process.env[WORKER_RUNTIME_ENV];
6168
+ installWorkerRuntimeFailureLogging();
6061
6169
  try {
6062
6170
  await startWorker(parsed.options);
6063
6171
  } catch (error) {
@@ -6110,6 +6218,7 @@ export {
6110
6218
  syncSessionArtifacts,
6111
6219
  workerChildProcessEnvironment,
6112
6220
  workerGitSecurityTestHarness,
6221
+ workerLifecycleTestHarness,
6113
6222
  workerPtyBridgeTestHarness,
6114
6223
  workerPtyTestHarness,
6115
6224
  writeWorkerTextFile
@@ -1,5 +1,5 @@
1
1
  {
2
2
  "name": "@ricsam/r5d-worker",
3
- "version": "0.0.91",
3
+ "version": "0.0.93",
4
4
  "type": "module"
5
5
  }
@@ -5,6 +5,7 @@ const WORKER_RELOAD_EXIT_CODE = 75;
5
5
  const WORKER_RECONNECT_EXIT_CODE = 76;
6
6
  const WORKER_CREDENTIAL_RESTART_EXIT_CODE = 77;
7
7
  const WORKER_RECONNECT_DELAY_MS = 2e3;
8
+ const WORKER_RECONNECT_JITTER_MS = 3e3;
8
9
  const SUPERVISOR_SIGNALS = ["SIGINT", "SIGTERM", "SIGHUP"];
9
10
  function defaultRuntimeSpawner(argv, options) {
10
11
  return Bun.spawn(argv, options);
@@ -12,6 +13,10 @@ function defaultRuntimeSpawner(argv, options) {
12
13
  function defaultRuntimeDelay(delayMs) {
13
14
  return new Promise((resolve) => setTimeout(resolve, delayMs));
14
15
  }
16
+ function workerReconnectDelayMs(random = Math.random) {
17
+ const sample = Math.max(0, Math.min(1, random()));
18
+ return WORKER_RECONNECT_DELAY_MS + Math.floor(sample * WORKER_RECONNECT_JITTER_MS);
19
+ }
15
20
  async function defaultRuntimeReaper(runtime) {
16
21
  await terminateProcessTree(runtime);
17
22
  }
@@ -24,7 +29,7 @@ function signalExitCode(signal) {
24
29
  function osSignalNumber(signal) {
25
30
  return osConstants.signals[signal];
26
31
  }
27
- async function superviseWorkerRuntime(argv, env = process.env, spawnRuntime = defaultRuntimeSpawner, waitBeforeReconnect = defaultRuntimeDelay, reapCredentialRuntime = defaultRuntimeReaper) {
32
+ async function superviseWorkerRuntime(argv, env = process.env, spawnRuntime = defaultRuntimeSpawner, waitBeforeReconnect = defaultRuntimeDelay, reapCredentialRuntime = defaultRuntimeReaper, reconnectDelay = workerReconnectDelayMs) {
28
33
  if (!process.argv[1]) {
29
34
  throw new Error("Cannot locate the r5d-worker entrypoint");
30
35
  }
@@ -76,9 +81,10 @@ async function superviseWorkerRuntime(argv, env = process.env, spawnRuntime = de
76
81
  continue;
77
82
  }
78
83
  if (exitCode === WORKER_RECONNECT_EXIT_CODE) {
79
- process.stderr.write(`[r5d-worker] reconnecting in ${WORKER_RECONNECT_DELAY_MS}ms
84
+ const reconnectDelayMs = reconnectDelay();
85
+ process.stderr.write(`[r5d-worker] reconnecting in ${reconnectDelayMs}ms
80
86
  `);
81
- await waitBeforeReconnect(WORKER_RECONNECT_DELAY_MS);
87
+ await waitBeforeReconnect(reconnectDelayMs);
82
88
  if (receivedSignal) return signalExitCode(receivedSignal);
83
89
  continue;
84
90
  }
@@ -93,8 +99,10 @@ export {
93
99
  WORKER_CREDENTIAL_RESTART_EXIT_CODE,
94
100
  WORKER_RECONNECT_DELAY_MS,
95
101
  WORKER_RECONNECT_EXIT_CODE,
102
+ WORKER_RECONNECT_JITTER_MS,
96
103
  WORKER_RELOAD_EXIT_CODE,
97
104
  WORKER_RUNTIME_ENV,
98
105
  isRetryableWorkerServerStatus,
99
- superviseWorkerRuntime
106
+ superviseWorkerRuntime,
107
+ workerReconnectDelayMs
100
108
  };
@@ -23,6 +23,26 @@ type WorkerProjectConfig = {
23
23
  baseCommitHash: string;
24
24
  }>;
25
25
  };
26
+ type WorkerLifecycleProgress = {
27
+ phase: "starting" | "recovering" | "configuring" | "syncing" | "ready" | "failed";
28
+ operationId: string;
29
+ detail?: string;
30
+ completedBytes?: number;
31
+ totalBytes?: number;
32
+ completedItems?: number;
33
+ totalItems?: number;
34
+ };
35
+ declare function boundedWorkerLifecycleProgress(progress: WorkerLifecycleProgress): WorkerLifecycleProgress;
36
+ declare function workerServerHeartbeatAction(input: {
37
+ elapsedMs: number;
38
+ probeElapsedMs?: number;
39
+ timeoutMs: number;
40
+ watchdogSweepDelayed: boolean;
41
+ }): "wait" | "start_probe" | "terminate";
42
+ export declare const workerLifecycleTestHarness: {
43
+ boundedWorkerLifecycleProgress: typeof boundedWorkerLifecycleProgress;
44
+ workerServerHeartbeatAction: typeof workerServerHeartbeatAction;
45
+ };
26
46
  type WorkspaceSyncTrigger = {
27
47
  type: "connect" | "inbound_head" | "write" | "edit" | "shell_inline" | "process_terminal" | "process_cancel" | "periodic" | "manual" | "remediation" | "remediation_confirm" | "remediation_reset";
28
48
  sessionId?: string;
@@ -813,12 +833,16 @@ declare function createNodePtyBridge(options: {
813
833
  }): WorkerPty;
814
834
  export declare const workerPtyBridgeTestHarness: {
815
835
  create: typeof createNodePtyBridge;
836
+ closeAllBestEffort: typeof closePtysBestEffort;
816
837
  };
817
838
  export declare function resolveHostShell(command?: string, platform?: NodeJS.Platform): {
818
839
  file: string;
819
840
  args: string[];
820
841
  env?: Record<string, string>;
821
842
  };
843
+ declare function closePtysBestEffort(ptys: Iterable<readonly [string, {
844
+ kill: () => void;
845
+ }]>, reportFailure: (ptyId: string, error: unknown) => void): void;
822
846
  declare function terminateCredentialBearingChildren(children: readonly {
823
847
  label: string;
824
848
  terminate: () => void | Promise<void>;
@@ -4,6 +4,7 @@ export declare const WORKER_RELOAD_EXIT_CODE = 75;
4
4
  export declare const WORKER_RECONNECT_EXIT_CODE = 76;
5
5
  export declare const WORKER_CREDENTIAL_RESTART_EXIT_CODE = 77;
6
6
  export declare const WORKER_RECONNECT_DELAY_MS = 2000;
7
+ export declare const WORKER_RECONNECT_JITTER_MS = 3000;
7
8
  type RuntimeProcess = KillableSubprocess;
8
9
  export type RuntimeSpawner = (argv: string[], options: {
9
10
  stdin: "inherit";
@@ -14,6 +15,8 @@ export type RuntimeSpawner = (argv: string[], options: {
14
15
  }) => RuntimeProcess;
15
16
  export type RuntimeDelay = (delayMs: number) => Promise<void>;
16
17
  export type RuntimeReaper = (runtime: RuntimeProcess) => Promise<void>;
18
+ export type RuntimeReconnectDelay = () => number;
19
+ export declare function workerReconnectDelayMs(random?: () => number): number;
17
20
  export declare function isRetryableWorkerServerStatus(status: number): boolean;
18
- export declare function superviseWorkerRuntime(argv: string[], env?: Record<string, string | undefined>, spawnRuntime?: RuntimeSpawner, waitBeforeReconnect?: RuntimeDelay, reapCredentialRuntime?: RuntimeReaper): Promise<number>;
21
+ export declare function superviseWorkerRuntime(argv: string[], env?: Record<string, string | undefined>, spawnRuntime?: RuntimeSpawner, waitBeforeReconnect?: RuntimeDelay, reapCredentialRuntime?: RuntimeReaper, reconnectDelay?: RuntimeReconnectDelay): Promise<number>;
19
22
  export {};
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@ricsam/r5d-worker",
3
- "version": "0.0.91",
3
+ "version": "0.0.93",
4
4
  "type": "module",
5
5
  "main": "./dist/cjs/main.cjs",
6
6
  "module": "./dist/mjs/main.mjs",