@ricsam/r5d-worker 0.0.91 → 0.0.93
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/cjs/main.cjs +150 -40
- package/dist/cjs/package.json +1 -1
- package/dist/cjs/supervisor.cjs +15 -5
- package/dist/mjs/main.mjs +151 -42
- package/dist/mjs/package.json +1 -1
- package/dist/mjs/supervisor.mjs +12 -4
- package/dist/types/main.d.ts +24 -0
- package/dist/types/supervisor.d.ts +4 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -43,7 +43,7 @@ The supervisor also reconnects automatically after server rollouts and transient
|
|
|
43
43
|
|
|
44
44
|
The runtime opens its control connection before recovering interrupted project snapshots. During recovery and initial project configuration it remains connected but non-dispatchable, reports its current phase plus branch/byte progress to the server, and prints the same progress locally. Recovery runs in a credential-sanitized helper process so the control loop can continue answering heartbeats. A durable `restoring` transaction state permits same-filesystem snapshots to be restored with one atomic directory move; cross-filesystem layouts retain the copy-and-fsync fallback.
|
|
45
45
|
|
|
46
|
-
The server
|
|
46
|
+
Both sides use monotonic, two-stage heartbeat recovery. The server sends pings, grants bounded leases to synchronous workspace and command-preparation work, and requires a fresh missed probe window before force-evicting a stale socket. The worker answers pings and uses a deliberately slower backup watchdog only for a half-open connection; reconnect delays are jittered across workers. A delayed watchdog sweep renews the probe instead of disconnecting, so synchronous local work or a stalled server event loop cannot turn one overdue timer callback into a mass disconnect.
|
|
47
47
|
|
|
48
48
|
Credential-generation restart fencing protects against stale or accidentally detached descendants. It is not a hostile same-UID sandbox: code running as the worker's OS user can read or tamper with that user's config and auth files, install a persistent user service, or return after reboot. Run untrusted agents in a disposable VM that is reset at the boundary, or use a separately privileged credential broker/service account. When rotating the worker's own local `r5dctl` credential, stop the worker and reboot/reset the host before writing the replacement credential so an old same-UID descendant cannot read the new config.
|
|
49
49
|
|
package/dist/cjs/main.cjs
CHANGED
|
@@ -50,6 +50,7 @@ __export(main_exports, {
|
|
|
50
50
|
syncSessionArtifacts: () => syncSessionArtifacts,
|
|
51
51
|
workerChildProcessEnvironment: () => workerChildProcessEnvironment,
|
|
52
52
|
workerGitSecurityTestHarness: () => workerGitSecurityTestHarness,
|
|
53
|
+
workerLifecycleTestHarness: () => workerLifecycleTestHarness,
|
|
53
54
|
workerPtyBridgeTestHarness: () => workerPtyBridgeTestHarness,
|
|
54
55
|
workerPtyTestHarness: () => workerPtyTestHarness,
|
|
55
56
|
writeWorkerTextFile: () => writeWorkerTextFile
|
|
@@ -84,6 +85,21 @@ var import_project_snapshot_recovery_runner = require("./project-snapshot-recove
|
|
|
84
85
|
var import_project_workspace_state = require("./project-workspace-state.cjs");
|
|
85
86
|
var import_workspace_git_sync = require("./workspace-git-sync.cjs");
|
|
86
87
|
const import_meta = {};
|
|
88
|
+
const WORKER_LIFECYCLE_DETAIL_MAX_LENGTH = 500;
|
|
89
|
+
const WORKER_SERVER_HEARTBEAT_INTERVAL_MS = 15e3;
|
|
90
|
+
const WORKER_SERVER_HEARTBEAT_TIMEOUT_MS = 2 * 60 * 1e3;
|
|
91
|
+
function boundedWorkerLifecycleProgress(progress) {
|
|
92
|
+
return progress.detail === void 0 ? progress : { ...progress, detail: progress.detail.slice(0, WORKER_LIFECYCLE_DETAIL_MAX_LENGTH) };
|
|
93
|
+
}
|
|
94
|
+
function workerServerHeartbeatAction(input) {
|
|
95
|
+
if (input.elapsedMs < input.timeoutMs) return "wait";
|
|
96
|
+
if (input.watchdogSweepDelayed || input.probeElapsedMs === void 0) return "start_probe";
|
|
97
|
+
return input.probeElapsedMs >= input.timeoutMs ? "terminate" : "wait";
|
|
98
|
+
}
|
|
99
|
+
const workerLifecycleTestHarness = {
|
|
100
|
+
boundedWorkerLifecycleProgress,
|
|
101
|
+
workerServerHeartbeatAction
|
|
102
|
+
};
|
|
87
103
|
class ProjectWorkspaceConfigurationDeferredError extends Error {
|
|
88
104
|
}
|
|
89
105
|
class WorkerServerUnavailableError extends Error {
|
|
@@ -2597,6 +2613,13 @@ async function executeCommand(input) {
|
|
|
2597
2613
|
const startedAt = Date.now();
|
|
2598
2614
|
let timeout;
|
|
2599
2615
|
let spawnedProcess;
|
|
2616
|
+
let preparationSettled = false;
|
|
2617
|
+
const settlePreparation = () => {
|
|
2618
|
+
if (preparationSettled) return;
|
|
2619
|
+
preparationSettled = true;
|
|
2620
|
+
input.setPreparationHeartbeatLease(false);
|
|
2621
|
+
};
|
|
2622
|
+
input.setPreparationHeartbeatLease(true);
|
|
2600
2623
|
try {
|
|
2601
2624
|
const targetProcessEnv = await prepareShellEnvForTarget({
|
|
2602
2625
|
target: input.resolvedTarget.target,
|
|
@@ -2634,6 +2657,7 @@ async function executeCommand(input) {
|
|
|
2634
2657
|
startedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2635
2658
|
...input.message.workspaceEffect === "none" ? { workspaceEffect: "none" } : {}
|
|
2636
2659
|
});
|
|
2660
|
+
settlePreparation();
|
|
2637
2661
|
if (input.message.timeoutMs) {
|
|
2638
2662
|
timeout = setTimeout(() => {
|
|
2639
2663
|
void (0, import_process_tree.terminateProcessTree)(subprocess).catch((error) => {
|
|
@@ -2669,6 +2693,7 @@ async function executeCommand(input) {
|
|
|
2669
2693
|
error: error instanceof Error ? error.message : String(error)
|
|
2670
2694
|
};
|
|
2671
2695
|
} finally {
|
|
2696
|
+
settlePreparation();
|
|
2672
2697
|
if (timeout) {
|
|
2673
2698
|
clearTimeout(timeout);
|
|
2674
2699
|
}
|
|
@@ -2684,6 +2709,13 @@ async function executeStreamingCommand(input) {
|
|
|
2684
2709
|
let timedOut = false;
|
|
2685
2710
|
let timeout;
|
|
2686
2711
|
let spawnedProcess;
|
|
2712
|
+
let preparationSettled = false;
|
|
2713
|
+
const settlePreparation = () => {
|
|
2714
|
+
if (preparationSettled) return;
|
|
2715
|
+
preparationSettled = true;
|
|
2716
|
+
input.setPreparationHeartbeatLease(false);
|
|
2717
|
+
};
|
|
2718
|
+
input.setPreparationHeartbeatLease(true);
|
|
2687
2719
|
try {
|
|
2688
2720
|
if (cancelledProcessRuns.delete(input.message.runId)) {
|
|
2689
2721
|
sendWorkerMessage(input.ws, {
|
|
@@ -2736,6 +2768,7 @@ async function executeStreamingCommand(input) {
|
|
|
2736
2768
|
...interactive ? { interactive: true, stdin: subprocess.stdin } : {},
|
|
2737
2769
|
...input.message.workspaceEffect === "none" ? { workspaceEffect: "none" } : {}
|
|
2738
2770
|
});
|
|
2771
|
+
settlePreparation();
|
|
2739
2772
|
started = true;
|
|
2740
2773
|
sendWorkerMessage(input.ws, {
|
|
2741
2774
|
type: "exec_started",
|
|
@@ -2812,6 +2845,7 @@ async function executeStreamingCommand(input) {
|
|
|
2812
2845
|
});
|
|
2813
2846
|
}
|
|
2814
2847
|
} finally {
|
|
2848
|
+
settlePreparation();
|
|
2815
2849
|
if (timeout) {
|
|
2816
2850
|
clearTimeout(timeout);
|
|
2817
2851
|
}
|
|
@@ -3153,7 +3187,8 @@ function createNodePtyBridge(options) {
|
|
|
3153
3187
|
};
|
|
3154
3188
|
}
|
|
3155
3189
|
const workerPtyBridgeTestHarness = {
|
|
3156
|
-
create: createNodePtyBridge
|
|
3190
|
+
create: createNodePtyBridge,
|
|
3191
|
+
closeAllBestEffort: closePtysBestEffort
|
|
3157
3192
|
};
|
|
3158
3193
|
function resolveHostShell(command, platform = process.platform) {
|
|
3159
3194
|
if (platform === "win32") {
|
|
@@ -3302,11 +3337,23 @@ function closePty(message) {
|
|
|
3302
3337
|
}
|
|
3303
3338
|
ptyProcess.kill();
|
|
3304
3339
|
}
|
|
3305
|
-
function
|
|
3306
|
-
for (const ptyProcess of
|
|
3307
|
-
|
|
3340
|
+
function closePtysBestEffort(ptys, reportFailure) {
|
|
3341
|
+
for (const [ptyId, ptyProcess] of ptys) {
|
|
3342
|
+
try {
|
|
3343
|
+
ptyProcess.kill();
|
|
3344
|
+
} catch (error) {
|
|
3345
|
+
reportFailure(ptyId, error);
|
|
3346
|
+
}
|
|
3308
3347
|
}
|
|
3309
3348
|
}
|
|
3349
|
+
function closeAllPtys() {
|
|
3350
|
+
closePtysBestEffort(activePtys, (ptyId, error) => {
|
|
3351
|
+
process.stderr.write(
|
|
3352
|
+
`[r5d-worker] failed to close PTY ${ptyId} after control disconnect: ${error instanceof Error ? error.message : String(error)}
|
|
3353
|
+
`
|
|
3354
|
+
);
|
|
3355
|
+
});
|
|
3356
|
+
}
|
|
3310
3357
|
async function terminateCredentialBearingChildren(children) {
|
|
3311
3358
|
const results = await Promise.allSettled(children.map(async ({ terminate }) => await terminate()));
|
|
3312
3359
|
const failures = results.flatMap(
|
|
@@ -3366,16 +3413,6 @@ function websocketUrl(baseUrl, label) {
|
|
|
3366
3413
|
}
|
|
3367
3414
|
async function startWorker(options) {
|
|
3368
3415
|
advanceWorkerAdmissionGeneration();
|
|
3369
|
-
process.on("uncaughtException", (error) => {
|
|
3370
|
-
process.stderr.write(`[r5d-worker] uncaught exception: ${error instanceof Error ? error.stack ?? error.message : String(error)}
|
|
3371
|
-
`);
|
|
3372
|
-
});
|
|
3373
|
-
process.on("unhandledRejection", (reason) => {
|
|
3374
|
-
process.stderr.write(
|
|
3375
|
-
`[r5d-worker] unhandled rejection: ${reason instanceof Error ? reason.stack ?? reason.message : String(reason)}
|
|
3376
|
-
`
|
|
3377
|
-
);
|
|
3378
|
-
});
|
|
3379
3416
|
const config = readConfig(options.configPath);
|
|
3380
3417
|
const { baseUrl, token } = resolveCredentials(options, config);
|
|
3381
3418
|
initializeCredentialStoreDirectoryOnce();
|
|
@@ -4388,11 +4425,12 @@ async function startWorker(options) {
|
|
|
4388
4425
|
}
|
|
4389
4426
|
try {
|
|
4390
4427
|
assertStillAdmitted();
|
|
4428
|
+
const hasDurableBranchDeletion = projectWorkspaceState.tombstones.some((tombstone) => tombstone.kind === "branch");
|
|
4391
4429
|
return await performWorkspaceSync({
|
|
4392
4430
|
attemptId,
|
|
4393
4431
|
trigger: input.trigger,
|
|
4394
|
-
confirmedLargeDiff: input.confirmedLargeDiff,
|
|
4395
|
-
confirmationReason: input.confirmationReason,
|
|
4432
|
+
confirmedLargeDiff: input.confirmedLargeDiff === true || hasDurableBranchDeletion,
|
|
4433
|
+
confirmationReason: input.confirmationReason ?? (hasDurableBranchDeletion ? "durable project branch deletion" : void 0),
|
|
4396
4434
|
resetToCanonical: input.resetToCanonical,
|
|
4397
4435
|
requiredAncestorHeads: input.requiredAncestorHeads,
|
|
4398
4436
|
assertStillAdmitted
|
|
@@ -4929,9 +4967,12 @@ async function startWorker(options) {
|
|
|
4929
4967
|
);
|
|
4930
4968
|
}
|
|
4931
4969
|
};
|
|
4970
|
+
const hasDurableBranchDeletion = projectWorkspaceState.tombstones.some((tombstone) => tombstone.kind === "branch");
|
|
4932
4971
|
const result = await performWorkspaceSync({
|
|
4933
4972
|
attemptId: crypto.randomUUID(),
|
|
4934
4973
|
trigger: { type: "connect" },
|
|
4974
|
+
confirmedLargeDiff: hasDurableBranchDeletion,
|
|
4975
|
+
confirmationReason: hasDurableBranchDeletion ? "durable project branch deletion" : void 0,
|
|
4935
4976
|
resetToCanonical: workspaceConfigurationResetToCanonicalIsAllowed(message.resetToCanonical, incidentDeferral.incidentId),
|
|
4936
4977
|
assertStillAdmitted: assertConfigurationSyncStillAdmitted
|
|
4937
4978
|
});
|
|
@@ -5019,9 +5060,22 @@ async function startWorker(options) {
|
|
|
5019
5060
|
Authorization: `Bearer ${token}`
|
|
5020
5061
|
}
|
|
5021
5062
|
});
|
|
5063
|
+
let lastServerControlMessageAtMs = performance.now();
|
|
5064
|
+
let serverHeartbeatProbeStartedAtMs;
|
|
5065
|
+
let lastServerHeartbeatSweepAtMs = performance.now();
|
|
5066
|
+
let serverHeartbeatWatchdog;
|
|
5067
|
+
const noteServerControlActivity = () => {
|
|
5068
|
+
lastServerControlMessageAtMs = performance.now();
|
|
5069
|
+
serverHeartbeatProbeStartedAtMs = void 0;
|
|
5070
|
+
};
|
|
5071
|
+
const stopServerHeartbeatWatchdog = () => {
|
|
5072
|
+
if (!serverHeartbeatWatchdog) return;
|
|
5073
|
+
clearInterval(serverHeartbeatWatchdog);
|
|
5074
|
+
serverHeartbeatWatchdog = void 0;
|
|
5075
|
+
};
|
|
5022
5076
|
const sendLifecycleProgress = (progress) => {
|
|
5023
5077
|
if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) return;
|
|
5024
|
-
sendWorkerMessage(ws, { type: "worker_progress", progress });
|
|
5078
|
+
sendWorkerMessage(ws, { type: "worker_progress", progress: boundedWorkerLifecycleProgress(progress) });
|
|
5025
5079
|
};
|
|
5026
5080
|
const describeSnapshotProgress = (progress) => {
|
|
5027
5081
|
const project = import_node_path.default.relative(projectsRoot, progress.projectRoot) || import_node_path.default.basename(progress.projectRoot);
|
|
@@ -5102,6 +5156,29 @@ async function startWorker(options) {
|
|
|
5102
5156
|
process.on("SIGINT", handleGracefulShutdown);
|
|
5103
5157
|
ws.addEventListener("open", () => {
|
|
5104
5158
|
currentWorkerSocket = ws;
|
|
5159
|
+
noteServerControlActivity();
|
|
5160
|
+
lastServerHeartbeatSweepAtMs = performance.now();
|
|
5161
|
+
serverHeartbeatWatchdog = setInterval(() => {
|
|
5162
|
+
if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) return;
|
|
5163
|
+
const nowMs = performance.now();
|
|
5164
|
+
const heartbeatAction = workerServerHeartbeatAction({
|
|
5165
|
+
elapsedMs: Math.max(0, nowMs - lastServerControlMessageAtMs),
|
|
5166
|
+
...serverHeartbeatProbeStartedAtMs === void 0 ? {} : { probeElapsedMs: Math.max(0, nowMs - serverHeartbeatProbeStartedAtMs) },
|
|
5167
|
+
timeoutMs: WORKER_SERVER_HEARTBEAT_TIMEOUT_MS,
|
|
5168
|
+
watchdogSweepDelayed: nowMs - lastServerHeartbeatSweepAtMs >= WORKER_SERVER_HEARTBEAT_INTERVAL_MS * 2
|
|
5169
|
+
});
|
|
5170
|
+
lastServerHeartbeatSweepAtMs = nowMs;
|
|
5171
|
+
if (heartbeatAction === "start_probe") {
|
|
5172
|
+
serverHeartbeatProbeStartedAtMs = nowMs;
|
|
5173
|
+
return;
|
|
5174
|
+
}
|
|
5175
|
+
if (heartbeatAction === "terminate") {
|
|
5176
|
+
stopServerHeartbeatWatchdog();
|
|
5177
|
+
process.stderr.write("[r5d-worker] server control heartbeat timed out; reconnecting\n");
|
|
5178
|
+
ws.terminate();
|
|
5179
|
+
}
|
|
5180
|
+
}, WORKER_SERVER_HEARTBEAT_INTERVAL_MS);
|
|
5181
|
+
serverHeartbeatWatchdog.unref();
|
|
5105
5182
|
const startupOperationId = crypto.randomUUID();
|
|
5106
5183
|
startupRecoveryPromise = Promise.resolve().then(async () => await performStartupRecovery(startupOperationId)).then(() => {
|
|
5107
5184
|
startupRecoveryComplete = true;
|
|
@@ -5137,7 +5214,8 @@ async function startWorker(options) {
|
|
|
5137
5214
|
ptyEnvFilesV1: true,
|
|
5138
5215
|
workspaceRemediationAncestorGuardV1: true,
|
|
5139
5216
|
workspaceIncidentConfigDeferralV1: true,
|
|
5140
|
-
workspaceConfigResetToCanonicalV1: true
|
|
5217
|
+
workspaceConfigResetToCanonicalV1: true,
|
|
5218
|
+
projectBranchDeletionFastAckV1: true
|
|
5141
5219
|
},
|
|
5142
5220
|
projectRoot: projectsRoot,
|
|
5143
5221
|
artifactRoot,
|
|
@@ -5158,6 +5236,7 @@ async function startWorker(options) {
|
|
|
5158
5236
|
process.stdout.write("[r5d-worker] connected\n");
|
|
5159
5237
|
});
|
|
5160
5238
|
ws.addEventListener("message", (event) => {
|
|
5239
|
+
noteServerControlActivity();
|
|
5161
5240
|
void (async () => {
|
|
5162
5241
|
const message = JSON.parse(String(event.data));
|
|
5163
5242
|
if (!startupRecoveryComplete && message.type !== "ping") await startupRecoveryPromise;
|
|
@@ -5390,23 +5469,15 @@ async function startWorker(options) {
|
|
|
5390
5469
|
return true;
|
|
5391
5470
|
});
|
|
5392
5471
|
if (deletionNeedsSync) {
|
|
5393
|
-
|
|
5394
|
-
|
|
5472
|
+
scheduleAutomaticWorkspaceSync(
|
|
5473
|
+
{
|
|
5395
5474
|
type: "manual",
|
|
5396
5475
|
projectId: message.projectId,
|
|
5397
5476
|
branchName: message.branchName,
|
|
5398
5477
|
detail: "deleted project branch"
|
|
5399
5478
|
},
|
|
5400
|
-
|
|
5401
|
-
|
|
5402
|
-
sendResult: false
|
|
5403
|
-
});
|
|
5404
|
-
if (result.outcome === "failed" || result.outcome === "conflict_blocked" || result.outcome === "large_diff_blocked") {
|
|
5405
|
-
throw new Error(result.error ?? `Project branch deletion workspace sync ended with ${result.outcome}`);
|
|
5406
|
-
}
|
|
5407
|
-
if (pendingMirrorDeletes.has(deletionKey)) {
|
|
5408
|
-
throw new Error("Project branch deletion did not remove the hidden mirror ref");
|
|
5409
|
-
}
|
|
5479
|
+
0
|
|
5480
|
+
);
|
|
5410
5481
|
}
|
|
5411
5482
|
sendWorkerMessage(ws, {
|
|
5412
5483
|
type: "operation_result",
|
|
@@ -5765,6 +5836,13 @@ async function startWorker(options) {
|
|
|
5765
5836
|
token,
|
|
5766
5837
|
artifactRoot,
|
|
5767
5838
|
planRoot,
|
|
5839
|
+
setPreparationHeartbeatLease: (active) => {
|
|
5840
|
+
sendWorkerMessageFromCurrentSource(ws, {
|
|
5841
|
+
type: "heartbeat_lease",
|
|
5842
|
+
operationId: `exec:${message.runId}`,
|
|
5843
|
+
active
|
|
5844
|
+
});
|
|
5845
|
+
},
|
|
5768
5846
|
assertAdmission: () => {
|
|
5769
5847
|
if (!workspaceCommandTransportIsAllowed(deferredWorkspaceConfiguration, activeWorkspaceIncidentId, "exec")) {
|
|
5770
5848
|
throw new Error(
|
|
@@ -5828,6 +5906,13 @@ async function startWorker(options) {
|
|
|
5828
5906
|
token,
|
|
5829
5907
|
artifactRoot,
|
|
5830
5908
|
planRoot,
|
|
5909
|
+
setPreparationHeartbeatLease: (active) => {
|
|
5910
|
+
sendWorkerMessageFromCurrentSource(ws, {
|
|
5911
|
+
type: "heartbeat_lease",
|
|
5912
|
+
operationId: `exec:${message.runId}`,
|
|
5913
|
+
active
|
|
5914
|
+
});
|
|
5915
|
+
},
|
|
5831
5916
|
assertAdmission: assertMessageAdmission
|
|
5832
5917
|
});
|
|
5833
5918
|
} finally {
|
|
@@ -5960,6 +6045,7 @@ async function startWorker(options) {
|
|
|
5960
6045
|
const processEvents = process;
|
|
5961
6046
|
processEvents.off("SIGTERM", handleGracefulShutdown);
|
|
5962
6047
|
processEvents.off("SIGINT", handleGracefulShutdown);
|
|
6048
|
+
stopServerHeartbeatWatchdog();
|
|
5963
6049
|
if (workspaceAutomaticTimer) {
|
|
5964
6050
|
clearTimeout(workspaceAutomaticTimer);
|
|
5965
6051
|
workspaceAutomaticTimer = void 0;
|
|
@@ -5997,31 +6083,40 @@ async function startWorker(options) {
|
|
|
5997
6083
|
process.exit(1);
|
|
5998
6084
|
return;
|
|
5999
6085
|
}
|
|
6000
|
-
|
|
6001
|
-
|
|
6086
|
+
const reconnectDelayMs = (0, import_supervisor.workerReconnectDelayMs)();
|
|
6087
|
+
process.stderr.write(`[r5d-worker] workspace recovery complete; reconnecting in ${reconnectDelayMs}ms
|
|
6002
6088
|
`);
|
|
6003
|
-
|
|
6004
|
-
|
|
6089
|
+
setTimeout(() => {
|
|
6090
|
+
void startWorker(options).catch((error) => {
|
|
6091
|
+
process.stderr.write(`[r5d-worker] reconnect failed: ${error instanceof Error ? error.message : String(error)}
|
|
6092
|
+
`);
|
|
6093
|
+
process.exit(error instanceof WorkerServerUnavailableError ? import_supervisor.WORKER_RECONNECT_EXIT_CODE : 1);
|
|
6094
|
+
});
|
|
6095
|
+
}, reconnectDelayMs);
|
|
6005
6096
|
});
|
|
6006
6097
|
return;
|
|
6007
6098
|
}
|
|
6008
6099
|
if (activeProcesses.size > 0 || credentialBearingProcessGroups.size > 0 || activePtys.size > 0 || pendingProcessTerminals.size > 0) {
|
|
6009
|
-
|
|
6010
|
-
|
|
6100
|
+
const scheduleReconnect = (reconnect2) => {
|
|
6101
|
+
const reconnectDelayMs = (0, import_supervisor.workerReconnectDelayMs)();
|
|
6102
|
+
process.stderr.write(
|
|
6103
|
+
`[r5d-worker] ${activeProcesses.size} process(es), ${credentialBearingProcessGroups.size} owned process group(s), and ${activePtys.size} PTY(s) active with ${pendingProcessTerminals.size} terminal report(s) pending; reconnecting in ${reconnectDelayMs}ms
|
|
6011
6104
|
`
|
|
6012
|
-
|
|
6105
|
+
);
|
|
6106
|
+
setTimeout(reconnect2, reconnectDelayMs);
|
|
6107
|
+
};
|
|
6013
6108
|
const reconnect = () => {
|
|
6014
6109
|
void startWorker(options).catch((error) => {
|
|
6015
6110
|
process.stderr.write(`[r5d-worker] reconnect failed: ${error instanceof Error ? error.message : String(error)}
|
|
6016
6111
|
`);
|
|
6017
6112
|
if (activeProcesses.size > 0 || credentialBearingProcessGroups.size > 0 || activePtys.size > 0 || pendingProcessTerminals.size > 0) {
|
|
6018
|
-
|
|
6113
|
+
scheduleReconnect(reconnect);
|
|
6019
6114
|
return;
|
|
6020
6115
|
}
|
|
6021
6116
|
process.exit(error instanceof WorkerServerUnavailableError ? import_supervisor.WORKER_RECONNECT_EXIT_CODE : 1);
|
|
6022
6117
|
});
|
|
6023
6118
|
};
|
|
6024
|
-
|
|
6119
|
+
scheduleReconnect(reconnect);
|
|
6025
6120
|
return;
|
|
6026
6121
|
}
|
|
6027
6122
|
process.exit(import_supervisor.WORKER_RECONNECT_EXIT_CODE);
|
|
@@ -6031,7 +6126,20 @@ async function startWorker(options) {
|
|
|
6031
6126
|
process.stderr.write(`[r5d-worker] websocket error${message}
|
|
6032
6127
|
`);
|
|
6033
6128
|
});
|
|
6034
|
-
await new Promise(() => {
|
|
6129
|
+
await new Promise((resolve) => {
|
|
6130
|
+
ws.addEventListener("close", () => resolve(), { once: true });
|
|
6131
|
+
});
|
|
6132
|
+
}
|
|
6133
|
+
function installWorkerRuntimeFailureLogging() {
|
|
6134
|
+
process.on("uncaughtException", (error) => {
|
|
6135
|
+
process.stderr.write(`[r5d-worker] uncaught exception: ${error instanceof Error ? error.stack ?? error.message : String(error)}
|
|
6136
|
+
`);
|
|
6137
|
+
});
|
|
6138
|
+
process.on("unhandledRejection", (reason) => {
|
|
6139
|
+
process.stderr.write(
|
|
6140
|
+
`[r5d-worker] unhandled rejection: ${reason instanceof Error ? reason.stack ?? reason.message : String(reason)}
|
|
6141
|
+
`
|
|
6142
|
+
);
|
|
6035
6143
|
});
|
|
6036
6144
|
}
|
|
6037
6145
|
async function main() {
|
|
@@ -6051,6 +6159,7 @@ async function main() {
|
|
|
6051
6159
|
}
|
|
6052
6160
|
if (process.env[import_supervisor.WORKER_RUNTIME_ENV] === "1") {
|
|
6053
6161
|
delete process.env[import_supervisor.WORKER_RUNTIME_ENV];
|
|
6162
|
+
installWorkerRuntimeFailureLogging();
|
|
6054
6163
|
try {
|
|
6055
6164
|
await startWorker(parsed.options);
|
|
6056
6165
|
} catch (error) {
|
|
@@ -6104,6 +6213,7 @@ if (isCliEntrypoint()) {
|
|
|
6104
6213
|
syncSessionArtifacts,
|
|
6105
6214
|
workerChildProcessEnvironment,
|
|
6106
6215
|
workerGitSecurityTestHarness,
|
|
6216
|
+
workerLifecycleTestHarness,
|
|
6107
6217
|
workerPtyBridgeTestHarness,
|
|
6108
6218
|
workerPtyTestHarness,
|
|
6109
6219
|
writeWorkerTextFile
|
package/dist/cjs/package.json
CHANGED
package/dist/cjs/supervisor.cjs
CHANGED
|
@@ -21,10 +21,12 @@ __export(supervisor_exports, {
|
|
|
21
21
|
WORKER_CREDENTIAL_RESTART_EXIT_CODE: () => WORKER_CREDENTIAL_RESTART_EXIT_CODE,
|
|
22
22
|
WORKER_RECONNECT_DELAY_MS: () => WORKER_RECONNECT_DELAY_MS,
|
|
23
23
|
WORKER_RECONNECT_EXIT_CODE: () => WORKER_RECONNECT_EXIT_CODE,
|
|
24
|
+
WORKER_RECONNECT_JITTER_MS: () => WORKER_RECONNECT_JITTER_MS,
|
|
24
25
|
WORKER_RELOAD_EXIT_CODE: () => WORKER_RELOAD_EXIT_CODE,
|
|
25
26
|
WORKER_RUNTIME_ENV: () => WORKER_RUNTIME_ENV,
|
|
26
27
|
isRetryableWorkerServerStatus: () => isRetryableWorkerServerStatus,
|
|
27
|
-
superviseWorkerRuntime: () => superviseWorkerRuntime
|
|
28
|
+
superviseWorkerRuntime: () => superviseWorkerRuntime,
|
|
29
|
+
workerReconnectDelayMs: () => workerReconnectDelayMs
|
|
28
30
|
});
|
|
29
31
|
module.exports = __toCommonJS(supervisor_exports);
|
|
30
32
|
var import_node_os = require("node:os");
|
|
@@ -34,6 +36,7 @@ const WORKER_RELOAD_EXIT_CODE = 75;
|
|
|
34
36
|
const WORKER_RECONNECT_EXIT_CODE = 76;
|
|
35
37
|
const WORKER_CREDENTIAL_RESTART_EXIT_CODE = 77;
|
|
36
38
|
const WORKER_RECONNECT_DELAY_MS = 2e3;
|
|
39
|
+
const WORKER_RECONNECT_JITTER_MS = 3e3;
|
|
37
40
|
const SUPERVISOR_SIGNALS = ["SIGINT", "SIGTERM", "SIGHUP"];
|
|
38
41
|
function defaultRuntimeSpawner(argv, options) {
|
|
39
42
|
return Bun.spawn(argv, options);
|
|
@@ -41,6 +44,10 @@ function defaultRuntimeSpawner(argv, options) {
|
|
|
41
44
|
function defaultRuntimeDelay(delayMs) {
|
|
42
45
|
return new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
43
46
|
}
|
|
47
|
+
function workerReconnectDelayMs(random = Math.random) {
|
|
48
|
+
const sample = Math.max(0, Math.min(1, random()));
|
|
49
|
+
return WORKER_RECONNECT_DELAY_MS + Math.floor(sample * WORKER_RECONNECT_JITTER_MS);
|
|
50
|
+
}
|
|
44
51
|
async function defaultRuntimeReaper(runtime) {
|
|
45
52
|
await (0, import_process_tree.terminateProcessTree)(runtime);
|
|
46
53
|
}
|
|
@@ -53,7 +60,7 @@ function signalExitCode(signal) {
|
|
|
53
60
|
function osSignalNumber(signal) {
|
|
54
61
|
return import_node_os.constants.signals[signal];
|
|
55
62
|
}
|
|
56
|
-
async function superviseWorkerRuntime(argv, env = process.env, spawnRuntime = defaultRuntimeSpawner, waitBeforeReconnect = defaultRuntimeDelay, reapCredentialRuntime = defaultRuntimeReaper) {
|
|
63
|
+
async function superviseWorkerRuntime(argv, env = process.env, spawnRuntime = defaultRuntimeSpawner, waitBeforeReconnect = defaultRuntimeDelay, reapCredentialRuntime = defaultRuntimeReaper, reconnectDelay = workerReconnectDelayMs) {
|
|
57
64
|
if (!process.argv[1]) {
|
|
58
65
|
throw new Error("Cannot locate the r5d-worker entrypoint");
|
|
59
66
|
}
|
|
@@ -105,9 +112,10 @@ async function superviseWorkerRuntime(argv, env = process.env, spawnRuntime = de
|
|
|
105
112
|
continue;
|
|
106
113
|
}
|
|
107
114
|
if (exitCode === WORKER_RECONNECT_EXIT_CODE) {
|
|
108
|
-
|
|
115
|
+
const reconnectDelayMs = reconnectDelay();
|
|
116
|
+
process.stderr.write(`[r5d-worker] reconnecting in ${reconnectDelayMs}ms
|
|
109
117
|
`);
|
|
110
|
-
await waitBeforeReconnect(
|
|
118
|
+
await waitBeforeReconnect(reconnectDelayMs);
|
|
111
119
|
if (receivedSignal) return signalExitCode(receivedSignal);
|
|
112
120
|
continue;
|
|
113
121
|
}
|
|
@@ -123,8 +131,10 @@ async function superviseWorkerRuntime(argv, env = process.env, spawnRuntime = de
|
|
|
123
131
|
WORKER_CREDENTIAL_RESTART_EXIT_CODE,
|
|
124
132
|
WORKER_RECONNECT_DELAY_MS,
|
|
125
133
|
WORKER_RECONNECT_EXIT_CODE,
|
|
134
|
+
WORKER_RECONNECT_JITTER_MS,
|
|
126
135
|
WORKER_RELOAD_EXIT_CODE,
|
|
127
136
|
WORKER_RUNTIME_ENV,
|
|
128
137
|
isRetryableWorkerServerStatus,
|
|
129
|
-
superviseWorkerRuntime
|
|
138
|
+
superviseWorkerRuntime,
|
|
139
|
+
workerReconnectDelayMs
|
|
130
140
|
});
|
package/dist/mjs/main.mjs
CHANGED
|
@@ -49,10 +49,10 @@ import {
|
|
|
49
49
|
isRetryableWorkerServerStatus,
|
|
50
50
|
superviseWorkerRuntime,
|
|
51
51
|
WORKER_CREDENTIAL_RESTART_EXIT_CODE,
|
|
52
|
-
WORKER_RECONNECT_DELAY_MS,
|
|
53
52
|
WORKER_RECONNECT_EXIT_CODE,
|
|
54
53
|
WORKER_RELOAD_EXIT_CODE,
|
|
55
|
-
WORKER_RUNTIME_ENV
|
|
54
|
+
WORKER_RUNTIME_ENV,
|
|
55
|
+
workerReconnectDelayMs
|
|
56
56
|
} from "./supervisor.mjs";
|
|
57
57
|
import { assertDisjointManagedRoots, managedProjectRoot, validateManagedBranchName } from "./managed-paths.mjs";
|
|
58
58
|
import {
|
|
@@ -91,6 +91,21 @@ import {
|
|
|
91
91
|
workspaceGitMountsForRemediation,
|
|
92
92
|
WorkspaceRemediationAncestryError
|
|
93
93
|
} from "./workspace-git-sync.mjs";
|
|
94
|
+
const WORKER_LIFECYCLE_DETAIL_MAX_LENGTH = 500;
|
|
95
|
+
const WORKER_SERVER_HEARTBEAT_INTERVAL_MS = 15e3;
|
|
96
|
+
const WORKER_SERVER_HEARTBEAT_TIMEOUT_MS = 2 * 60 * 1e3;
|
|
97
|
+
function boundedWorkerLifecycleProgress(progress) {
|
|
98
|
+
return progress.detail === void 0 ? progress : { ...progress, detail: progress.detail.slice(0, WORKER_LIFECYCLE_DETAIL_MAX_LENGTH) };
|
|
99
|
+
}
|
|
100
|
+
function workerServerHeartbeatAction(input) {
|
|
101
|
+
if (input.elapsedMs < input.timeoutMs) return "wait";
|
|
102
|
+
if (input.watchdogSweepDelayed || input.probeElapsedMs === void 0) return "start_probe";
|
|
103
|
+
return input.probeElapsedMs >= input.timeoutMs ? "terminate" : "wait";
|
|
104
|
+
}
|
|
105
|
+
const workerLifecycleTestHarness = {
|
|
106
|
+
boundedWorkerLifecycleProgress,
|
|
107
|
+
workerServerHeartbeatAction
|
|
108
|
+
};
|
|
94
109
|
class ProjectWorkspaceConfigurationDeferredError extends Error {
|
|
95
110
|
}
|
|
96
111
|
class WorkerServerUnavailableError extends Error {
|
|
@@ -2604,6 +2619,13 @@ async function executeCommand(input) {
|
|
|
2604
2619
|
const startedAt = Date.now();
|
|
2605
2620
|
let timeout;
|
|
2606
2621
|
let spawnedProcess;
|
|
2622
|
+
let preparationSettled = false;
|
|
2623
|
+
const settlePreparation = () => {
|
|
2624
|
+
if (preparationSettled) return;
|
|
2625
|
+
preparationSettled = true;
|
|
2626
|
+
input.setPreparationHeartbeatLease(false);
|
|
2627
|
+
};
|
|
2628
|
+
input.setPreparationHeartbeatLease(true);
|
|
2607
2629
|
try {
|
|
2608
2630
|
const targetProcessEnv = await prepareShellEnvForTarget({
|
|
2609
2631
|
target: input.resolvedTarget.target,
|
|
@@ -2641,6 +2663,7 @@ async function executeCommand(input) {
|
|
|
2641
2663
|
startedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2642
2664
|
...input.message.workspaceEffect === "none" ? { workspaceEffect: "none" } : {}
|
|
2643
2665
|
});
|
|
2666
|
+
settlePreparation();
|
|
2644
2667
|
if (input.message.timeoutMs) {
|
|
2645
2668
|
timeout = setTimeout(() => {
|
|
2646
2669
|
void terminateProcessTree(subprocess).catch((error) => {
|
|
@@ -2676,6 +2699,7 @@ async function executeCommand(input) {
|
|
|
2676
2699
|
error: error instanceof Error ? error.message : String(error)
|
|
2677
2700
|
};
|
|
2678
2701
|
} finally {
|
|
2702
|
+
settlePreparation();
|
|
2679
2703
|
if (timeout) {
|
|
2680
2704
|
clearTimeout(timeout);
|
|
2681
2705
|
}
|
|
@@ -2691,6 +2715,13 @@ async function executeStreamingCommand(input) {
|
|
|
2691
2715
|
let timedOut = false;
|
|
2692
2716
|
let timeout;
|
|
2693
2717
|
let spawnedProcess;
|
|
2718
|
+
let preparationSettled = false;
|
|
2719
|
+
const settlePreparation = () => {
|
|
2720
|
+
if (preparationSettled) return;
|
|
2721
|
+
preparationSettled = true;
|
|
2722
|
+
input.setPreparationHeartbeatLease(false);
|
|
2723
|
+
};
|
|
2724
|
+
input.setPreparationHeartbeatLease(true);
|
|
2694
2725
|
try {
|
|
2695
2726
|
if (cancelledProcessRuns.delete(input.message.runId)) {
|
|
2696
2727
|
sendWorkerMessage(input.ws, {
|
|
@@ -2743,6 +2774,7 @@ async function executeStreamingCommand(input) {
|
|
|
2743
2774
|
...interactive ? { interactive: true, stdin: subprocess.stdin } : {},
|
|
2744
2775
|
...input.message.workspaceEffect === "none" ? { workspaceEffect: "none" } : {}
|
|
2745
2776
|
});
|
|
2777
|
+
settlePreparation();
|
|
2746
2778
|
started = true;
|
|
2747
2779
|
sendWorkerMessage(input.ws, {
|
|
2748
2780
|
type: "exec_started",
|
|
@@ -2819,6 +2851,7 @@ async function executeStreamingCommand(input) {
|
|
|
2819
2851
|
});
|
|
2820
2852
|
}
|
|
2821
2853
|
} finally {
|
|
2854
|
+
settlePreparation();
|
|
2822
2855
|
if (timeout) {
|
|
2823
2856
|
clearTimeout(timeout);
|
|
2824
2857
|
}
|
|
@@ -3160,7 +3193,8 @@ function createNodePtyBridge(options) {
|
|
|
3160
3193
|
};
|
|
3161
3194
|
}
|
|
3162
3195
|
const workerPtyBridgeTestHarness = {
|
|
3163
|
-
create: createNodePtyBridge
|
|
3196
|
+
create: createNodePtyBridge,
|
|
3197
|
+
closeAllBestEffort: closePtysBestEffort
|
|
3164
3198
|
};
|
|
3165
3199
|
function resolveHostShell(command, platform = process.platform) {
|
|
3166
3200
|
if (platform === "win32") {
|
|
@@ -3309,11 +3343,23 @@ function closePty(message) {
|
|
|
3309
3343
|
}
|
|
3310
3344
|
ptyProcess.kill();
|
|
3311
3345
|
}
|
|
3312
|
-
function
|
|
3313
|
-
for (const ptyProcess of
|
|
3314
|
-
|
|
3346
|
+
function closePtysBestEffort(ptys, reportFailure) {
|
|
3347
|
+
for (const [ptyId, ptyProcess] of ptys) {
|
|
3348
|
+
try {
|
|
3349
|
+
ptyProcess.kill();
|
|
3350
|
+
} catch (error) {
|
|
3351
|
+
reportFailure(ptyId, error);
|
|
3352
|
+
}
|
|
3315
3353
|
}
|
|
3316
3354
|
}
|
|
3355
|
+
function closeAllPtys() {
|
|
3356
|
+
closePtysBestEffort(activePtys, (ptyId, error) => {
|
|
3357
|
+
process.stderr.write(
|
|
3358
|
+
`[r5d-worker] failed to close PTY ${ptyId} after control disconnect: ${error instanceof Error ? error.message : String(error)}
|
|
3359
|
+
`
|
|
3360
|
+
);
|
|
3361
|
+
});
|
|
3362
|
+
}
|
|
3317
3363
|
async function terminateCredentialBearingChildren(children) {
|
|
3318
3364
|
const results = await Promise.allSettled(children.map(async ({ terminate }) => await terminate()));
|
|
3319
3365
|
const failures = results.flatMap(
|
|
@@ -3373,16 +3419,6 @@ function websocketUrl(baseUrl, label) {
|
|
|
3373
3419
|
}
|
|
3374
3420
|
async function startWorker(options) {
|
|
3375
3421
|
advanceWorkerAdmissionGeneration();
|
|
3376
|
-
process.on("uncaughtException", (error) => {
|
|
3377
|
-
process.stderr.write(`[r5d-worker] uncaught exception: ${error instanceof Error ? error.stack ?? error.message : String(error)}
|
|
3378
|
-
`);
|
|
3379
|
-
});
|
|
3380
|
-
process.on("unhandledRejection", (reason) => {
|
|
3381
|
-
process.stderr.write(
|
|
3382
|
-
`[r5d-worker] unhandled rejection: ${reason instanceof Error ? reason.stack ?? reason.message : String(reason)}
|
|
3383
|
-
`
|
|
3384
|
-
);
|
|
3385
|
-
});
|
|
3386
3422
|
const config = readConfig(options.configPath);
|
|
3387
3423
|
const { baseUrl, token } = resolveCredentials(options, config);
|
|
3388
3424
|
initializeCredentialStoreDirectoryOnce();
|
|
@@ -4395,11 +4431,12 @@ async function startWorker(options) {
|
|
|
4395
4431
|
}
|
|
4396
4432
|
try {
|
|
4397
4433
|
assertStillAdmitted();
|
|
4434
|
+
const hasDurableBranchDeletion = projectWorkspaceState.tombstones.some((tombstone) => tombstone.kind === "branch");
|
|
4398
4435
|
return await performWorkspaceSync({
|
|
4399
4436
|
attemptId,
|
|
4400
4437
|
trigger: input.trigger,
|
|
4401
|
-
confirmedLargeDiff: input.confirmedLargeDiff,
|
|
4402
|
-
confirmationReason: input.confirmationReason,
|
|
4438
|
+
confirmedLargeDiff: input.confirmedLargeDiff === true || hasDurableBranchDeletion,
|
|
4439
|
+
confirmationReason: input.confirmationReason ?? (hasDurableBranchDeletion ? "durable project branch deletion" : void 0),
|
|
4403
4440
|
resetToCanonical: input.resetToCanonical,
|
|
4404
4441
|
requiredAncestorHeads: input.requiredAncestorHeads,
|
|
4405
4442
|
assertStillAdmitted
|
|
@@ -4936,9 +4973,12 @@ async function startWorker(options) {
|
|
|
4936
4973
|
);
|
|
4937
4974
|
}
|
|
4938
4975
|
};
|
|
4976
|
+
const hasDurableBranchDeletion = projectWorkspaceState.tombstones.some((tombstone) => tombstone.kind === "branch");
|
|
4939
4977
|
const result = await performWorkspaceSync({
|
|
4940
4978
|
attemptId: crypto.randomUUID(),
|
|
4941
4979
|
trigger: { type: "connect" },
|
|
4980
|
+
confirmedLargeDiff: hasDurableBranchDeletion,
|
|
4981
|
+
confirmationReason: hasDurableBranchDeletion ? "durable project branch deletion" : void 0,
|
|
4942
4982
|
resetToCanonical: workspaceConfigurationResetToCanonicalIsAllowed(message.resetToCanonical, incidentDeferral.incidentId),
|
|
4943
4983
|
assertStillAdmitted: assertConfigurationSyncStillAdmitted
|
|
4944
4984
|
});
|
|
@@ -5026,9 +5066,22 @@ async function startWorker(options) {
|
|
|
5026
5066
|
Authorization: `Bearer ${token}`
|
|
5027
5067
|
}
|
|
5028
5068
|
});
|
|
5069
|
+
let lastServerControlMessageAtMs = performance.now();
|
|
5070
|
+
let serverHeartbeatProbeStartedAtMs;
|
|
5071
|
+
let lastServerHeartbeatSweepAtMs = performance.now();
|
|
5072
|
+
let serverHeartbeatWatchdog;
|
|
5073
|
+
const noteServerControlActivity = () => {
|
|
5074
|
+
lastServerControlMessageAtMs = performance.now();
|
|
5075
|
+
serverHeartbeatProbeStartedAtMs = void 0;
|
|
5076
|
+
};
|
|
5077
|
+
const stopServerHeartbeatWatchdog = () => {
|
|
5078
|
+
if (!serverHeartbeatWatchdog) return;
|
|
5079
|
+
clearInterval(serverHeartbeatWatchdog);
|
|
5080
|
+
serverHeartbeatWatchdog = void 0;
|
|
5081
|
+
};
|
|
5029
5082
|
const sendLifecycleProgress = (progress) => {
|
|
5030
5083
|
if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) return;
|
|
5031
|
-
sendWorkerMessage(ws, { type: "worker_progress", progress });
|
|
5084
|
+
sendWorkerMessage(ws, { type: "worker_progress", progress: boundedWorkerLifecycleProgress(progress) });
|
|
5032
5085
|
};
|
|
5033
5086
|
const describeSnapshotProgress = (progress) => {
|
|
5034
5087
|
const project = path.relative(projectsRoot, progress.projectRoot) || path.basename(progress.projectRoot);
|
|
@@ -5109,6 +5162,29 @@ async function startWorker(options) {
|
|
|
5109
5162
|
process.on("SIGINT", handleGracefulShutdown);
|
|
5110
5163
|
ws.addEventListener("open", () => {
|
|
5111
5164
|
currentWorkerSocket = ws;
|
|
5165
|
+
noteServerControlActivity();
|
|
5166
|
+
lastServerHeartbeatSweepAtMs = performance.now();
|
|
5167
|
+
serverHeartbeatWatchdog = setInterval(() => {
|
|
5168
|
+
if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) return;
|
|
5169
|
+
const nowMs = performance.now();
|
|
5170
|
+
const heartbeatAction = workerServerHeartbeatAction({
|
|
5171
|
+
elapsedMs: Math.max(0, nowMs - lastServerControlMessageAtMs),
|
|
5172
|
+
...serverHeartbeatProbeStartedAtMs === void 0 ? {} : { probeElapsedMs: Math.max(0, nowMs - serverHeartbeatProbeStartedAtMs) },
|
|
5173
|
+
timeoutMs: WORKER_SERVER_HEARTBEAT_TIMEOUT_MS,
|
|
5174
|
+
watchdogSweepDelayed: nowMs - lastServerHeartbeatSweepAtMs >= WORKER_SERVER_HEARTBEAT_INTERVAL_MS * 2
|
|
5175
|
+
});
|
|
5176
|
+
lastServerHeartbeatSweepAtMs = nowMs;
|
|
5177
|
+
if (heartbeatAction === "start_probe") {
|
|
5178
|
+
serverHeartbeatProbeStartedAtMs = nowMs;
|
|
5179
|
+
return;
|
|
5180
|
+
}
|
|
5181
|
+
if (heartbeatAction === "terminate") {
|
|
5182
|
+
stopServerHeartbeatWatchdog();
|
|
5183
|
+
process.stderr.write("[r5d-worker] server control heartbeat timed out; reconnecting\n");
|
|
5184
|
+
ws.terminate();
|
|
5185
|
+
}
|
|
5186
|
+
}, WORKER_SERVER_HEARTBEAT_INTERVAL_MS);
|
|
5187
|
+
serverHeartbeatWatchdog.unref();
|
|
5112
5188
|
const startupOperationId = crypto.randomUUID();
|
|
5113
5189
|
startupRecoveryPromise = Promise.resolve().then(async () => await performStartupRecovery(startupOperationId)).then(() => {
|
|
5114
5190
|
startupRecoveryComplete = true;
|
|
@@ -5144,7 +5220,8 @@ async function startWorker(options) {
|
|
|
5144
5220
|
ptyEnvFilesV1: true,
|
|
5145
5221
|
workspaceRemediationAncestorGuardV1: true,
|
|
5146
5222
|
workspaceIncidentConfigDeferralV1: true,
|
|
5147
|
-
workspaceConfigResetToCanonicalV1: true
|
|
5223
|
+
workspaceConfigResetToCanonicalV1: true,
|
|
5224
|
+
projectBranchDeletionFastAckV1: true
|
|
5148
5225
|
},
|
|
5149
5226
|
projectRoot: projectsRoot,
|
|
5150
5227
|
artifactRoot,
|
|
@@ -5165,6 +5242,7 @@ async function startWorker(options) {
|
|
|
5165
5242
|
process.stdout.write("[r5d-worker] connected\n");
|
|
5166
5243
|
});
|
|
5167
5244
|
ws.addEventListener("message", (event) => {
|
|
5245
|
+
noteServerControlActivity();
|
|
5168
5246
|
void (async () => {
|
|
5169
5247
|
const message = JSON.parse(String(event.data));
|
|
5170
5248
|
if (!startupRecoveryComplete && message.type !== "ping") await startupRecoveryPromise;
|
|
@@ -5397,23 +5475,15 @@ async function startWorker(options) {
|
|
|
5397
5475
|
return true;
|
|
5398
5476
|
});
|
|
5399
5477
|
if (deletionNeedsSync) {
|
|
5400
|
-
|
|
5401
|
-
|
|
5478
|
+
scheduleAutomaticWorkspaceSync(
|
|
5479
|
+
{
|
|
5402
5480
|
type: "manual",
|
|
5403
5481
|
projectId: message.projectId,
|
|
5404
5482
|
branchName: message.branchName,
|
|
5405
5483
|
detail: "deleted project branch"
|
|
5406
5484
|
},
|
|
5407
|
-
|
|
5408
|
-
|
|
5409
|
-
sendResult: false
|
|
5410
|
-
});
|
|
5411
|
-
if (result.outcome === "failed" || result.outcome === "conflict_blocked" || result.outcome === "large_diff_blocked") {
|
|
5412
|
-
throw new Error(result.error ?? `Project branch deletion workspace sync ended with ${result.outcome}`);
|
|
5413
|
-
}
|
|
5414
|
-
if (pendingMirrorDeletes.has(deletionKey)) {
|
|
5415
|
-
throw new Error("Project branch deletion did not remove the hidden mirror ref");
|
|
5416
|
-
}
|
|
5485
|
+
0
|
|
5486
|
+
);
|
|
5417
5487
|
}
|
|
5418
5488
|
sendWorkerMessage(ws, {
|
|
5419
5489
|
type: "operation_result",
|
|
@@ -5772,6 +5842,13 @@ async function startWorker(options) {
|
|
|
5772
5842
|
token,
|
|
5773
5843
|
artifactRoot,
|
|
5774
5844
|
planRoot,
|
|
5845
|
+
setPreparationHeartbeatLease: (active) => {
|
|
5846
|
+
sendWorkerMessageFromCurrentSource(ws, {
|
|
5847
|
+
type: "heartbeat_lease",
|
|
5848
|
+
operationId: `exec:${message.runId}`,
|
|
5849
|
+
active
|
|
5850
|
+
});
|
|
5851
|
+
},
|
|
5775
5852
|
assertAdmission: () => {
|
|
5776
5853
|
if (!workspaceCommandTransportIsAllowed(deferredWorkspaceConfiguration, activeWorkspaceIncidentId, "exec")) {
|
|
5777
5854
|
throw new Error(
|
|
@@ -5835,6 +5912,13 @@ async function startWorker(options) {
|
|
|
5835
5912
|
token,
|
|
5836
5913
|
artifactRoot,
|
|
5837
5914
|
planRoot,
|
|
5915
|
+
setPreparationHeartbeatLease: (active) => {
|
|
5916
|
+
sendWorkerMessageFromCurrentSource(ws, {
|
|
5917
|
+
type: "heartbeat_lease",
|
|
5918
|
+
operationId: `exec:${message.runId}`,
|
|
5919
|
+
active
|
|
5920
|
+
});
|
|
5921
|
+
},
|
|
5838
5922
|
assertAdmission: assertMessageAdmission
|
|
5839
5923
|
});
|
|
5840
5924
|
} finally {
|
|
@@ -5967,6 +6051,7 @@ async function startWorker(options) {
|
|
|
5967
6051
|
const processEvents = process;
|
|
5968
6052
|
processEvents.off("SIGTERM", handleGracefulShutdown);
|
|
5969
6053
|
processEvents.off("SIGINT", handleGracefulShutdown);
|
|
6054
|
+
stopServerHeartbeatWatchdog();
|
|
5970
6055
|
if (workspaceAutomaticTimer) {
|
|
5971
6056
|
clearTimeout(workspaceAutomaticTimer);
|
|
5972
6057
|
workspaceAutomaticTimer = void 0;
|
|
@@ -6004,31 +6089,40 @@ async function startWorker(options) {
|
|
|
6004
6089
|
process.exit(1);
|
|
6005
6090
|
return;
|
|
6006
6091
|
}
|
|
6007
|
-
|
|
6008
|
-
|
|
6092
|
+
const reconnectDelayMs = workerReconnectDelayMs();
|
|
6093
|
+
process.stderr.write(`[r5d-worker] workspace recovery complete; reconnecting in ${reconnectDelayMs}ms
|
|
6009
6094
|
`);
|
|
6010
|
-
|
|
6011
|
-
|
|
6095
|
+
setTimeout(() => {
|
|
6096
|
+
void startWorker(options).catch((error) => {
|
|
6097
|
+
process.stderr.write(`[r5d-worker] reconnect failed: ${error instanceof Error ? error.message : String(error)}
|
|
6098
|
+
`);
|
|
6099
|
+
process.exit(error instanceof WorkerServerUnavailableError ? WORKER_RECONNECT_EXIT_CODE : 1);
|
|
6100
|
+
});
|
|
6101
|
+
}, reconnectDelayMs);
|
|
6012
6102
|
});
|
|
6013
6103
|
return;
|
|
6014
6104
|
}
|
|
6015
6105
|
if (activeProcesses.size > 0 || credentialBearingProcessGroups.size > 0 || activePtys.size > 0 || pendingProcessTerminals.size > 0) {
|
|
6016
|
-
|
|
6017
|
-
|
|
6106
|
+
const scheduleReconnect = (reconnect2) => {
|
|
6107
|
+
const reconnectDelayMs = workerReconnectDelayMs();
|
|
6108
|
+
process.stderr.write(
|
|
6109
|
+
`[r5d-worker] ${activeProcesses.size} process(es), ${credentialBearingProcessGroups.size} owned process group(s), and ${activePtys.size} PTY(s) active with ${pendingProcessTerminals.size} terminal report(s) pending; reconnecting in ${reconnectDelayMs}ms
|
|
6018
6110
|
`
|
|
6019
|
-
|
|
6111
|
+
);
|
|
6112
|
+
setTimeout(reconnect2, reconnectDelayMs);
|
|
6113
|
+
};
|
|
6020
6114
|
const reconnect = () => {
|
|
6021
6115
|
void startWorker(options).catch((error) => {
|
|
6022
6116
|
process.stderr.write(`[r5d-worker] reconnect failed: ${error instanceof Error ? error.message : String(error)}
|
|
6023
6117
|
`);
|
|
6024
6118
|
if (activeProcesses.size > 0 || credentialBearingProcessGroups.size > 0 || activePtys.size > 0 || pendingProcessTerminals.size > 0) {
|
|
6025
|
-
|
|
6119
|
+
scheduleReconnect(reconnect);
|
|
6026
6120
|
return;
|
|
6027
6121
|
}
|
|
6028
6122
|
process.exit(error instanceof WorkerServerUnavailableError ? WORKER_RECONNECT_EXIT_CODE : 1);
|
|
6029
6123
|
});
|
|
6030
6124
|
};
|
|
6031
|
-
|
|
6125
|
+
scheduleReconnect(reconnect);
|
|
6032
6126
|
return;
|
|
6033
6127
|
}
|
|
6034
6128
|
process.exit(WORKER_RECONNECT_EXIT_CODE);
|
|
@@ -6038,7 +6132,20 @@ async function startWorker(options) {
|
|
|
6038
6132
|
process.stderr.write(`[r5d-worker] websocket error${message}
|
|
6039
6133
|
`);
|
|
6040
6134
|
});
|
|
6041
|
-
await new Promise(() => {
|
|
6135
|
+
await new Promise((resolve) => {
|
|
6136
|
+
ws.addEventListener("close", () => resolve(), { once: true });
|
|
6137
|
+
});
|
|
6138
|
+
}
|
|
6139
|
+
function installWorkerRuntimeFailureLogging() {
|
|
6140
|
+
process.on("uncaughtException", (error) => {
|
|
6141
|
+
process.stderr.write(`[r5d-worker] uncaught exception: ${error instanceof Error ? error.stack ?? error.message : String(error)}
|
|
6142
|
+
`);
|
|
6143
|
+
});
|
|
6144
|
+
process.on("unhandledRejection", (reason) => {
|
|
6145
|
+
process.stderr.write(
|
|
6146
|
+
`[r5d-worker] unhandled rejection: ${reason instanceof Error ? reason.stack ?? reason.message : String(reason)}
|
|
6147
|
+
`
|
|
6148
|
+
);
|
|
6042
6149
|
});
|
|
6043
6150
|
}
|
|
6044
6151
|
async function main() {
|
|
@@ -6058,6 +6165,7 @@ async function main() {
|
|
|
6058
6165
|
}
|
|
6059
6166
|
if (process.env[WORKER_RUNTIME_ENV] === "1") {
|
|
6060
6167
|
delete process.env[WORKER_RUNTIME_ENV];
|
|
6168
|
+
installWorkerRuntimeFailureLogging();
|
|
6061
6169
|
try {
|
|
6062
6170
|
await startWorker(parsed.options);
|
|
6063
6171
|
} catch (error) {
|
|
@@ -6110,6 +6218,7 @@ export {
|
|
|
6110
6218
|
syncSessionArtifacts,
|
|
6111
6219
|
workerChildProcessEnvironment,
|
|
6112
6220
|
workerGitSecurityTestHarness,
|
|
6221
|
+
workerLifecycleTestHarness,
|
|
6113
6222
|
workerPtyBridgeTestHarness,
|
|
6114
6223
|
workerPtyTestHarness,
|
|
6115
6224
|
writeWorkerTextFile
|
package/dist/mjs/package.json
CHANGED
package/dist/mjs/supervisor.mjs
CHANGED
|
@@ -5,6 +5,7 @@ const WORKER_RELOAD_EXIT_CODE = 75;
|
|
|
5
5
|
const WORKER_RECONNECT_EXIT_CODE = 76;
|
|
6
6
|
const WORKER_CREDENTIAL_RESTART_EXIT_CODE = 77;
|
|
7
7
|
const WORKER_RECONNECT_DELAY_MS = 2e3;
|
|
8
|
+
const WORKER_RECONNECT_JITTER_MS = 3e3;
|
|
8
9
|
const SUPERVISOR_SIGNALS = ["SIGINT", "SIGTERM", "SIGHUP"];
|
|
9
10
|
function defaultRuntimeSpawner(argv, options) {
|
|
10
11
|
return Bun.spawn(argv, options);
|
|
@@ -12,6 +13,10 @@ function defaultRuntimeSpawner(argv, options) {
|
|
|
12
13
|
function defaultRuntimeDelay(delayMs) {
|
|
13
14
|
return new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
14
15
|
}
|
|
16
|
+
function workerReconnectDelayMs(random = Math.random) {
|
|
17
|
+
const sample = Math.max(0, Math.min(1, random()));
|
|
18
|
+
return WORKER_RECONNECT_DELAY_MS + Math.floor(sample * WORKER_RECONNECT_JITTER_MS);
|
|
19
|
+
}
|
|
15
20
|
async function defaultRuntimeReaper(runtime) {
|
|
16
21
|
await terminateProcessTree(runtime);
|
|
17
22
|
}
|
|
@@ -24,7 +29,7 @@ function signalExitCode(signal) {
|
|
|
24
29
|
function osSignalNumber(signal) {
|
|
25
30
|
return osConstants.signals[signal];
|
|
26
31
|
}
|
|
27
|
-
async function superviseWorkerRuntime(argv, env = process.env, spawnRuntime = defaultRuntimeSpawner, waitBeforeReconnect = defaultRuntimeDelay, reapCredentialRuntime = defaultRuntimeReaper) {
|
|
32
|
+
async function superviseWorkerRuntime(argv, env = process.env, spawnRuntime = defaultRuntimeSpawner, waitBeforeReconnect = defaultRuntimeDelay, reapCredentialRuntime = defaultRuntimeReaper, reconnectDelay = workerReconnectDelayMs) {
|
|
28
33
|
if (!process.argv[1]) {
|
|
29
34
|
throw new Error("Cannot locate the r5d-worker entrypoint");
|
|
30
35
|
}
|
|
@@ -76,9 +81,10 @@ async function superviseWorkerRuntime(argv, env = process.env, spawnRuntime = de
|
|
|
76
81
|
continue;
|
|
77
82
|
}
|
|
78
83
|
if (exitCode === WORKER_RECONNECT_EXIT_CODE) {
|
|
79
|
-
|
|
84
|
+
const reconnectDelayMs = reconnectDelay();
|
|
85
|
+
process.stderr.write(`[r5d-worker] reconnecting in ${reconnectDelayMs}ms
|
|
80
86
|
`);
|
|
81
|
-
await waitBeforeReconnect(
|
|
87
|
+
await waitBeforeReconnect(reconnectDelayMs);
|
|
82
88
|
if (receivedSignal) return signalExitCode(receivedSignal);
|
|
83
89
|
continue;
|
|
84
90
|
}
|
|
@@ -93,8 +99,10 @@ export {
|
|
|
93
99
|
WORKER_CREDENTIAL_RESTART_EXIT_CODE,
|
|
94
100
|
WORKER_RECONNECT_DELAY_MS,
|
|
95
101
|
WORKER_RECONNECT_EXIT_CODE,
|
|
102
|
+
WORKER_RECONNECT_JITTER_MS,
|
|
96
103
|
WORKER_RELOAD_EXIT_CODE,
|
|
97
104
|
WORKER_RUNTIME_ENV,
|
|
98
105
|
isRetryableWorkerServerStatus,
|
|
99
|
-
superviseWorkerRuntime
|
|
106
|
+
superviseWorkerRuntime,
|
|
107
|
+
workerReconnectDelayMs
|
|
100
108
|
};
|
package/dist/types/main.d.ts
CHANGED
|
@@ -23,6 +23,26 @@ type WorkerProjectConfig = {
|
|
|
23
23
|
baseCommitHash: string;
|
|
24
24
|
}>;
|
|
25
25
|
};
|
|
26
|
+
type WorkerLifecycleProgress = {
|
|
27
|
+
phase: "starting" | "recovering" | "configuring" | "syncing" | "ready" | "failed";
|
|
28
|
+
operationId: string;
|
|
29
|
+
detail?: string;
|
|
30
|
+
completedBytes?: number;
|
|
31
|
+
totalBytes?: number;
|
|
32
|
+
completedItems?: number;
|
|
33
|
+
totalItems?: number;
|
|
34
|
+
};
|
|
35
|
+
declare function boundedWorkerLifecycleProgress(progress: WorkerLifecycleProgress): WorkerLifecycleProgress;
|
|
36
|
+
declare function workerServerHeartbeatAction(input: {
|
|
37
|
+
elapsedMs: number;
|
|
38
|
+
probeElapsedMs?: number;
|
|
39
|
+
timeoutMs: number;
|
|
40
|
+
watchdogSweepDelayed: boolean;
|
|
41
|
+
}): "wait" | "start_probe" | "terminate";
|
|
42
|
+
export declare const workerLifecycleTestHarness: {
|
|
43
|
+
boundedWorkerLifecycleProgress: typeof boundedWorkerLifecycleProgress;
|
|
44
|
+
workerServerHeartbeatAction: typeof workerServerHeartbeatAction;
|
|
45
|
+
};
|
|
26
46
|
type WorkspaceSyncTrigger = {
|
|
27
47
|
type: "connect" | "inbound_head" | "write" | "edit" | "shell_inline" | "process_terminal" | "process_cancel" | "periodic" | "manual" | "remediation" | "remediation_confirm" | "remediation_reset";
|
|
28
48
|
sessionId?: string;
|
|
@@ -813,12 +833,16 @@ declare function createNodePtyBridge(options: {
|
|
|
813
833
|
}): WorkerPty;
|
|
814
834
|
export declare const workerPtyBridgeTestHarness: {
|
|
815
835
|
create: typeof createNodePtyBridge;
|
|
836
|
+
closeAllBestEffort: typeof closePtysBestEffort;
|
|
816
837
|
};
|
|
817
838
|
export declare function resolveHostShell(command?: string, platform?: NodeJS.Platform): {
|
|
818
839
|
file: string;
|
|
819
840
|
args: string[];
|
|
820
841
|
env?: Record<string, string>;
|
|
821
842
|
};
|
|
843
|
+
declare function closePtysBestEffort(ptys: Iterable<readonly [string, {
|
|
844
|
+
kill: () => void;
|
|
845
|
+
}]>, reportFailure: (ptyId: string, error: unknown) => void): void;
|
|
822
846
|
declare function terminateCredentialBearingChildren(children: readonly {
|
|
823
847
|
label: string;
|
|
824
848
|
terminate: () => void | Promise<void>;
|
|
@@ -4,6 +4,7 @@ export declare const WORKER_RELOAD_EXIT_CODE = 75;
|
|
|
4
4
|
export declare const WORKER_RECONNECT_EXIT_CODE = 76;
|
|
5
5
|
export declare const WORKER_CREDENTIAL_RESTART_EXIT_CODE = 77;
|
|
6
6
|
export declare const WORKER_RECONNECT_DELAY_MS = 2000;
|
|
7
|
+
export declare const WORKER_RECONNECT_JITTER_MS = 3000;
|
|
7
8
|
type RuntimeProcess = KillableSubprocess;
|
|
8
9
|
export type RuntimeSpawner = (argv: string[], options: {
|
|
9
10
|
stdin: "inherit";
|
|
@@ -14,6 +15,8 @@ export type RuntimeSpawner = (argv: string[], options: {
|
|
|
14
15
|
}) => RuntimeProcess;
|
|
15
16
|
export type RuntimeDelay = (delayMs: number) => Promise<void>;
|
|
16
17
|
export type RuntimeReaper = (runtime: RuntimeProcess) => Promise<void>;
|
|
18
|
+
export type RuntimeReconnectDelay = () => number;
|
|
19
|
+
export declare function workerReconnectDelayMs(random?: () => number): number;
|
|
17
20
|
export declare function isRetryableWorkerServerStatus(status: number): boolean;
|
|
18
|
-
export declare function superviseWorkerRuntime(argv: string[], env?: Record<string, string | undefined>, spawnRuntime?: RuntimeSpawner, waitBeforeReconnect?: RuntimeDelay, reapCredentialRuntime?: RuntimeReaper): Promise<number>;
|
|
21
|
+
export declare function superviseWorkerRuntime(argv: string[], env?: Record<string, string | undefined>, spawnRuntime?: RuntimeSpawner, waitBeforeReconnect?: RuntimeDelay, reapCredentialRuntime?: RuntimeReaper, reconnectDelay?: RuntimeReconnectDelay): Promise<number>;
|
|
19
22
|
export {};
|