@ricsam/r5d-worker 0.0.109 → 0.0.111
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -0
- package/dist/cjs/job-resources.cjs +335 -0
- package/dist/cjs/main.cjs +158 -37
- package/dist/cjs/package.json +1 -1
- package/dist/mjs/job-resources.mjs +293 -0
- package/dist/mjs/main.mjs +162 -37
- package/dist/mjs/package.json +1 -1
- package/dist/types/job-resources.d.ts +61 -0
- package/package.json +1 -1
package/dist/mjs/main.mjs
CHANGED
|
@@ -5,6 +5,11 @@ import { lintPlans, formatPlanLintErrors } from "./plan-lint.mjs";
|
|
|
5
5
|
import fs from "node:fs";
|
|
6
6
|
import { WorkerRecoveryStore, workerOperationDigest } from "./recovery-store.mjs";
|
|
7
7
|
import { WorkerOutputWindow } from "./recovery-output-window.mjs";
|
|
8
|
+
import {
|
|
9
|
+
createWorkerJobResources,
|
|
10
|
+
collectWorkerInlineOutput,
|
|
11
|
+
WorkerJobCancelledError
|
|
12
|
+
} from "./job-resources.mjs";
|
|
8
13
|
import {
|
|
9
14
|
WORKER_RESUMABLE_PROTOCOL,
|
|
10
15
|
WORKER_RECOVERY_TIMEOUT_MS
|
|
@@ -168,6 +173,7 @@ const PTY_INPUT_BUSY_GRACE_MS = 3e3;
|
|
|
168
173
|
const PTY_FOREGROUND_POLL_MS = 1e3;
|
|
169
174
|
const PTY_FOREGROUND_IDLE_ENABLED = process.env.R5D_PTY_FOREGROUND_IDLE !== "0";
|
|
170
175
|
const PTY_TMP_PATH_PREFIX = "r5d-worker-tmp://";
|
|
176
|
+
let workerJobResources;
|
|
171
177
|
const activeProcesses = /* @__PURE__ */ new Map();
|
|
172
178
|
const credentialBearingProcessGroups = /* @__PURE__ */ new Map();
|
|
173
179
|
const credentialBearingProcessGroupTargets = /* @__PURE__ */ new Map();
|
|
@@ -216,8 +222,23 @@ function renewWorkerExecutionLease(leaseMs) {
|
|
|
216
222
|
() => {
|
|
217
223
|
if (Date.now() < workerExecutionLeaseDeadline) return;
|
|
218
224
|
workerLeaseExpired = true;
|
|
225
|
+
try {
|
|
226
|
+
workerJobResources?.cancelMatching((id) => id.startsWith("pty:"));
|
|
227
|
+
} catch (error) {
|
|
228
|
+
process.stderr.write(`[r5d-worker] expired PTY resource termination failed: ${error}
|
|
229
|
+
`);
|
|
230
|
+
}
|
|
231
|
+
closeAllPtys();
|
|
219
232
|
for (const operation of recoveryStore?.operations() ?? []) {
|
|
220
|
-
if (operation.sessionId && operation.state !== "completed")
|
|
233
|
+
if (operation.sessionId && operation.state !== "completed") {
|
|
234
|
+
recoveryStore?.cancelSession(operation.sessionId);
|
|
235
|
+
try {
|
|
236
|
+
workerJobResources?.cancelSession(operation.sessionId);
|
|
237
|
+
} catch (error) {
|
|
238
|
+
process.stderr.write(`[r5d-worker] expired command resource termination failed: ${error}
|
|
239
|
+
`);
|
|
240
|
+
}
|
|
241
|
+
}
|
|
221
242
|
}
|
|
222
243
|
for (const [runId, active] of activeProcesses) {
|
|
223
244
|
if (!active.sessionId) continue;
|
|
@@ -1611,7 +1632,7 @@ function credentialReapContractStatus(probe) {
|
|
|
1611
1632
|
if (process.platform !== "linux") return "direct";
|
|
1612
1633
|
try {
|
|
1613
1634
|
const cgroupText = fs.readFileSync("/proc/self/cgroup", "utf8");
|
|
1614
|
-
const unit =
|
|
1635
|
+
const unit = /(?:^|\/)r5d-worker\.service(?:\/|$|\n)/m.test(cgroupText) ? "r5d-worker.service" : void 0;
|
|
1615
1636
|
if (unit !== "r5d-worker.service") {
|
|
1616
1637
|
return SYSTEMD_INVOCATION_IDENTIFIER_PATTERN.test(process.env.INVOCATION_ID ?? "") ? "systemd_unverified" : "direct";
|
|
1617
1638
|
}
|
|
@@ -1645,7 +1666,7 @@ function credentialReapContractStatus(probe) {
|
|
|
1645
1666
|
}
|
|
1646
1667
|
}
|
|
1647
1668
|
if (resolved.platform !== "linux") return "direct";
|
|
1648
|
-
if (!/(?:^|\/)r5d-worker\.service(
|
|
1669
|
+
if (!/(?:^|\/)r5d-worker\.service(?:\/|$|\n)/m.test(resolved.cgroupText)) {
|
|
1649
1670
|
return SYSTEMD_INVOCATION_IDENTIFIER_PATTERN.test(resolved.invocationId ?? "") ? "systemd_unverified" : "direct";
|
|
1650
1671
|
}
|
|
1651
1672
|
if (!SYSTEMD_INVOCATION_IDENTIFIER_PATTERN.test(resolved.invocationId ?? "")) return "systemd_unverified";
|
|
@@ -2027,12 +2048,6 @@ function resolveCommandCwd(branchPath, cwd) {
|
|
|
2027
2048
|
}
|
|
2028
2049
|
return resolved;
|
|
2029
2050
|
}
|
|
2030
|
-
async function collectStream(stream) {
|
|
2031
|
-
if (!stream) {
|
|
2032
|
-
return "";
|
|
2033
|
-
}
|
|
2034
|
-
return await new Response(stream).text();
|
|
2035
|
-
}
|
|
2036
2051
|
async function streamCommandOutput(stream, onData) {
|
|
2037
2052
|
if (!stream) {
|
|
2038
2053
|
return;
|
|
@@ -2753,8 +2768,37 @@ async function prepareShellEnvForTarget(input) {
|
|
|
2753
2768
|
...targetProcessEnv(input.rootDir, input.target)
|
|
2754
2769
|
};
|
|
2755
2770
|
}
|
|
2771
|
+
async function releaseWorkerJobResources(resources) {
|
|
2772
|
+
if (!resources) return;
|
|
2773
|
+
try {
|
|
2774
|
+
await resources.release();
|
|
2775
|
+
} catch (error) {
|
|
2776
|
+
process.stderr.write(`[r5d-worker] job resource cleanup failed: ${error instanceof Error ? error.message : String(error)}
|
|
2777
|
+
`);
|
|
2778
|
+
}
|
|
2779
|
+
}
|
|
2780
|
+
async function acquireWorkerCommandResources(ws, message, assertAdmission) {
|
|
2781
|
+
const id = message.type === "pty_open" ? `pty:${message.ptyId}` : message.runId;
|
|
2782
|
+
const operationId = message.type === "pty_open" ? id : `exec:${id}`;
|
|
2783
|
+
sendWorkerMessage(ws, { type: "heartbeat_lease", operationId, active: true });
|
|
2784
|
+
try {
|
|
2785
|
+
return await workerJobResources.acquire({
|
|
2786
|
+
id,
|
|
2787
|
+
lane: message.type === "exec" && message.workspaceEffect === "none" ? "utility" : "general",
|
|
2788
|
+
..."sessionId" in message && message.sessionId ? { sessionId: message.sessionId } : {},
|
|
2789
|
+
assertAdmission: () => {
|
|
2790
|
+
assertAdmission();
|
|
2791
|
+
if (cancelledProcessRuns.has(id)) throw new WorkerJobCancelledError();
|
|
2792
|
+
}
|
|
2793
|
+
});
|
|
2794
|
+
} finally {
|
|
2795
|
+
sendWorkerMessage(ws, { type: "heartbeat_lease", operationId, active: false });
|
|
2796
|
+
}
|
|
2797
|
+
}
|
|
2756
2798
|
async function executeCommand(input) {
|
|
2757
2799
|
const startedAt = Date.now();
|
|
2800
|
+
const outputAbort = new AbortController();
|
|
2801
|
+
let outputReaders = [];
|
|
2758
2802
|
let timeout;
|
|
2759
2803
|
let spawnedProcess;
|
|
2760
2804
|
let preparationSettled = false;
|
|
@@ -2775,7 +2819,7 @@ async function executeCommand(input) {
|
|
|
2775
2819
|
});
|
|
2776
2820
|
const cwd = resolveCommandCwd(input.resolvedTarget.rootPath, input.message.cwd);
|
|
2777
2821
|
input.assertAdmission();
|
|
2778
|
-
const subprocess = Bun.spawn(input.message.argv, {
|
|
2822
|
+
const subprocess = Bun.spawn(input.resources.wrap(input.message.argv), {
|
|
2779
2823
|
cwd,
|
|
2780
2824
|
stdout: "pipe",
|
|
2781
2825
|
stderr: "pipe",
|
|
@@ -2803,6 +2847,7 @@ async function executeCommand(input) {
|
|
|
2803
2847
|
settlePreparation();
|
|
2804
2848
|
if (input.message.timeoutMs) {
|
|
2805
2849
|
timeout = setTimeout(() => {
|
|
2850
|
+
input.resources.kill();
|
|
2806
2851
|
void terminateProcessTree(subprocess).catch((error) => {
|
|
2807
2852
|
process.stderr.write(
|
|
2808
2853
|
`[r5d-worker] failed to terminate timed-out command ${input.message.runId}: ${error instanceof Error ? error.message : String(error)}
|
|
@@ -2811,21 +2856,26 @@ async function executeCommand(input) {
|
|
|
2811
2856
|
});
|
|
2812
2857
|
}, input.message.timeoutMs);
|
|
2813
2858
|
}
|
|
2814
|
-
|
|
2815
|
-
|
|
2816
|
-
|
|
2817
|
-
|
|
2818
|
-
]);
|
|
2859
|
+
outputReaders = [
|
|
2860
|
+
collectWorkerInlineOutput(subprocess.stdout, void 0, outputAbort.signal),
|
|
2861
|
+
collectWorkerInlineOutput(subprocess.stderr, void 0, outputAbort.signal)
|
|
2862
|
+
];
|
|
2863
|
+
const [stdout, stderr, exitCode] = await Promise.all([outputReaders[0], outputReaders[1], subprocess.exited]);
|
|
2819
2864
|
return {
|
|
2820
2865
|
type: "exec_result",
|
|
2821
2866
|
requestId: input.message.requestId,
|
|
2822
2867
|
stdout,
|
|
2823
2868
|
stderr,
|
|
2824
2869
|
exitCode,
|
|
2825
|
-
durationMs: Date.now() - startedAt
|
|
2870
|
+
durationMs: Date.now() - startedAt,
|
|
2871
|
+
...input.resources.memoryLimitExceeded() ? {
|
|
2872
|
+
error: `Command exceeded its worker memory limit (${input.resources.memoryBytes} bytes). Reduce parallelism or memory use and retry.`
|
|
2873
|
+
} : {}
|
|
2826
2874
|
};
|
|
2827
2875
|
} catch (error) {
|
|
2876
|
+
outputAbort.abort(error);
|
|
2828
2877
|
if (error instanceof StaleWorkerAdmissionError) throw error;
|
|
2878
|
+
input.resources.kill();
|
|
2829
2879
|
return {
|
|
2830
2880
|
type: "exec_result",
|
|
2831
2881
|
requestId: input.message.requestId,
|
|
@@ -2843,6 +2893,8 @@ async function executeCommand(input) {
|
|
|
2843
2893
|
if (spawnedProcess) {
|
|
2844
2894
|
await reapCompletedCredentialBearingProcessGroup(input.message.runId, spawnedProcess);
|
|
2845
2895
|
}
|
|
2896
|
+
outputAbort.abort();
|
|
2897
|
+
await Promise.allSettled(outputReaders);
|
|
2846
2898
|
activeProcesses.delete(input.message.runId);
|
|
2847
2899
|
}
|
|
2848
2900
|
}
|
|
@@ -2880,7 +2932,7 @@ async function executeStreamingCommand(input) {
|
|
|
2880
2932
|
const cwd = resolveCommandCwd(input.resolvedTarget.rootPath, input.message.cwd);
|
|
2881
2933
|
const interactive = input.message.interactive === true;
|
|
2882
2934
|
input.assertAdmission();
|
|
2883
|
-
const subprocess = Bun.spawn(input.message.argv, {
|
|
2935
|
+
const subprocess = Bun.spawn(input.resources.wrap(input.message.argv), {
|
|
2884
2936
|
cwd,
|
|
2885
2937
|
// Without an explicit stdin the process reads /dev/null and interactive
|
|
2886
2938
|
// prompts see immediate EOF; "pipe" keeps stdin open for exec_stdin.
|
|
@@ -2923,6 +2975,7 @@ async function executeStreamingCommand(input) {
|
|
|
2923
2975
|
if (input.message.timeoutMs) {
|
|
2924
2976
|
timeout = setTimeout(() => {
|
|
2925
2977
|
timedOut = true;
|
|
2978
|
+
input.resources.kill();
|
|
2926
2979
|
closeProcessStdin(activeProcesses.get(input.message.runId));
|
|
2927
2980
|
void terminateProcessTree(subprocess).catch((error) => {
|
|
2928
2981
|
process.stderr.write(
|
|
@@ -2951,6 +3004,16 @@ async function executeStreamingCommand(input) {
|
|
|
2951
3004
|
});
|
|
2952
3005
|
})
|
|
2953
3006
|
]);
|
|
3007
|
+
if (input.resources.memoryLimitExceeded()) {
|
|
3008
|
+
sendWorkerMessage(input.ws, {
|
|
3009
|
+
type: "exec_output",
|
|
3010
|
+
runId: input.message.runId,
|
|
3011
|
+
stream: "stderr",
|
|
3012
|
+
data: `
|
|
3013
|
+
[r5d-worker] Command exceeded its memory limit (${input.resources.memoryBytes} bytes). Reduce parallelism or memory use and retry.
|
|
3014
|
+
`
|
|
3015
|
+
});
|
|
3016
|
+
}
|
|
2954
3017
|
const canonicalWorkspaceHead = canonicalSyncTerminalHead(input.resolvedTarget);
|
|
2955
3018
|
const terminal = {
|
|
2956
3019
|
type: "exec_exit",
|
|
@@ -3380,6 +3443,7 @@ async function openPty(input) {
|
|
|
3380
3443
|
}) : {};
|
|
3381
3444
|
const targetEnv = targetProcessEnv(input.rootDir, input.resolvedTarget.target);
|
|
3382
3445
|
const shell = resolveHostShell(input.message.command);
|
|
3446
|
+
const shellArgv = input.resources.wrap([shell.file, ...shell.args]);
|
|
3383
3447
|
const outputCoalescer = createPtyOutputCoalescer({
|
|
3384
3448
|
send: (data) => {
|
|
3385
3449
|
sendWorkerMessage(input.ws, {
|
|
@@ -3399,8 +3463,8 @@ async function openPty(input) {
|
|
|
3399
3463
|
};
|
|
3400
3464
|
try {
|
|
3401
3465
|
const ptyProcess = createNodePtyBridge({
|
|
3402
|
-
file:
|
|
3403
|
-
args:
|
|
3466
|
+
file: shellArgv[0],
|
|
3467
|
+
args: shellArgv.slice(1),
|
|
3404
3468
|
ptyOptions: {
|
|
3405
3469
|
name: "xterm-256color",
|
|
3406
3470
|
cols: Math.max(1, Math.min(Math.floor(input.message.cols || 80), 500)),
|
|
@@ -3429,11 +3493,18 @@ async function openPty(input) {
|
|
|
3429
3493
|
outputCoalescer.push(data);
|
|
3430
3494
|
},
|
|
3431
3495
|
onExit: (event) => {
|
|
3496
|
+
if (input.resources.memoryLimitExceeded())
|
|
3497
|
+
outputCoalescer.push(
|
|
3498
|
+
`\r
|
|
3499
|
+
[r5d-worker] Terminal exceeded its memory limit (${input.resources.memoryBytes} bytes). Reduce parallelism or memory use and retry.\r
|
|
3500
|
+
`
|
|
3501
|
+
);
|
|
3432
3502
|
outputCoalescer.flush();
|
|
3433
3503
|
input.releaseWorkspaceMutation?.();
|
|
3434
3504
|
activePtys.delete(input.message.ptyId);
|
|
3435
3505
|
cleanupEnvFiles();
|
|
3436
3506
|
input.onTerminal?.();
|
|
3507
|
+
void releaseWorkerJobResources(input.resources);
|
|
3437
3508
|
sendWorkerMessage(input.ws, {
|
|
3438
3509
|
type: "pty_exit",
|
|
3439
3510
|
ptyId: input.message.ptyId,
|
|
@@ -3447,6 +3518,7 @@ async function openPty(input) {
|
|
|
3447
3518
|
activePtys.delete(input.message.ptyId);
|
|
3448
3519
|
cleanupEnvFiles();
|
|
3449
3520
|
input.onTerminal?.();
|
|
3521
|
+
void releaseWorkerJobResources(input.resources);
|
|
3450
3522
|
sendWorkerMessage(input.ws, {
|
|
3451
3523
|
type: "pty_error",
|
|
3452
3524
|
requestId: input.message.requestId,
|
|
@@ -3496,6 +3568,7 @@ function resizePty(message) {
|
|
|
3496
3568
|
ptyProcess.resize(Math.max(1, Math.min(Math.floor(message.cols || 80), 500)), Math.max(1, Math.min(Math.floor(message.rows || 24), 500)));
|
|
3497
3569
|
}
|
|
3498
3570
|
function closePty(message) {
|
|
3571
|
+
workerJobResources?.cancel(`pty:${message.ptyId}`);
|
|
3499
3572
|
const ptyProcess = activePtys.get(message.ptyId);
|
|
3500
3573
|
if (!ptyProcess) {
|
|
3501
3574
|
return;
|
|
@@ -3514,7 +3587,7 @@ function closePtysBestEffort(ptys, reportFailure) {
|
|
|
3514
3587
|
function closeAllPtys() {
|
|
3515
3588
|
closePtysBestEffort(activePtys, (ptyId, error) => {
|
|
3516
3589
|
process.stderr.write(
|
|
3517
|
-
`[r5d-worker] failed to close PTY ${ptyId} after
|
|
3590
|
+
`[r5d-worker] failed to close PTY ${ptyId} after execution lease expiry: ${error instanceof Error ? error.message : String(error)}
|
|
3518
3591
|
`
|
|
3519
3592
|
);
|
|
3520
3593
|
});
|
|
@@ -3587,6 +3660,11 @@ async function startWorker(options, projectRuntime = {
|
|
|
3587
3660
|
initializeCredentialStoreDirectoryOnce();
|
|
3588
3661
|
const label = requireValue(options.label, "Missing required --label <label>");
|
|
3589
3662
|
validateLabel(label);
|
|
3663
|
+
if (!workerJobResources) {
|
|
3664
|
+
workerJobResources = createWorkerJobResources();
|
|
3665
|
+
process.stdout.write(`[r5d-worker] job resources: ${workerJobResources.isolation}; ${JSON.stringify(workerJobResources.budget)}
|
|
3666
|
+
`);
|
|
3667
|
+
}
|
|
3590
3668
|
const rootDir = path.resolve(options.rootDir ?? process.env.R5D_ROOT ?? defaultRootDir());
|
|
3591
3669
|
recoveryStore ??= new WorkerRecoveryStore(
|
|
3592
3670
|
path.join(rootDir, ".worker-recovery", createHash("sha256").update(`${baseUrl}\0${label}`).digest("hex"), "protocol.sqlite")
|
|
@@ -5791,6 +5869,7 @@ async function startWorker(options, projectRuntime = {
|
|
|
5791
5869
|
}
|
|
5792
5870
|
if (message.type === "cancel_session") {
|
|
5793
5871
|
recoveryStore.cancelSession(message.sessionId);
|
|
5872
|
+
workerJobResources?.cancelSession(message.sessionId);
|
|
5794
5873
|
await Promise.all(
|
|
5795
5874
|
[...activeProcesses].filter(([, active]) => active.sessionId === message.sessionId).map(async ([runId, active]) => {
|
|
5796
5875
|
cancelledProcessRuns.add(runId);
|
|
@@ -5854,18 +5933,13 @@ async function startWorker(options, projectRuntime = {
|
|
|
5854
5933
|
const refreshesDeferredConfiguration = deferredWorkspaceConfiguration !== null && activeWorkspaceIncidentId === null && message.deferWorkspaceSyncForIncidentId === void 0;
|
|
5855
5934
|
const digest = workerOperationDigest(message);
|
|
5856
5935
|
const cached = projectRuntime.configuration;
|
|
5857
|
-
|
|
5858
|
-
|
|
5859
|
-
|
|
5860
|
-
|
|
5861
|
-
workspaceConfigured = true;
|
|
5862
|
-
activeWorkspaceIncidentId = cached.incidentId;
|
|
5863
|
-
deferredWorkspaceConfiguration = cached.deferred;
|
|
5936
|
+
const canReuseConfiguration = cached?.digest === digest && cached.workspaceSync?.remoteUrl === message.workspaceRemoteUrl && Boolean(cached.workspaceSync.credentialHelper) && cached.workspaceSync.credentialUsername === workerCredentialUsername && cached.workspaceSync.gitIdentity?.name === message.gitIdentity.name && cached.workspaceSync.gitIdentity?.email === message.gitIdentity.email && !unsafeWorkspaceSyncBouncePending && credentialGenerationFingerprint({ ...message, workerBaseUrl: baseUrl, workerAuthHeader: bearerAuthHeader }) === configuredCredentialGenerationFingerprint;
|
|
5937
|
+
if (!canReuseConfiguration) {
|
|
5938
|
+
advanceWorkerAdmissionGeneration();
|
|
5939
|
+
projectRuntime.configuration = void 0;
|
|
5864
5940
|
}
|
|
5865
|
-
|
|
5866
|
-
|
|
5867
|
-
if (configured.result.outcome !== "failed") unsafeWorkspaceSyncRecoveryAttempts = 0;
|
|
5868
|
-
sendWorkerMessageFromCurrentSource(ws, {
|
|
5941
|
+
const configured = canReuseConfiguration ? cached.configured : await configureWorkerWorkspace(message, receiptGeneration);
|
|
5942
|
+
const sendConfigured = () => sendWorkerMessageFromCurrentSource(ws, {
|
|
5869
5943
|
type: "workspace_configured",
|
|
5870
5944
|
requestId: message.requestId,
|
|
5871
5945
|
result: configured.result,
|
|
@@ -5873,6 +5947,34 @@ async function startWorker(options, projectRuntime = {
|
|
|
5873
5947
|
aheadOfOriginBranches: configured.aheadOfOriginBranches,
|
|
5874
5948
|
...configured.deferredWorkspaceSyncForIncidentId ? { deferredWorkspaceSyncForIncidentId: configured.deferredWorkspaceSyncForIncidentId } : {}
|
|
5875
5949
|
});
|
|
5950
|
+
if (receiptGeneration !== workspaceConfigurationReceiptGeneration || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
|
|
5951
|
+
sendConfigured();
|
|
5952
|
+
return;
|
|
5953
|
+
}
|
|
5954
|
+
if (canReuseConfiguration && configured.result.outcome !== "failed") {
|
|
5955
|
+
workspaceRemoteUrl = cached.workspaceSync.remoteUrl;
|
|
5956
|
+
workspaceCredentialHelper = cached.workspaceSync.credentialHelper;
|
|
5957
|
+
workspaceCredentialUsername = cached.workspaceSync.credentialUsername;
|
|
5958
|
+
workspaceGitIdentity = cached.workspaceSync.gitIdentity;
|
|
5959
|
+
workspaceConfigured = true;
|
|
5960
|
+
activeWorkspaceIncidentId = cached.incidentId;
|
|
5961
|
+
deferredWorkspaceConfiguration = cached.deferred;
|
|
5962
|
+
}
|
|
5963
|
+
projectRuntime.configuration = configured.result.outcome !== "failed" && workspaceRemoteUrl && workspaceCredentialHelper && workspaceCredentialUsername && workspaceGitIdentity ? {
|
|
5964
|
+
digest,
|
|
5965
|
+
workspaceSync: {
|
|
5966
|
+
remoteUrl: workspaceRemoteUrl,
|
|
5967
|
+
credentialHelper: workspaceCredentialHelper,
|
|
5968
|
+
credentialUsername: workspaceCredentialUsername,
|
|
5969
|
+
gitIdentity: workspaceGitIdentity
|
|
5970
|
+
},
|
|
5971
|
+
configured,
|
|
5972
|
+
incidentId: activeWorkspaceIncidentId,
|
|
5973
|
+
deferred: deferredWorkspaceConfiguration
|
|
5974
|
+
} : void 0;
|
|
5975
|
+
unsafeWorkspaceSyncBouncePending = false;
|
|
5976
|
+
if (configured.result.outcome !== "failed") unsafeWorkspaceSyncRecoveryAttempts = 0;
|
|
5977
|
+
sendConfigured();
|
|
5876
5978
|
sendLifecycleProgress(
|
|
5877
5979
|
configured.result.outcome === "failed" ? {
|
|
5878
5980
|
phase: "failed",
|
|
@@ -6254,8 +6356,14 @@ async function startWorker(options, projectRuntime = {
|
|
|
6254
6356
|
return;
|
|
6255
6357
|
}
|
|
6256
6358
|
if (message.type === "cancel") {
|
|
6359
|
+
let resourceCancellationError;
|
|
6360
|
+
try {
|
|
6361
|
+
workerJobResources?.cancel(message.runId);
|
|
6362
|
+
} catch (error) {
|
|
6363
|
+
resourceCancellationError = error instanceof Error ? error.message : String(error);
|
|
6364
|
+
}
|
|
6257
6365
|
const active = activeProcesses.get(message.runId);
|
|
6258
|
-
let cancelled =
|
|
6366
|
+
let cancelled = resourceCancellationError === void 0;
|
|
6259
6367
|
let cancelMessage;
|
|
6260
6368
|
if (active) {
|
|
6261
6369
|
try {
|
|
@@ -6270,6 +6378,7 @@ async function startWorker(options, projectRuntime = {
|
|
|
6270
6378
|
cancelledProcessRuns.add(message.runId);
|
|
6271
6379
|
cancelMessage = "Cancellation queued before command start";
|
|
6272
6380
|
}
|
|
6381
|
+
if (resourceCancellationError) cancelMessage += `; job resource termination failed: ${resourceCancellationError}`;
|
|
6273
6382
|
sendSerializedWorkerMessage(
|
|
6274
6383
|
ws,
|
|
6275
6384
|
JSON.stringify({
|
|
@@ -6356,13 +6465,16 @@ async function startWorker(options, projectRuntime = {
|
|
|
6356
6465
|
});
|
|
6357
6466
|
return;
|
|
6358
6467
|
}
|
|
6359
|
-
|
|
6360
|
-
|
|
6361
|
-
if (targetMayMutateVisibleWorkspace(message.target)) recordVisibleWorkspaceMutation(message.target);
|
|
6362
|
-
});
|
|
6468
|
+
let resources;
|
|
6469
|
+
let resourcesTransferred = false;
|
|
6363
6470
|
let releaseWorkspaceMutation;
|
|
6364
6471
|
let mutationLeaseTransferred = false;
|
|
6365
6472
|
try {
|
|
6473
|
+
resources = await acquireWorkerCommandResources(ws, message, assertMessageAdmission);
|
|
6474
|
+
await reserveWorkspaceCommandAfterCurrentSync(message.target, workspaceSyncSingleFlight, () => {
|
|
6475
|
+
workspaceSyncPriorityPtyTargets.set(message.ptyId, message.target);
|
|
6476
|
+
if (targetMayMutateVisibleWorkspace(message.target)) recordVisibleWorkspaceMutation(message.target);
|
|
6477
|
+
});
|
|
6366
6478
|
releaseWorkspaceMutation = await acquireWorkspaceCommandMutation(message.target, workspaceSyncSingleFlight);
|
|
6367
6479
|
if (workspaceOperationsAreFenced(deferredWorkspaceConfiguration, activeWorkspaceIncidentId)) {
|
|
6368
6480
|
throw new Error(
|
|
@@ -6373,6 +6485,7 @@ async function startWorker(options, projectRuntime = {
|
|
|
6373
6485
|
process.stdout.write(`[r5d-worker] pty ${message.ptyId}: ${describeWorkerSessionTarget(message.target)}
|
|
6374
6486
|
`);
|
|
6375
6487
|
await openPty({
|
|
6488
|
+
resources,
|
|
6376
6489
|
ws,
|
|
6377
6490
|
message,
|
|
6378
6491
|
resolvedTarget,
|
|
@@ -6385,6 +6498,7 @@ async function startWorker(options, projectRuntime = {
|
|
|
6385
6498
|
}
|
|
6386
6499
|
} : {}
|
|
6387
6500
|
});
|
|
6501
|
+
resourcesTransferred = true;
|
|
6388
6502
|
mutationLeaseTransferred = releaseWorkspaceMutation !== void 0;
|
|
6389
6503
|
} catch (error) {
|
|
6390
6504
|
sendWorkerMessageFromCurrentSource(ws, {
|
|
@@ -6396,6 +6510,7 @@ async function startWorker(options, projectRuntime = {
|
|
|
6396
6510
|
} finally {
|
|
6397
6511
|
workspaceSyncPriorityPtyTargets.delete(message.ptyId);
|
|
6398
6512
|
if (!mutationLeaseTransferred) releaseWorkspaceMutation?.();
|
|
6513
|
+
if (!resourcesTransferred) await releaseWorkerJobResources(resources);
|
|
6399
6514
|
}
|
|
6400
6515
|
return;
|
|
6401
6516
|
}
|
|
@@ -6454,9 +6569,11 @@ async function startWorker(options, projectRuntime = {
|
|
|
6454
6569
|
return;
|
|
6455
6570
|
}
|
|
6456
6571
|
let result;
|
|
6572
|
+
let resources;
|
|
6457
6573
|
let targetReserved = false;
|
|
6458
6574
|
const hasWorkspaceEffect = workerCommandHasWorkspaceEffect(message);
|
|
6459
6575
|
try {
|
|
6576
|
+
resources = await acquireWorkerCommandResources(ws, message, assertMessageAdmission);
|
|
6460
6577
|
if (hasWorkspaceEffect) {
|
|
6461
6578
|
await reserveWorkspaceCommandAfterCurrentSync(message.target, workspaceSyncSingleFlight, () => {
|
|
6462
6579
|
workspaceSyncPriorityProcessTargets.set(message.runId, message.target);
|
|
@@ -6474,6 +6591,7 @@ async function startWorker(options, projectRuntime = {
|
|
|
6474
6591
|
process.stdout.write(`[r5d-worker] exec ${message.runId}: ${message.argv.join(" ")}
|
|
6475
6592
|
`);
|
|
6476
6593
|
return await executeCommand({
|
|
6594
|
+
resources,
|
|
6477
6595
|
message,
|
|
6478
6596
|
resolvedTarget,
|
|
6479
6597
|
baseUrl,
|
|
@@ -6509,6 +6627,8 @@ async function startWorker(options, projectRuntime = {
|
|
|
6509
6627
|
};
|
|
6510
6628
|
} finally {
|
|
6511
6629
|
if (targetReserved) workspaceSyncPriorityProcessTargets.delete(message.runId);
|
|
6630
|
+
await releaseWorkerJobResources(resources);
|
|
6631
|
+
cancelledProcessRuns.delete(message.runId);
|
|
6512
6632
|
}
|
|
6513
6633
|
if (infrastructureStoppedRunIds.has(message.runId)) {
|
|
6514
6634
|
recoveryStore.unknown(message.requestId);
|
|
@@ -6540,12 +6660,14 @@ async function startWorker(options, projectRuntime = {
|
|
|
6540
6660
|
runId: message.runId
|
|
6541
6661
|
});
|
|
6542
6662
|
const hasWorkspaceEffect = workerCommandHasWorkspaceEffect(message);
|
|
6663
|
+
let resources;
|
|
6543
6664
|
const runCommand = async () => {
|
|
6544
6665
|
try {
|
|
6545
6666
|
const resolvedTarget = resolveMessageTarget(message.target);
|
|
6546
6667
|
process.stdout.write(`[r5d-worker] exec_start ${message.runId}: ${message.argv.join(" ")}
|
|
6547
6668
|
`);
|
|
6548
6669
|
await executeStreamingCommand({
|
|
6670
|
+
resources,
|
|
6549
6671
|
ws,
|
|
6550
6672
|
message,
|
|
6551
6673
|
resolvedTarget,
|
|
@@ -6579,6 +6701,7 @@ async function startWorker(options, projectRuntime = {
|
|
|
6579
6701
|
const execution = (async () => {
|
|
6580
6702
|
let targetReserved = false;
|
|
6581
6703
|
try {
|
|
6704
|
+
resources = await acquireWorkerCommandResources(ws, message, assertMessageAdmission);
|
|
6582
6705
|
if (hasWorkspaceEffect) {
|
|
6583
6706
|
await reserveWorkspaceCommandAfterCurrentSync(message.target, workspaceSyncSingleFlight, () => {
|
|
6584
6707
|
workspaceSyncPriorityProcessTargets.set(message.runId, message.target);
|
|
@@ -6594,6 +6717,8 @@ async function startWorker(options, projectRuntime = {
|
|
|
6594
6717
|
});
|
|
6595
6718
|
} finally {
|
|
6596
6719
|
if (targetReserved) workspaceSyncPriorityProcessTargets.delete(message.runId);
|
|
6720
|
+
await releaseWorkerJobResources(resources);
|
|
6721
|
+
cancelledProcessRuns.delete(message.runId);
|
|
6597
6722
|
}
|
|
6598
6723
|
})();
|
|
6599
6724
|
void execution.catch((error) => {
|
|
@@ -6715,7 +6840,7 @@ async function startWorker(options, projectRuntime = {
|
|
|
6715
6840
|
clearInterval(terminalReplayTimer);
|
|
6716
6841
|
terminalReplayTimer = void 0;
|
|
6717
6842
|
}
|
|
6718
|
-
if (projectRuntime.configuration) {
|
|
6843
|
+
if (currentWorkerSocket === ws && projectRuntime.configuration) {
|
|
6719
6844
|
projectRuntime.configuration.incidentId = activeWorkspaceIncidentId;
|
|
6720
6845
|
projectRuntime.configuration.deferred = deferredWorkspaceConfiguration;
|
|
6721
6846
|
}
|
package/dist/mjs/package.json
CHANGED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
export declare const WORKER_INLINE_OUTPUT_MAX_BYTES: number;
|
|
2
|
+
/** Inline RPC output lives in the control process, outside the child's cgroup. */
|
|
3
|
+
export declare function collectWorkerInlineOutput(stream: ReadableStream<Uint8Array> | null, maxBytes?: number, signal?: AbortSignal): Promise<string>;
|
|
4
|
+
export type WorkerJobLane = "general" | "utility";
|
|
5
|
+
export type WorkerJobBudget = {
|
|
6
|
+
aggregateBytes: number;
|
|
7
|
+
generalBytes: number;
|
|
8
|
+
utilityBytes: number;
|
|
9
|
+
generalSlots: number;
|
|
10
|
+
utilitySlots: number;
|
|
11
|
+
queueTimeoutMs: number;
|
|
12
|
+
maxQueued: number;
|
|
13
|
+
pidsPerJob: number;
|
|
14
|
+
};
|
|
15
|
+
export declare function workerJobBudget(effectiveMemoryBytes: number, env?: NodeJS.ProcessEnv): WorkerJobBudget;
|
|
16
|
+
export declare function workerDelegatedCgroupRoot(cgroupText: string, mount?: string): string | undefined;
|
|
17
|
+
export declare function effectiveWorkerMemory(root: string | undefined, hostBytes?: number, mount?: string): number;
|
|
18
|
+
export declare class WorkerJobCapacityError extends Error {
|
|
19
|
+
constructor();
|
|
20
|
+
}
|
|
21
|
+
export declare class WorkerJobCancelledError extends Error {
|
|
22
|
+
constructor();
|
|
23
|
+
}
|
|
24
|
+
type Acquisition = {
|
|
25
|
+
id: string;
|
|
26
|
+
lane: WorkerJobLane;
|
|
27
|
+
sessionId?: string;
|
|
28
|
+
assertAdmission: () => void;
|
|
29
|
+
};
|
|
30
|
+
export type WorkerJobLease = {
|
|
31
|
+
readonly cgroupPath?: string;
|
|
32
|
+
readonly memoryBytes: number;
|
|
33
|
+
wrap(argv: string[]): string[];
|
|
34
|
+
kill(): void;
|
|
35
|
+
memoryLimitExceeded(): boolean;
|
|
36
|
+
release(): Promise<void>;
|
|
37
|
+
};
|
|
38
|
+
/** One controller per runtime, shared by every project and socket generation. */
|
|
39
|
+
export declare class WorkerJobResourceController {
|
|
40
|
+
readonly budget: WorkerJobBudget;
|
|
41
|
+
readonly isolation: "cgroup-v2" | "concurrency-only";
|
|
42
|
+
private readonly jobsRoot?;
|
|
43
|
+
private readonly active;
|
|
44
|
+
private readonly waiters;
|
|
45
|
+
private timer?;
|
|
46
|
+
constructor(budget: WorkerJobBudget, delegatedRoot?: string);
|
|
47
|
+
get status(): {
|
|
48
|
+
isolation: string;
|
|
49
|
+
active: number;
|
|
50
|
+
queued: number;
|
|
51
|
+
};
|
|
52
|
+
acquire(input: Acquisition): Promise<WorkerJobLease>;
|
|
53
|
+
cancel(id: string): void;
|
|
54
|
+
cancelSession(sessionId: string): void;
|
|
55
|
+
cancelMatching(matches: (id: string, sessionId?: string) => boolean): void;
|
|
56
|
+
private available;
|
|
57
|
+
private drain;
|
|
58
|
+
private start;
|
|
59
|
+
}
|
|
60
|
+
export declare function createWorkerJobResources(env?: NodeJS.ProcessEnv): WorkerJobResourceController;
|
|
61
|
+
export {};
|