@ricsam/r5d-worker 0.0.110 → 0.0.111

package/dist/mjs/main.mjs

@@ -5,6 +5,11 @@ import { lintPlans, formatPlanLintErrors } from "./plan-lint.mjs";
 import fs from "node:fs";
 import { WorkerRecoveryStore, workerOperationDigest } from "./recovery-store.mjs";
 import { WorkerOutputWindow } from "./recovery-output-window.mjs";
+import {
+  createWorkerJobResources,
+  collectWorkerInlineOutput,
+  WorkerJobCancelledError
+} from "./job-resources.mjs";
 import {
   WORKER_RESUMABLE_PROTOCOL,
   WORKER_RECOVERY_TIMEOUT_MS
@@ -168,6 +173,7 @@ const PTY_INPUT_BUSY_GRACE_MS = 3e3;
 const PTY_FOREGROUND_POLL_MS = 1e3;
 const PTY_FOREGROUND_IDLE_ENABLED = process.env.R5D_PTY_FOREGROUND_IDLE !== "0";
 const PTY_TMP_PATH_PREFIX = "r5d-worker-tmp://";
+let workerJobResources;
 const activeProcesses = /* @__PURE__ */ new Map();
 const credentialBearingProcessGroups = /* @__PURE__ */ new Map();
 const credentialBearingProcessGroupTargets = /* @__PURE__ */ new Map();
@@ -216,8 +222,23 @@ function renewWorkerExecutionLease(leaseMs) {
     () => {
       if (Date.now() < workerExecutionLeaseDeadline) return;
       workerLeaseExpired = true;
+      try {
+        workerJobResources?.cancelMatching((id) => id.startsWith("pty:"));
+      } catch (error) {
+        process.stderr.write(`[r5d-worker] expired PTY resource termination failed: ${error}
+`);
+      }
+      closeAllPtys();
       for (const operation of recoveryStore?.operations() ?? []) {
-        if (operation.sessionId && operation.state !== "completed") recoveryStore?.cancelSession(operation.sessionId);
+        if (operation.sessionId && operation.state !== "completed") {
+          recoveryStore?.cancelSession(operation.sessionId);
+          try {
+            workerJobResources?.cancelSession(operation.sessionId);
+          } catch (error) {
+            process.stderr.write(`[r5d-worker] expired command resource termination failed: ${error}
+`);
+          }
+        }
       }
       for (const [runId, active] of activeProcesses) {
         if (!active.sessionId) continue;
@@ -1611,7 +1632,7 @@ function credentialReapContractStatus(probe) {
     if (process.platform !== "linux") return "direct";
     try {
       const cgroupText = fs.readFileSync("/proc/self/cgroup", "utf8");
-      const unit = cgroupText.match(/(?:^|\/)r5d-worker\.service(?:$|\n)/m)?.[0]?.replace(/^\//, "").trim();
+      const unit = /(?:^|\/)r5d-worker\.service(?:\/|$|\n)/m.test(cgroupText) ? "r5d-worker.service" : void 0;
       if (unit !== "r5d-worker.service") {
         return SYSTEMD_INVOCATION_IDENTIFIER_PATTERN.test(process.env.INVOCATION_ID ?? "") ? "systemd_unverified" : "direct";
       }
@@ -1645,7 +1666,7 @@ function credentialReapContractStatus(probe) {
     }
   }
   if (resolved.platform !== "linux") return "direct";
-  if (!/(?:^|\/)r5d-worker\.service(?:$|\n)/m.test(resolved.cgroupText)) {
+  if (!/(?:^|\/)r5d-worker\.service(?:\/|$|\n)/m.test(resolved.cgroupText)) {
     return SYSTEMD_INVOCATION_IDENTIFIER_PATTERN.test(resolved.invocationId ?? "") ? "systemd_unverified" : "direct";
   }
   if (!SYSTEMD_INVOCATION_IDENTIFIER_PATTERN.test(resolved.invocationId ?? "")) return "systemd_unverified";
@@ -2027,12 +2048,6 @@ function resolveCommandCwd(branchPath, cwd) {
   }
   return resolved;
 }
-async function collectStream(stream) {
-  if (!stream) {
-    return "";
-  }
-  return await new Response(stream).text();
-}
 async function streamCommandOutput(stream, onData) {
   if (!stream) {
     return;
@@ -2753,8 +2768,37 @@ async function prepareShellEnvForTarget(input) {
     ...targetProcessEnv(input.rootDir, input.target)
   };
 }
+async function releaseWorkerJobResources(resources) {
+  if (!resources) return;
+  try {
+    await resources.release();
+  } catch (error) {
+    process.stderr.write(`[r5d-worker] job resource cleanup failed: ${error instanceof Error ? error.message : String(error)}
+`);
+  }
+}
+async function acquireWorkerCommandResources(ws, message, assertAdmission) {
+  const id = message.type === "pty_open" ? `pty:${message.ptyId}` : message.runId;
+  const operationId = message.type === "pty_open" ? id : `exec:${id}`;
+  sendWorkerMessage(ws, { type: "heartbeat_lease", operationId, active: true });
+  try {
+    return await workerJobResources.acquire({
+      id,
+      lane: message.type === "exec" && message.workspaceEffect === "none" ? "utility" : "general",
+      ..."sessionId" in message && message.sessionId ? { sessionId: message.sessionId } : {},
+      assertAdmission: () => {
+        assertAdmission();
+        if (cancelledProcessRuns.has(id)) throw new WorkerJobCancelledError();
+      }
+    });
+  } finally {
+    sendWorkerMessage(ws, { type: "heartbeat_lease", operationId, active: false });
+  }
+}
 async function executeCommand(input) {
   const startedAt = Date.now();
+  const outputAbort = new AbortController();
+  let outputReaders = [];
   let timeout;
   let spawnedProcess;
   let preparationSettled = false;
@@ -2775,7 +2819,7 @@ async function executeCommand(input) {
     });
     const cwd = resolveCommandCwd(input.resolvedTarget.rootPath, input.message.cwd);
     input.assertAdmission();
-    const subprocess = Bun.spawn(input.message.argv, {
+    const subprocess = Bun.spawn(input.resources.wrap(input.message.argv), {
       cwd,
       stdout: "pipe",
       stderr: "pipe",
@@ -2803,6 +2847,7 @@ async function executeCommand(input) {
     settlePreparation();
     if (input.message.timeoutMs) {
       timeout = setTimeout(() => {
+        input.resources.kill();
         void terminateProcessTree(subprocess).catch((error) => {
           process.stderr.write(
             `[r5d-worker] failed to terminate timed-out command ${input.message.runId}: ${error instanceof Error ? error.message : String(error)}
@@ -2811,21 +2856,26 @@ async function executeCommand(input) {
         });
       }, input.message.timeoutMs);
     }
-    const [stdout, stderr, exitCode] = await Promise.all([
-      collectStream(subprocess.stdout),
-      collectStream(subprocess.stderr),
-      subprocess.exited
-    ]);
+    outputReaders = [
+      collectWorkerInlineOutput(subprocess.stdout, void 0, outputAbort.signal),
+      collectWorkerInlineOutput(subprocess.stderr, void 0, outputAbort.signal)
+    ];
+    const [stdout, stderr, exitCode] = await Promise.all([outputReaders[0], outputReaders[1], subprocess.exited]);
     return {
       type: "exec_result",
       requestId: input.message.requestId,
       stdout,
       stderr,
       exitCode,
-      durationMs: Date.now() - startedAt
+      durationMs: Date.now() - startedAt,
+      ...input.resources.memoryLimitExceeded() ? {
+        error: `Command exceeded its worker memory limit (${input.resources.memoryBytes} bytes). Reduce parallelism or memory use and retry.`
+      } : {}
     };
   } catch (error) {
+    outputAbort.abort(error);
     if (error instanceof StaleWorkerAdmissionError) throw error;
+    input.resources.kill();
     return {
       type: "exec_result",
       requestId: input.message.requestId,
@@ -2843,6 +2893,8 @@ async function executeCommand(input) {
     if (spawnedProcess) {
       await reapCompletedCredentialBearingProcessGroup(input.message.runId, spawnedProcess);
     }
+    outputAbort.abort();
+    await Promise.allSettled(outputReaders);
     activeProcesses.delete(input.message.runId);
   }
 }
@@ -2880,7 +2932,7 @@ async function executeStreamingCommand(input) {
     const cwd = resolveCommandCwd(input.resolvedTarget.rootPath, input.message.cwd);
     const interactive = input.message.interactive === true;
     input.assertAdmission();
-    const subprocess = Bun.spawn(input.message.argv, {
+    const subprocess = Bun.spawn(input.resources.wrap(input.message.argv), {
       cwd,
       // Without an explicit stdin the process reads /dev/null and interactive
       // prompts see immediate EOF; "pipe" keeps stdin open for exec_stdin.
@@ -2923,6 +2975,7 @@ async function executeStreamingCommand(input) {
     if (input.message.timeoutMs) {
       timeout = setTimeout(() => {
         timedOut = true;
+        input.resources.kill();
         closeProcessStdin(activeProcesses.get(input.message.runId));
         void terminateProcessTree(subprocess).catch((error) => {
           process.stderr.write(
@@ -2951,6 +3004,16 @@ async function executeStreamingCommand(input) {
         });
       })
     ]);
+    if (input.resources.memoryLimitExceeded()) {
+      sendWorkerMessage(input.ws, {
+        type: "exec_output",
+        runId: input.message.runId,
+        stream: "stderr",
+        data: `
+[r5d-worker] Command exceeded its memory limit (${input.resources.memoryBytes} bytes). Reduce parallelism or memory use and retry.
+`
+      });
+    }
     const canonicalWorkspaceHead = canonicalSyncTerminalHead(input.resolvedTarget);
     const terminal = {
       type: "exec_exit",
@@ -3380,6 +3443,7 @@ async function openPty(input) {
   }) : {};
   const targetEnv = targetProcessEnv(input.rootDir, input.resolvedTarget.target);
   const shell = resolveHostShell(input.message.command);
+  const shellArgv = input.resources.wrap([shell.file, ...shell.args]);
   const outputCoalescer = createPtyOutputCoalescer({
     send: (data) => {
       sendWorkerMessage(input.ws, {
@@ -3399,8 +3463,8 @@ async function openPty(input) {
   };
   try {
     const ptyProcess = createNodePtyBridge({
-      file: shell.file,
-      args: shell.args,
+      file: shellArgv[0],
+      args: shellArgv.slice(1),
       ptyOptions: {
         name: "xterm-256color",
         cols: Math.max(1, Math.min(Math.floor(input.message.cols || 80), 500)),
@@ -3429,11 +3493,18 @@ async function openPty(input) {
         outputCoalescer.push(data);
       },
       onExit: (event) => {
+        if (input.resources.memoryLimitExceeded())
+          outputCoalescer.push(
+            `\r
+[r5d-worker] Terminal exceeded its memory limit (${input.resources.memoryBytes} bytes). Reduce parallelism or memory use and retry.\r
+`
+          );
         outputCoalescer.flush();
         input.releaseWorkspaceMutation?.();
         activePtys.delete(input.message.ptyId);
         cleanupEnvFiles();
         input.onTerminal?.();
+        void releaseWorkerJobResources(input.resources);
         sendWorkerMessage(input.ws, {
           type: "pty_exit",
           ptyId: input.message.ptyId,
@@ -3447,6 +3518,7 @@ async function openPty(input) {
         activePtys.delete(input.message.ptyId);
         cleanupEnvFiles();
         input.onTerminal?.();
+        void releaseWorkerJobResources(input.resources);
         sendWorkerMessage(input.ws, {
           type: "pty_error",
           requestId: input.message.requestId,
@@ -3496,6 +3568,7 @@ function resizePty(message) {
   ptyProcess.resize(Math.max(1, Math.min(Math.floor(message.cols || 80), 500)), Math.max(1, Math.min(Math.floor(message.rows || 24), 500)));
 }
 function closePty(message) {
+  workerJobResources?.cancel(`pty:${message.ptyId}`);
   const ptyProcess = activePtys.get(message.ptyId);
   if (!ptyProcess) {
     return;
@@ -3514,7 +3587,7 @@ function closePtysBestEffort(ptys, reportFailure) {
 function closeAllPtys() {
   closePtysBestEffort(activePtys, (ptyId, error) => {
     process.stderr.write(
-      `[r5d-worker] failed to close PTY ${ptyId} after control disconnect: ${error instanceof Error ? error.message : String(error)}
+      `[r5d-worker] failed to close PTY ${ptyId} after execution lease expiry: ${error instanceof Error ? error.message : String(error)}
 `
     );
   });
@@ -3587,6 +3660,11 @@ async function startWorker(options, projectRuntime = {
   initializeCredentialStoreDirectoryOnce();
   const label = requireValue(options.label, "Missing required --label <label>");
   validateLabel(label);
+  if (!workerJobResources) {
+    workerJobResources = createWorkerJobResources();
+    process.stdout.write(`[r5d-worker] job resources: ${workerJobResources.isolation}; ${JSON.stringify(workerJobResources.budget)}
+`);
+  }
   const rootDir = path.resolve(options.rootDir ?? process.env.R5D_ROOT ?? defaultRootDir());
   recoveryStore ??= new WorkerRecoveryStore(
     path.join(rootDir, ".worker-recovery", createHash("sha256").update(`${baseUrl}\0${label}`).digest("hex"), "protocol.sqlite")
@@ -5791,6 +5869,7 @@ async function startWorker(options, projectRuntime = {
       }
       if (message.type === "cancel_session") {
         recoveryStore.cancelSession(message.sessionId);
+        workerJobResources?.cancelSession(message.sessionId);
         await Promise.all(
           [...activeProcesses].filter(([, active]) => active.sessionId === message.sessionId).map(async ([runId, active]) => {
             cancelledProcessRuns.add(runId);
@@ -6277,8 +6356,14 @@ async function startWorker(options, projectRuntime = {
         return;
       }
       if (message.type === "cancel") {
+        let resourceCancellationError;
+        try {
+          workerJobResources?.cancel(message.runId);
+        } catch (error) {
+          resourceCancellationError = error instanceof Error ? error.message : String(error);
+        }
         const active = activeProcesses.get(message.runId);
-        let cancelled = true;
+        let cancelled = resourceCancellationError === void 0;
         let cancelMessage;
         if (active) {
           try {
@@ -6293,6 +6378,7 @@ async function startWorker(options, projectRuntime = {
           cancelledProcessRuns.add(message.runId);
           cancelMessage = "Cancellation queued before command start";
         }
+        if (resourceCancellationError) cancelMessage += `; job resource termination failed: ${resourceCancellationError}`;
         sendSerializedWorkerMessage(
           ws,
           JSON.stringify({
@@ -6379,13 +6465,16 @@ async function startWorker(options, projectRuntime = {
           });
           return;
         }
-        await reserveWorkspaceCommandAfterCurrentSync(message.target, workspaceSyncSingleFlight, () => {
-          workspaceSyncPriorityPtyTargets.set(message.ptyId, message.target);
-          if (targetMayMutateVisibleWorkspace(message.target)) recordVisibleWorkspaceMutation(message.target);
-        });
+        let resources;
+        let resourcesTransferred = false;
         let releaseWorkspaceMutation;
         let mutationLeaseTransferred = false;
         try {
+          resources = await acquireWorkerCommandResources(ws, message, assertMessageAdmission);
+          await reserveWorkspaceCommandAfterCurrentSync(message.target, workspaceSyncSingleFlight, () => {
+            workspaceSyncPriorityPtyTargets.set(message.ptyId, message.target);
+            if (targetMayMutateVisibleWorkspace(message.target)) recordVisibleWorkspaceMutation(message.target);
+          });
           releaseWorkspaceMutation = await acquireWorkspaceCommandMutation(message.target, workspaceSyncSingleFlight);
           if (workspaceOperationsAreFenced(deferredWorkspaceConfiguration, activeWorkspaceIncidentId)) {
             throw new Error(
@@ -6396,6 +6485,7 @@ async function startWorker(options, projectRuntime = {
           process.stdout.write(`[r5d-worker] pty ${message.ptyId}: ${describeWorkerSessionTarget(message.target)}
 `);
           await openPty({
+            resources,
             ws,
             message,
             resolvedTarget,
@@ -6408,6 +6498,7 @@ async function startWorker(options, projectRuntime = {
               }
             } : {}
           });
+          resourcesTransferred = true;
           mutationLeaseTransferred = releaseWorkspaceMutation !== void 0;
         } catch (error) {
           sendWorkerMessageFromCurrentSource(ws, {
@@ -6419,6 +6510,7 @@ async function startWorker(options, projectRuntime = {
         } finally {
           workspaceSyncPriorityPtyTargets.delete(message.ptyId);
           if (!mutationLeaseTransferred) releaseWorkspaceMutation?.();
+          if (!resourcesTransferred) await releaseWorkerJobResources(resources);
         }
         return;
       }
@@ -6477,9 +6569,11 @@ async function startWorker(options, projectRuntime = {
           return;
         }
         let result;
+        let resources;
         let targetReserved = false;
         const hasWorkspaceEffect = workerCommandHasWorkspaceEffect(message);
         try {
+          resources = await acquireWorkerCommandResources(ws, message, assertMessageAdmission);
           if (hasWorkspaceEffect) {
             await reserveWorkspaceCommandAfterCurrentSync(message.target, workspaceSyncSingleFlight, () => {
               workspaceSyncPriorityProcessTargets.set(message.runId, message.target);
@@ -6497,6 +6591,7 @@ async function startWorker(options, projectRuntime = {
             process.stdout.write(`[r5d-worker] exec ${message.runId}: ${message.argv.join(" ")}
 `);
             return await executeCommand({
+              resources,
               message,
               resolvedTarget,
               baseUrl,
@@ -6532,6 +6627,8 @@ async function startWorker(options, projectRuntime = {
           };
         } finally {
           if (targetReserved) workspaceSyncPriorityProcessTargets.delete(message.runId);
+          await releaseWorkerJobResources(resources);
+          cancelledProcessRuns.delete(message.runId);
         }
         if (infrastructureStoppedRunIds.has(message.runId)) {
           recoveryStore.unknown(message.requestId);
@@ -6563,12 +6660,14 @@ async function startWorker(options, projectRuntime = {
           runId: message.runId
         });
         const hasWorkspaceEffect = workerCommandHasWorkspaceEffect(message);
+        let resources;
         const runCommand = async () => {
           try {
             const resolvedTarget = resolveMessageTarget(message.target);
             process.stdout.write(`[r5d-worker] exec_start ${message.runId}: ${message.argv.join(" ")}
 `);
             await executeStreamingCommand({
+              resources,
               ws,
               message,
               resolvedTarget,
@@ -6602,6 +6701,7 @@ async function startWorker(options, projectRuntime = {
         const execution = (async () => {
           let targetReserved = false;
           try {
+            resources = await acquireWorkerCommandResources(ws, message, assertMessageAdmission);
             if (hasWorkspaceEffect) {
               await reserveWorkspaceCommandAfterCurrentSync(message.target, workspaceSyncSingleFlight, () => {
                 workspaceSyncPriorityProcessTargets.set(message.runId, message.target);
@@ -6617,6 +6717,8 @@ async function startWorker(options, projectRuntime = {
             });
           } finally {
             if (targetReserved) workspaceSyncPriorityProcessTargets.delete(message.runId);
+            await releaseWorkerJobResources(resources);
+            cancelledProcessRuns.delete(message.runId);
           }
         })();
         void execution.catch((error) => {

package/dist/mjs/package.json

@@ -1,5 +1,5 @@
 {
   "name": "@ricsam/r5d-worker",
-  "version": "0.0.110",
+  "version": "0.0.111",
   "type": "module"
 }

package/dist/types/job-resources.d.ts

@@ -0,0 +1,61 @@
+export declare const WORKER_INLINE_OUTPUT_MAX_BYTES: number;
+/** Inline RPC output lives in the control process, outside the child's cgroup. */
+export declare function collectWorkerInlineOutput(stream: ReadableStream<Uint8Array> | null, maxBytes?: number, signal?: AbortSignal): Promise<string>;
+export type WorkerJobLane = "general" | "utility";
+export type WorkerJobBudget = {
+    aggregateBytes: number;
+    generalBytes: number;
+    utilityBytes: number;
+    generalSlots: number;
+    utilitySlots: number;
+    queueTimeoutMs: number;
+    maxQueued: number;
+    pidsPerJob: number;
+};
+export declare function workerJobBudget(effectiveMemoryBytes: number, env?: NodeJS.ProcessEnv): WorkerJobBudget;
+export declare function workerDelegatedCgroupRoot(cgroupText: string, mount?: string): string | undefined;
+export declare function effectiveWorkerMemory(root: string | undefined, hostBytes?: number, mount?: string): number;
+export declare class WorkerJobCapacityError extends Error {
+    constructor();
+}
+export declare class WorkerJobCancelledError extends Error {
+    constructor();
+}
+type Acquisition = {
+    id: string;
+    lane: WorkerJobLane;
+    sessionId?: string;
+    assertAdmission: () => void;
+};
+export type WorkerJobLease = {
+    readonly cgroupPath?: string;
+    readonly memoryBytes: number;
+    wrap(argv: string[]): string[];
+    kill(): void;
+    memoryLimitExceeded(): boolean;
+    release(): Promise<void>;
+};
+/** One controller per runtime, shared by every project and socket generation. */
+export declare class WorkerJobResourceController {
+    readonly budget: WorkerJobBudget;
+    readonly isolation: "cgroup-v2" | "concurrency-only";
+    private readonly jobsRoot?;
+    private readonly active;
+    private readonly waiters;
+    private timer?;
+    constructor(budget: WorkerJobBudget, delegatedRoot?: string);
+    get status(): {
+        isolation: string;
+        active: number;
+        queued: number;
+    };
+    acquire(input: Acquisition): Promise<WorkerJobLease>;
+    cancel(id: string): void;
+    cancelSession(sessionId: string): void;
+    cancelMatching(matches: (id: string, sessionId?: string) => boolean): void;
+    private available;
+    private drain;
+    private start;
+}
+export declare function createWorkerJobResources(env?: NodeJS.ProcessEnv): WorkerJobResourceController;
+export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ricsam/r5d-worker",
-  "version": "0.0.110",
+  "version": "0.0.111",
   "type": "module",
   "main": "./dist/cjs/main.cjs",
   "module": "./dist/mjs/main.mjs",