@ricsam/r5d-worker 0.0.110 → 0.0.111

package/dist/cjs/main.cjs

@@ -62,6 +62,7 @@ var import_plan_lint = require("./plan-lint.cjs");
 var import_node_fs = __toESM(require("node:fs"), 1);
 var import_recovery_store = require("./recovery-store.cjs");
 var import_recovery_output_window = require("./recovery-output-window.cjs");
+var import_job_resources = require("./job-resources.cjs");
 var import_recovery_protocol = require("./recovery-protocol.cjs");
 var import_node_path = __toESM(require("node:path"), 1);
 var import_node_crypto = require("node:crypto");
@@ -154,6 +155,7 @@ const PTY_INPUT_BUSY_GRACE_MS = 3e3;
 const PTY_FOREGROUND_POLL_MS = 1e3;
 const PTY_FOREGROUND_IDLE_ENABLED = process.env.R5D_PTY_FOREGROUND_IDLE !== "0";
 const PTY_TMP_PATH_PREFIX = "r5d-worker-tmp://";
+let workerJobResources;
 const activeProcesses = /* @__PURE__ */ new Map();
 const credentialBearingProcessGroups = /* @__PURE__ */ new Map();
 const credentialBearingProcessGroupTargets = /* @__PURE__ */ new Map();
@@ -202,8 +204,23 @@ function renewWorkerExecutionLease(leaseMs) {
     () => {
       if (Date.now() < workerExecutionLeaseDeadline) return;
       workerLeaseExpired = true;
+      try {
+        workerJobResources?.cancelMatching((id) => id.startsWith("pty:"));
+      } catch (error) {
+        process.stderr.write(`[r5d-worker] expired PTY resource termination failed: ${error}
+`);
+      }
+      closeAllPtys();
       for (const operation of recoveryStore?.operations() ?? []) {
-        if (operation.sessionId && operation.state !== "completed") recoveryStore?.cancelSession(operation.sessionId);
+        if (operation.sessionId && operation.state !== "completed") {
+          recoveryStore?.cancelSession(operation.sessionId);
+          try {
+            workerJobResources?.cancelSession(operation.sessionId);
+          } catch (error) {
+            process.stderr.write(`[r5d-worker] expired command resource termination failed: ${error}
+`);
+          }
+        }
       }
       for (const [runId, active] of activeProcesses) {
         if (!active.sessionId) continue;
@@ -1597,7 +1614,7 @@ function credentialReapContractStatus(probe) {
     if (process.platform !== "linux") return "direct";
     try {
       const cgroupText = import_node_fs.default.readFileSync("/proc/self/cgroup", "utf8");
-      const unit = cgroupText.match(/(?:^|\/)r5d-worker\.service(?:$|\n)/m)?.[0]?.replace(/^\//, "").trim();
+      const unit = /(?:^|\/)r5d-worker\.service(?:\/|$|\n)/m.test(cgroupText) ? "r5d-worker.service" : void 0;
       if (unit !== "r5d-worker.service") {
         return SYSTEMD_INVOCATION_IDENTIFIER_PATTERN.test(process.env.INVOCATION_ID ?? "") ? "systemd_unverified" : "direct";
       }
@@ -1631,7 +1648,7 @@ function credentialReapContractStatus(probe) {
     }
   }
   if (resolved.platform !== "linux") return "direct";
-  if (!/(?:^|\/)r5d-worker\.service(?:$|\n)/m.test(resolved.cgroupText)) {
+  if (!/(?:^|\/)r5d-worker\.service(?:\/|$|\n)/m.test(resolved.cgroupText)) {
     return SYSTEMD_INVOCATION_IDENTIFIER_PATTERN.test(resolved.invocationId ?? "") ? "systemd_unverified" : "direct";
   }
   if (!SYSTEMD_INVOCATION_IDENTIFIER_PATTERN.test(resolved.invocationId ?? "")) return "systemd_unverified";
@@ -2013,12 +2030,6 @@ function resolveCommandCwd(branchPath, cwd) {
   }
   return resolved;
 }
-async function collectStream(stream) {
-  if (!stream) {
-    return "";
-  }
-  return await new Response(stream).text();
-}
 async function streamCommandOutput(stream, onData) {
   if (!stream) {
     return;
@@ -2739,8 +2750,37 @@ async function prepareShellEnvForTarget(input) {
     ...targetProcessEnv(input.rootDir, input.target)
   };
 }
+async function releaseWorkerJobResources(resources) {
+  if (!resources) return;
+  try {
+    await resources.release();
+  } catch (error) {
+    process.stderr.write(`[r5d-worker] job resource cleanup failed: ${error instanceof Error ? error.message : String(error)}
+`);
+  }
+}
+async function acquireWorkerCommandResources(ws, message, assertAdmission) {
+  const id = message.type === "pty_open" ? `pty:${message.ptyId}` : message.runId;
+  const operationId = message.type === "pty_open" ? id : `exec:${id}`;
+  sendWorkerMessage(ws, { type: "heartbeat_lease", operationId, active: true });
+  try {
+    return await workerJobResources.acquire({
+      id,
+      lane: message.type === "exec" && message.workspaceEffect === "none" ? "utility" : "general",
+      ..."sessionId" in message && message.sessionId ? { sessionId: message.sessionId } : {},
+      assertAdmission: () => {
+        assertAdmission();
+        if (cancelledProcessRuns.has(id)) throw new import_job_resources.WorkerJobCancelledError();
+      }
+    });
+  } finally {
+    sendWorkerMessage(ws, { type: "heartbeat_lease", operationId, active: false });
+  }
+}
 async function executeCommand(input) {
   const startedAt = Date.now();
+  const outputAbort = new AbortController();
+  let outputReaders = [];
   let timeout;
   let spawnedProcess;
   let preparationSettled = false;
@@ -2761,7 +2801,7 @@ async function executeCommand(input) {
     });
     const cwd = resolveCommandCwd(input.resolvedTarget.rootPath, input.message.cwd);
     input.assertAdmission();
-    const subprocess = Bun.spawn(input.message.argv, {
+    const subprocess = Bun.spawn(input.resources.wrap(input.message.argv), {
       cwd,
       stdout: "pipe",
       stderr: "pipe",
@@ -2789,6 +2829,7 @@ async function executeCommand(input) {
     settlePreparation();
     if (input.message.timeoutMs) {
       timeout = setTimeout(() => {
+        input.resources.kill();
         void (0, import_process_tree.terminateProcessTree)(subprocess).catch((error) => {
           process.stderr.write(
             `[r5d-worker] failed to terminate timed-out command ${input.message.runId}: ${error instanceof Error ? error.message : String(error)}
@@ -2797,21 +2838,26 @@ async function executeCommand(input) {
         });
       }, input.message.timeoutMs);
     }
-    const [stdout, stderr, exitCode] = await Promise.all([
-      collectStream(subprocess.stdout),
-      collectStream(subprocess.stderr),
-      subprocess.exited
-    ]);
+    outputReaders = [
+      (0, import_job_resources.collectWorkerInlineOutput)(subprocess.stdout, void 0, outputAbort.signal),
+      (0, import_job_resources.collectWorkerInlineOutput)(subprocess.stderr, void 0, outputAbort.signal)
+    ];
+    const [stdout, stderr, exitCode] = await Promise.all([outputReaders[0], outputReaders[1], subprocess.exited]);
     return {
       type: "exec_result",
       requestId: input.message.requestId,
       stdout,
       stderr,
       exitCode,
-      durationMs: Date.now() - startedAt
+      durationMs: Date.now() - startedAt,
+      ...input.resources.memoryLimitExceeded() ? {
+        error: `Command exceeded its worker memory limit (${input.resources.memoryBytes} bytes). Reduce parallelism or memory use and retry.`
+      } : {}
     };
   } catch (error) {
+    outputAbort.abort(error);
     if (error instanceof StaleWorkerAdmissionError) throw error;
+    input.resources.kill();
     return {
       type: "exec_result",
       requestId: input.message.requestId,
@@ -2829,6 +2875,8 @@ async function executeCommand(input) {
     if (spawnedProcess) {
       await reapCompletedCredentialBearingProcessGroup(input.message.runId, spawnedProcess);
     }
+    outputAbort.abort();
+    await Promise.allSettled(outputReaders);
     activeProcesses.delete(input.message.runId);
   }
 }
@@ -2866,7 +2914,7 @@ async function executeStreamingCommand(input) {
     const cwd = resolveCommandCwd(input.resolvedTarget.rootPath, input.message.cwd);
     const interactive = input.message.interactive === true;
     input.assertAdmission();
-    const subprocess = Bun.spawn(input.message.argv, {
+    const subprocess = Bun.spawn(input.resources.wrap(input.message.argv), {
       cwd,
       // Without an explicit stdin the process reads /dev/null and interactive
       // prompts see immediate EOF; "pipe" keeps stdin open for exec_stdin.
@@ -2909,6 +2957,7 @@ async function executeStreamingCommand(input) {
     if (input.message.timeoutMs) {
       timeout = setTimeout(() => {
         timedOut = true;
+        input.resources.kill();
         closeProcessStdin(activeProcesses.get(input.message.runId));
         void (0, import_process_tree.terminateProcessTree)(subprocess).catch((error) => {
           process.stderr.write(
@@ -2937,6 +2986,16 @@ async function executeStreamingCommand(input) {
         });
       })
     ]);
+    if (input.resources.memoryLimitExceeded()) {
+      sendWorkerMessage(input.ws, {
+        type: "exec_output",
+        runId: input.message.runId,
+        stream: "stderr",
+        data: `
+[r5d-worker] Command exceeded its memory limit (${input.resources.memoryBytes} bytes). Reduce parallelism or memory use and retry.
+`
+      });
+    }
     const canonicalWorkspaceHead = canonicalSyncTerminalHead(input.resolvedTarget);
     const terminal = {
       type: "exec_exit",
@@ -3366,6 +3425,7 @@ async function openPty(input) {
   }) : {};
   const targetEnv = targetProcessEnv(input.rootDir, input.resolvedTarget.target);
   const shell = resolveHostShell(input.message.command);
+  const shellArgv = input.resources.wrap([shell.file, ...shell.args]);
   const outputCoalescer = (0, import_pty_output_coalescer.createPtyOutputCoalescer)({
     send: (data) => {
       sendWorkerMessage(input.ws, {
@@ -3385,8 +3445,8 @@ async function openPty(input) {
   };
   try {
     const ptyProcess = createNodePtyBridge({
-      file: shell.file,
-      args: shell.args,
+      file: shellArgv[0],
+      args: shellArgv.slice(1),
       ptyOptions: {
         name: "xterm-256color",
         cols: Math.max(1, Math.min(Math.floor(input.message.cols || 80), 500)),
@@ -3415,11 +3475,18 @@ async function openPty(input) {
         outputCoalescer.push(data);
       },
       onExit: (event) => {
+        if (input.resources.memoryLimitExceeded())
+          outputCoalescer.push(
+            `\r
+[r5d-worker] Terminal exceeded its memory limit (${input.resources.memoryBytes} bytes). Reduce parallelism or memory use and retry.\r
+`
+          );
         outputCoalescer.flush();
         input.releaseWorkspaceMutation?.();
         activePtys.delete(input.message.ptyId);
         cleanupEnvFiles();
         input.onTerminal?.();
+        void releaseWorkerJobResources(input.resources);
         sendWorkerMessage(input.ws, {
           type: "pty_exit",
           ptyId: input.message.ptyId,
@@ -3433,6 +3500,7 @@ async function openPty(input) {
         activePtys.delete(input.message.ptyId);
         cleanupEnvFiles();
         input.onTerminal?.();
+        void releaseWorkerJobResources(input.resources);
         sendWorkerMessage(input.ws, {
           type: "pty_error",
           requestId: input.message.requestId,
@@ -3482,6 +3550,7 @@ function resizePty(message) {
   ptyProcess.resize(Math.max(1, Math.min(Math.floor(message.cols || 80), 500)), Math.max(1, Math.min(Math.floor(message.rows || 24), 500)));
 }
 function closePty(message) {
+  workerJobResources?.cancel(`pty:${message.ptyId}`);
   const ptyProcess = activePtys.get(message.ptyId);
   if (!ptyProcess) {
     return;
@@ -3500,7 +3569,7 @@ function closePtysBestEffort(ptys, reportFailure) {
 function closeAllPtys() {
   closePtysBestEffort(activePtys, (ptyId, error) => {
     process.stderr.write(
-      `[r5d-worker] failed to close PTY ${ptyId} after control disconnect: ${error instanceof Error ? error.message : String(error)}
+      `[r5d-worker] failed to close PTY ${ptyId} after execution lease expiry: ${error instanceof Error ? error.message : String(error)}
 `
     );
   });
@@ -3573,6 +3642,11 @@ async function startWorker(options, projectRuntime = {
   initializeCredentialStoreDirectoryOnce();
   const label = requireValue(options.label, "Missing required --label <label>");
   validateLabel(label);
+  if (!workerJobResources) {
+    workerJobResources = (0, import_job_resources.createWorkerJobResources)();
+    process.stdout.write(`[r5d-worker] job resources: ${workerJobResources.isolation}; ${JSON.stringify(workerJobResources.budget)}
+`);
+  }
   const rootDir = import_node_path.default.resolve(options.rootDir ?? process.env.R5D_ROOT ?? defaultRootDir());
   recoveryStore ??= new import_recovery_store.WorkerRecoveryStore(
     import_node_path.default.join(rootDir, ".worker-recovery", (0, import_node_crypto.createHash)("sha256").update(`${baseUrl}\0${label}`).digest("hex"), "protocol.sqlite")
@@ -5777,6 +5851,7 @@ async function startWorker(options, projectRuntime = {
       }
       if (message.type === "cancel_session") {
         recoveryStore.cancelSession(message.sessionId);
+        workerJobResources?.cancelSession(message.sessionId);
         await Promise.all(
           [...activeProcesses].filter(([, active]) => active.sessionId === message.sessionId).map(async ([runId, active]) => {
             cancelledProcessRuns.add(runId);
@@ -6263,8 +6338,14 @@ async function startWorker(options, projectRuntime = {
         return;
       }
       if (message.type === "cancel") {
+        let resourceCancellationError;
+        try {
+          workerJobResources?.cancel(message.runId);
+        } catch (error) {
+          resourceCancellationError = error instanceof Error ? error.message : String(error);
+        }
         const active = activeProcesses.get(message.runId);
-        let cancelled = true;
+        let cancelled = resourceCancellationError === void 0;
         let cancelMessage;
         if (active) {
           try {
@@ -6279,6 +6360,7 @@ async function startWorker(options, projectRuntime = {
           cancelledProcessRuns.add(message.runId);
           cancelMessage = "Cancellation queued before command start";
         }
+        if (resourceCancellationError) cancelMessage += `; job resource termination failed: ${resourceCancellationError}`;
         sendSerializedWorkerMessage(
           ws,
           JSON.stringify({
@@ -6365,13 +6447,16 @@ async function startWorker(options, projectRuntime = {
           });
           return;
         }
-        await (0, import_workspace_command_sync_policy.reserveWorkspaceCommandAfterCurrentSync)(message.target, workspaceSyncSingleFlight, () => {
-          workspaceSyncPriorityPtyTargets.set(message.ptyId, message.target);
-          if (targetMayMutateVisibleWorkspace(message.target)) recordVisibleWorkspaceMutation(message.target);
-        });
+        let resources;
+        let resourcesTransferred = false;
         let releaseWorkspaceMutation;
         let mutationLeaseTransferred = false;
         try {
+          resources = await acquireWorkerCommandResources(ws, message, assertMessageAdmission);
+          await (0, import_workspace_command_sync_policy.reserveWorkspaceCommandAfterCurrentSync)(message.target, workspaceSyncSingleFlight, () => {
+            workspaceSyncPriorityPtyTargets.set(message.ptyId, message.target);
+            if (targetMayMutateVisibleWorkspace(message.target)) recordVisibleWorkspaceMutation(message.target);
+          });
           releaseWorkspaceMutation = await (0, import_workspace_command_sync_policy.acquireWorkspaceCommandMutation)(message.target, workspaceSyncSingleFlight);
           if (workspaceOperationsAreFenced(deferredWorkspaceConfiguration, activeWorkspaceIncidentId)) {
             throw new Error(
@@ -6382,6 +6467,7 @@ async function startWorker(options, projectRuntime = {
           process.stdout.write(`[r5d-worker] pty ${message.ptyId}: ${describeWorkerSessionTarget(message.target)}
 `);
           await openPty({
+            resources,
             ws,
             message,
             resolvedTarget,
@@ -6394,6 +6480,7 @@ async function startWorker(options, projectRuntime = {
               }
             } : {}
           });
+          resourcesTransferred = true;
           mutationLeaseTransferred = releaseWorkspaceMutation !== void 0;
         } catch (error) {
           sendWorkerMessageFromCurrentSource(ws, {
@@ -6405,6 +6492,7 @@ async function startWorker(options, projectRuntime = {
         } finally {
           workspaceSyncPriorityPtyTargets.delete(message.ptyId);
           if (!mutationLeaseTransferred) releaseWorkspaceMutation?.();
+          if (!resourcesTransferred) await releaseWorkerJobResources(resources);
         }
         return;
       }
@@ -6463,9 +6551,11 @@ async function startWorker(options, projectRuntime = {
           return;
         }
         let result;
+        let resources;
         let targetReserved = false;
         const hasWorkspaceEffect = workerCommandHasWorkspaceEffect(message);
         try {
+          resources = await acquireWorkerCommandResources(ws, message, assertMessageAdmission);
           if (hasWorkspaceEffect) {
             await (0, import_workspace_command_sync_policy.reserveWorkspaceCommandAfterCurrentSync)(message.target, workspaceSyncSingleFlight, () => {
               workspaceSyncPriorityProcessTargets.set(message.runId, message.target);
@@ -6483,6 +6573,7 @@ async function startWorker(options, projectRuntime = {
             process.stdout.write(`[r5d-worker] exec ${message.runId}: ${message.argv.join(" ")}
 `);
             return await executeCommand({
+              resources,
               message,
               resolvedTarget,
               baseUrl,
@@ -6518,6 +6609,8 @@ async function startWorker(options, projectRuntime = {
           };
         } finally {
           if (targetReserved) workspaceSyncPriorityProcessTargets.delete(message.runId);
+          await releaseWorkerJobResources(resources);
+          cancelledProcessRuns.delete(message.runId);
         }
         if (infrastructureStoppedRunIds.has(message.runId)) {
           recoveryStore.unknown(message.requestId);
@@ -6549,12 +6642,14 @@ async function startWorker(options, projectRuntime = {
           runId: message.runId
         });
         const hasWorkspaceEffect = workerCommandHasWorkspaceEffect(message);
+        let resources;
         const runCommand = async () => {
           try {
             const resolvedTarget = resolveMessageTarget(message.target);
             process.stdout.write(`[r5d-worker] exec_start ${message.runId}: ${message.argv.join(" ")}
 `);
             await executeStreamingCommand({
+              resources,
               ws,
               message,
               resolvedTarget,
@@ -6588,6 +6683,7 @@ async function startWorker(options, projectRuntime = {
         const execution = (async () => {
           let targetReserved = false;
           try {
+            resources = await acquireWorkerCommandResources(ws, message, assertMessageAdmission);
             if (hasWorkspaceEffect) {
               await (0, import_workspace_command_sync_policy.reserveWorkspaceCommandAfterCurrentSync)(message.target, workspaceSyncSingleFlight, () => {
                 workspaceSyncPriorityProcessTargets.set(message.runId, message.target);
@@ -6603,6 +6699,8 @@ async function startWorker(options, projectRuntime = {
             });
           } finally {
             if (targetReserved) workspaceSyncPriorityProcessTargets.delete(message.runId);
+            await releaseWorkerJobResources(resources);
+            cancelledProcessRuns.delete(message.runId);
           }
         })();
         void execution.catch((error) => {

package/dist/cjs/package.json

@@ -1,5 +1,5 @@
 {
   "name": "@ricsam/r5d-worker",
-  "version": "0.0.110",
+  "version": "0.0.111",
   "type": "commonjs"
 }