@ricsam/r5d-worker 0.0.110 → 0.0.112

package/dist/mjs/main.mjs

@@ -5,6 +5,13 @@ import { lintPlans, formatPlanLintErrors } from "./plan-lint.mjs";
 import fs from "node:fs";
 import { WorkerRecoveryStore, workerOperationDigest } from "./recovery-store.mjs";
 import { WorkerOutputWindow } from "./recovery-output-window.mjs";
+import {
+  createWorkerCommandLauncher,
+  commandLauncherEnvironment,
+  CommandLaunchCancelledError,
+  CommandLauncherUnavailableError
+} from "./command-launcher.mjs";
+import { collectWorkerInlineOutput } from "./inline-output.mjs";
 import {
   WORKER_RESUMABLE_PROTOCOL,
   WORKER_RECOVERY_TIMEOUT_MS
@@ -168,6 +175,7 @@ const PTY_INPUT_BUSY_GRACE_MS = 3e3;
 const PTY_FOREGROUND_POLL_MS = 1e3;
 const PTY_FOREGROUND_IDLE_ENABLED = process.env.R5D_PTY_FOREGROUND_IDLE !== "0";
 const PTY_TMP_PATH_PREFIX = "r5d-worker-tmp://";
+let workerCommandLauncher;
 const activeProcesses = /* @__PURE__ */ new Map();
 const credentialBearingProcessGroups = /* @__PURE__ */ new Map();
 const credentialBearingProcessGroupTargets = /* @__PURE__ */ new Map();
@@ -187,6 +195,41 @@ let workerExecutionLeaseDeadline = 0;
 let workerExecutionLeaseTimer;
 let workerLeaseExpired = false;
 const infrastructureStoppedRunIds = /* @__PURE__ */ new Set();
+let launcherFailureInProgress = false;
+let runtimeExitPromise;
+function fenceWorkerRuntimeOperations() {
+  launcherFailureInProgress = true;
+  workerAdmissionGeneration += 1;
+  for (const operation of recoveryStore?.operations() ?? []) {
+    if (operation.state !== "completed") recoveryStore?.unknown(operation.requestId);
+  }
+  for (const runId of workerCommandLauncher?.activeIds ?? []) {
+    infrastructureStoppedRunIds.add(runId);
+    cancelledProcessRuns.add(runId);
+    recoveryStore?.unknownRun(runId);
+  }
+  for (const runId of activeProcesses.keys()) infrastructureStoppedRunIds.add(runId);
+}
+function exitWorkerRuntime(code) {
+  if (runtimeExitPromise) return runtimeExitPromise;
+  fenceWorkerRuntimeOperations();
+  runtimeExitPromise = (async () => {
+    await Promise.race([
+      Promise.allSettled([terminateActiveCredentialBearingChildren(), workerCommandLauncher?.close()]),
+      Bun.sleep(4e3)
+    ]);
+    process.exit(code);
+  })();
+  return runtimeExitPromise;
+}
+function handleWorkerCommandLauncherFailure(error) {
+  if (!workerCommandLauncher || launcherFailureInProgress) return;
+  fenceWorkerRuntimeOperations();
+  process.stderr.write(`[r5d-worker] ${error.message}; stopping runtime for communication recovery
+`);
+  currentWorkerSocket?.close(1012, "Command launcher recovery");
+  void exitWorkerRuntime(WORKER_RECONNECT_EXIT_CODE);
+}
 function isWorkerCommunicationFailure(message) {
   return /Could not resolve host|Couldn't connect|Failed to connect|Connection reset|Connection timed out|remote end hung up|Empty reply|requested URL returned error: (?:408|429|5\d\d)|fetch failed|socket connection|ECONNRESET|ECONNREFUSED|ETIMEDOUT|Unable to connect|network connection was lost|Worker communication recovery expired|operation was aborted due to timeout|operation timed out|Failed to (?:fetch|download).*\((?:408|429|5\d\d)\)/i.test(
     message
@@ -216,8 +259,19 @@ function renewWorkerExecutionLease(leaseMs) {
     () => {
       if (Date.now() < workerExecutionLeaseDeadline) return;
       workerLeaseExpired = true;
+      void workerCommandLauncher?.cancelMatching((id) => id.startsWith("pty:")).catch((error) => {
+        process.stderr.write(`[r5d-worker] expired PTY launcher cancellation failed: ${error}
+`);
+      });
+      closeAllPtys();
       for (const operation of recoveryStore?.operations() ?? []) {
-        if (operation.sessionId && operation.state !== "completed") recoveryStore?.cancelSession(operation.sessionId);
+        if (operation.sessionId && operation.state !== "completed") {
+          recoveryStore?.cancelSession(operation.sessionId);
+          void workerCommandLauncher?.cancelSession(operation.sessionId).catch((error) => {
+            process.stderr.write(`[r5d-worker] expired command launcher cancellation failed: ${error}
+`);
+          });
+        }
       }
       for (const [runId, active] of activeProcesses) {
         if (!active.sessionId) continue;
@@ -1581,7 +1635,9 @@ async function fenceCredentialGenerationAtConfigureEntry(input) {
   await input.terminatePreviousGeneration();
   return true;
 }
-function requireCredentialGenerationRestart(required, fatalExit = (exitCode2) => process.exit(exitCode2), exitCode = 1) {
+function requireCredentialGenerationRestart(required, fatalExit = (exitCode2) => {
+  void exitWorkerRuntime(exitCode2);
+}, exitCode = 1) {
   if (!required) return;
   fatalExit(exitCode);
   throw new Error("Credential generation restart callback returned unexpectedly");
@@ -1611,7 +1667,7 @@ function credentialReapContractStatus(probe) {
     if (process.platform !== "linux") return "direct";
     try {
       const cgroupText = fs.readFileSync("/proc/self/cgroup", "utf8");
-      const unit = cgroupText.match(/(?:^|\/)r5d-worker\.service(?:$|\n)/m)?.[0]?.replace(/^\//, "").trim();
+      const unit = /(?:^|\/)r5d-worker\.service(?:\/|$|\n)/m.test(cgroupText) ? "r5d-worker.service" : void 0;
       if (unit !== "r5d-worker.service") {
         return SYSTEMD_INVOCATION_IDENTIFIER_PATTERN.test(process.env.INVOCATION_ID ?? "") ? "systemd_unverified" : "direct";
       }
@@ -1645,7 +1701,7 @@ function credentialReapContractStatus(probe) {
     }
   }
   if (resolved.platform !== "linux") return "direct";
-  if (!/(?:^|\/)r5d-worker\.service(?:$|\n)/m.test(resolved.cgroupText)) {
+  if (!/(?:^|\/)r5d-worker\.service(?:\/|$|\n)/m.test(resolved.cgroupText)) {
     return SYSTEMD_INVOCATION_IDENTIFIER_PATTERN.test(resolved.invocationId ?? "") ? "systemd_unverified" : "direct";
   }
   if (!SYSTEMD_INVOCATION_IDENTIFIER_PATTERN.test(resolved.invocationId ?? "")) return "systemd_unverified";
@@ -2027,12 +2083,6 @@ function resolveCommandCwd(branchPath, cwd) {
   }
   return resolved;
 }
-async function collectStream(stream) {
-  if (!stream) {
-    return "";
-  }
-  return await new Response(stream).text();
-}
 async function streamCommandOutput(stream, onData) {
   if (!stream) {
     return;
@@ -2753,8 +2803,38 @@ async function prepareShellEnvForTarget(input) {
     ...targetProcessEnv(input.rootDir, input.target)
   };
 }
+async function releaseWorkerCommandLaunch(resources) {
+  await resources?.release();
+}
+function cancelWorkerCommandLaunch(resources) {
+  void resources.cancel().catch((error) => {
+    process.stderr.write(`[r5d-worker] command launcher cancellation failed: ${error instanceof Error ? error.message : String(error)}
+`);
+  });
+}
+async function acquireWorkerCommandLaunch(ws, message, assertAdmission) {
+  const id = message.type === "pty_open" ? `pty:${message.ptyId}` : message.runId;
+  const operationId = message.type === "pty_open" ? id : `exec:${id}`;
+  sendWorkerMessage(ws, { type: "heartbeat_lease", operationId, active: true });
+  try {
+    return await workerCommandLauncher.acquire({
+      id,
+      kind: message.type,
+      ..."workspaceEffect" in message && message.workspaceEffect === "none" ? { workspaceEffect: "none" } : {},
+      ..."sessionId" in message && message.sessionId ? { sessionId: message.sessionId } : {},
+      assertAdmission: () => {
+        assertAdmission();
+        if (cancelledProcessRuns.has(id)) throw new CommandLaunchCancelledError();
+      }
+    });
+  } finally {
+    sendWorkerMessage(ws, { type: "heartbeat_lease", operationId, active: false });
+  }
+}
 async function executeCommand(input) {
   const startedAt = Date.now();
+  const outputAbort = new AbortController();
+  let outputReaders = [];
   let timeout;
   let spawnedProcess;
   let preparationSettled = false;
@@ -2775,7 +2855,7 @@ async function executeCommand(input) {
     });
     const cwd = resolveCommandCwd(input.resolvedTarget.rootPath, input.message.cwd);
     input.assertAdmission();
-    const subprocess = Bun.spawn(input.message.argv, {
+    const subprocess = Bun.spawn(input.resources.wrap(input.message.argv), {
       cwd,
       stdout: "pipe",
       stderr: "pipe",
@@ -2803,6 +2883,7 @@ async function executeCommand(input) {
     settlePreparation();
     if (input.message.timeoutMs) {
       timeout = setTimeout(() => {
+        cancelWorkerCommandLaunch(input.resources);
         void terminateProcessTree(subprocess).catch((error) => {
           process.stderr.write(
             `[r5d-worker] failed to terminate timed-out command ${input.message.runId}: ${error instanceof Error ? error.message : String(error)}
@@ -2811,21 +2892,25 @@ async function executeCommand(input) {
         });
       }, input.message.timeoutMs);
     }
-    const [stdout, stderr, exitCode] = await Promise.all([
-      collectStream(subprocess.stdout),
-      collectStream(subprocess.stderr),
-      subprocess.exited
-    ]);
+    outputReaders = [
+      collectWorkerInlineOutput(subprocess.stdout, void 0, outputAbort.signal),
+      collectWorkerInlineOutput(subprocess.stderr, void 0, outputAbort.signal)
+    ];
+    const [stdout, stderr, exitCode] = await Promise.all([outputReaders[0], outputReaders[1], subprocess.exited]);
+    const diagnostic = await input.resources.diagnose();
     return {
       type: "exec_result",
       requestId: input.message.requestId,
       stdout,
       stderr,
       exitCode,
-      durationMs: Date.now() - startedAt
+      durationMs: Date.now() - startedAt,
+      ...diagnostic ? { error: diagnostic } : {}
     };
   } catch (error) {
-    if (error instanceof StaleWorkerAdmissionError) throw error;
+    outputAbort.abort(error);
+    if (error instanceof StaleWorkerAdmissionError || error instanceof CommandLauncherUnavailableError) throw error;
+    cancelWorkerCommandLaunch(input.resources);
     return {
       type: "exec_result",
       requestId: input.message.requestId,
@@ -2843,12 +2928,16 @@ async function executeCommand(input) {
     if (spawnedProcess) {
       await reapCompletedCredentialBearingProcessGroup(input.message.runId, spawnedProcess);
     }
+    outputAbort.abort();
+    await Promise.allSettled(outputReaders);
+    await releaseWorkerCommandLaunch(input.resources);
     activeProcesses.delete(input.message.runId);
   }
 }
 async function executeStreamingCommand(input) {
   const startedAt = Date.now();
   let started = false;
+  let terminal;
   let timedOut = false;
   let timeout;
   let spawnedProcess;
@@ -2860,7 +2949,7 @@ async function executeStreamingCommand(input) {
   };
   input.setPreparationHeartbeatLease(true);
   try {
-    if (cancelledProcessRuns.delete(input.message.runId)) {
+    if (cancelledProcessRuns.has(input.message.runId)) {
       sendWorkerMessage(input.ws, {
         type: "exec_start_error",
         requestId: input.message.requestId,
@@ -2880,7 +2969,7 @@ async function executeStreamingCommand(input) {
     const cwd = resolveCommandCwd(input.resolvedTarget.rootPath, input.message.cwd);
     const interactive = input.message.interactive === true;
     input.assertAdmission();
-    const subprocess = Bun.spawn(input.message.argv, {
+    const subprocess = Bun.spawn(input.resources.wrap(input.message.argv), {
       cwd,
       // Without an explicit stdin the process reads /dev/null and interactive
       // prompts see immediate EOF; "pipe" keeps stdin open for exec_stdin.
@@ -2923,6 +3012,7 @@ async function executeStreamingCommand(input) {
     if (input.message.timeoutMs) {
       timeout = setTimeout(() => {
         timedOut = true;
+        cancelWorkerCommandLaunch(input.resources);
         closeProcessStdin(activeProcesses.get(input.message.runId));
         void terminateProcessTree(subprocess).catch((error) => {
           process.stderr.write(
@@ -2951,8 +3041,19 @@ async function executeStreamingCommand(input) {
         });
       })
     ]);
+    const diagnostic = await input.resources.diagnose();
+    if (diagnostic) {
+      sendWorkerMessage(input.ws, {
+        type: "exec_output",
+        runId: input.message.runId,
+        stream: "stderr",
+        data: `
+[r5d-worker] ${diagnostic}
+`
+      });
+    }
     const canonicalWorkspaceHead = canonicalSyncTerminalHead(input.resolvedTarget);
-    const terminal = {
+    terminal = {
       type: "exec_exit",
       runId: input.message.runId,
       exitCode,
@@ -2961,14 +3062,12 @@ async function executeStreamingCommand(input) {
       ...input.message.workspaceEffect === "none" ? { workspaceEffect: "none" } : {},
       ...canonicalWorkspaceHead ? { canonicalWorkspaceHead } : {}
     };
-    pendingProcessTerminals.set(input.message.runId, terminal);
-    sendWorkerMessage(input.ws, terminal);
   } catch (error) {
-    if (!started && error instanceof StaleWorkerAdmissionError) throw error;
+    if (error instanceof CommandLauncherUnavailableError || !started && error instanceof StaleWorkerAdmissionError) throw error;
     const message = error instanceof Error ? error.message : String(error);
     if (started) {
       const canonicalWorkspaceHead = canonicalSyncTerminalHead(input.resolvedTarget);
-      const terminal = {
+      terminal = {
         type: "exec_error",
         runId: input.message.runId,
         error: message,
@@ -2976,8 +3075,6 @@ async function executeStreamingCommand(input) {
         ...input.message.workspaceEffect === "none" ? { workspaceEffect: "none" } : {},
         ...canonicalWorkspaceHead ? { canonicalWorkspaceHead } : {}
       };
-      pendingProcessTerminals.set(input.message.runId, terminal);
-      sendWorkerMessage(input.ws, terminal);
     } else {
       sendWorkerMessage(input.ws, {
         type: "exec_start_error",
@@ -2995,9 +3092,14 @@ async function executeStreamingCommand(input) {
     if (spawnedProcess) {
       await reapCompletedCredentialBearingProcessGroup(input.message.runId, spawnedProcess);
     }
+    await releaseWorkerCommandLaunch(input.resources);
     activeProcesses.delete(input.message.runId);
     cancelledProcessRuns.delete(input.message.runId);
   }
+  if (terminal) {
+    pendingProcessTerminals.set(input.message.runId, terminal);
+    sendWorkerMessage(input.ws, terminal);
+  }
 }
 async function reapCompletedCredentialBearingProcessGroup(runId, subprocess) {
   try {
@@ -3037,6 +3139,7 @@ function sendReplayWorkerMessage(ws, message) {
   return message.type === "exec_output" ? workerOutputWindow.send(message, send, target.bufferedAmount) : send(message);
 }
 function sendWorkerMessage(ws, message) {
+  if (launcherFailureInProgress) return;
   if (recoveryStore && "requestId" in message && typeof message.requestId === "string" && recoveryStore.isUnknown(message.requestId))
     return;
   const transportError = "error" in message && typeof message.error === "string" ? message.error : message.type === "workspace_sync_result" && message.result.outcome === "failed" ? message.result.error : void 0;
@@ -3380,6 +3483,7 @@ async function openPty(input) {
   }) : {};
   const targetEnv = targetProcessEnv(input.rootDir, input.resolvedTarget.target);
   const shell = resolveHostShell(input.message.command);
+  const shellArgv = input.resources.wrap([shell.file, ...shell.args]);
   const outputCoalescer = createPtyOutputCoalescer({
     send: (data) => {
       sendWorkerMessage(input.ws, {
@@ -3397,10 +3501,33 @@ async function openPty(input) {
     envFilesRemoved = true;
     removePtyEnvFiles(stagedEnvFiles.paths);
   };
+  let terminalHandled = false;
+  const finishPty = async (terminal) => {
+    if (terminalHandled) return;
+    terminalHandled = true;
+    try {
+      const diagnostic = await input.resources.diagnose();
+      if (diagnostic) outputCoalescer.push(`\r
+[r5d-worker] ${diagnostic}\r
+`);
+    } finally {
+      cleanupEnvFiles();
+      await releaseWorkerCommandLaunch(input.resources);
+      outputCoalescer.flush();
+      activePtys.delete(input.message.ptyId);
+      input.releaseWorkspaceMutation?.();
+      input.onTerminal?.();
+    }
+    sendWorkerMessage(input.ws, terminal);
+  };
+  const reportPtyCleanupFailure = (error) => {
+    process.stderr.write(`[r5d-worker] PTY launcher cleanup failed: ${error instanceof Error ? error.message : String(error)}
+`);
+  };
   try {
     const ptyProcess = createNodePtyBridge({
-      file: shell.file,
-      args: shell.args,
+      file: shellArgv[0],
+      args: shellArgv.slice(1),
       ptyOptions: {
         name: "xterm-256color",
         cols: Math.max(1, Math.min(Math.floor(input.message.cols || 80), 500)),
@@ -3429,30 +3556,14 @@ async function openPty(input) {
         outputCoalescer.push(data);
       },
       onExit: (event) => {
-        outputCoalescer.flush();
-        input.releaseWorkspaceMutation?.();
-        activePtys.delete(input.message.ptyId);
-        cleanupEnvFiles();
-        input.onTerminal?.();
-        sendWorkerMessage(input.ws, {
-          type: "pty_exit",
-          ptyId: input.message.ptyId,
-          exitCode: event.exitCode,
-          signal: event.signal
-        });
+        void finishPty({ type: "pty_exit", ptyId: input.message.ptyId, exitCode: event.exitCode, signal: event.signal }).catch(
+          reportPtyCleanupFailure
+        );
       },
       onError: (error) => {
-        outputCoalescer.flush();
-        input.releaseWorkspaceMutation?.();
-        activePtys.delete(input.message.ptyId);
-        cleanupEnvFiles();
-        input.onTerminal?.();
-        sendWorkerMessage(input.ws, {
-          type: "pty_error",
-          requestId: input.message.requestId,
-          ptyId: input.message.ptyId,
-          error: error.message
-        });
+        void finishPty({ type: "pty_error", requestId: input.message.requestId, ptyId: input.message.ptyId, error: error.message }).catch(
+          reportPtyCleanupFailure
+        );
       },
       onTerminationError: (error) => {
         sendWorkerMessage(input.ws, {
@@ -3495,7 +3606,12 @@ function resizePty(message) {
   }
   ptyProcess.resize(Math.max(1, Math.min(Math.floor(message.cols || 80), 500)), Math.max(1, Math.min(Math.floor(message.rows || 24), 500)));
 }
-function closePty(message) {
+async function closePty(message) {
+  try {
+    await workerCommandLauncher?.cancel(`pty:${message.ptyId}`);
+  } finally {
+    activePtys.get(message.ptyId)?.kill();
+  }
   const ptyProcess = activePtys.get(message.ptyId);
   if (!ptyProcess) {
     return;
@@ -3514,7 +3630,7 @@ function closePtysBestEffort(ptys, reportFailure) {
 function closeAllPtys() {
   closePtysBestEffort(activePtys, (ptyId, error) => {
     process.stderr.write(
-      `[r5d-worker] failed to close PTY ${ptyId} after control disconnect: ${error instanceof Error ? error.message : String(error)}
+      `[r5d-worker] failed to close PTY ${ptyId} after execution lease expiry: ${error instanceof Error ? error.message : String(error)}
 `
     );
   });
@@ -3587,6 +3703,17 @@ async function startWorker(options, projectRuntime = {
   initializeCredentialStoreDirectoryOnce();
   const label = requireValue(options.label, "Missing required --label <label>");
   validateLabel(label);
+  if (!workerCommandLauncher) {
+    const launcher = await createWorkerCommandLauncher({
+      configuration: process.env.R5D_WORKER_COMMAND_LAUNCHER,
+      env: commandLauncherEnvironment(),
+      onFailure: handleWorkerCommandLauncherFailure
+    });
+    workerCommandLauncher = launcher;
+    await launcher.initialize();
+    process.stdout.write(`[r5d-worker] command launcher: ${launcher.mode}
+`);
+  }
   const rootDir = path.resolve(options.rootDir ?? process.env.R5D_ROOT ?? defaultRootDir());
   recoveryStore ??= new WorkerRecoveryStore(
     path.join(rootDir, ".worker-recovery", createHash("sha256").update(`${baseUrl}\0${label}`).digest("hex"), "protocol.sqlite")
@@ -4827,7 +4954,7 @@ async function startWorker(options, projectRuntime = {
             `[r5d-worker] workspace synchronization failed without a recoverable hydration basis; exiting for exact recovery after ${unsafeWorkspaceSyncRecoveryAttempts} in-process recovery attempt(s): ${errorMessage}
 `
           );
-          process.exit(exitCode);
+          void exitWorkerRuntime(exitCode);
         },
         exitCode: 1
       });
@@ -5638,7 +5765,7 @@ async function startWorker(options, projectRuntime = {
       if (ws.readyState === WebSocket.OPEN || ws.readyState === WebSocket.CONNECTING) {
         ws.close(1e3, "Worker shutting down");
       } else {
-        process.exit(0);
+        await exitWorkerRuntime(0);
       }
     })().catch((error) => {
       process.stderr.write(`[r5d-worker] graceful workspace sync failed: ${error instanceof Error ? error.message : String(error)}
@@ -5791,13 +5918,14 @@ async function startWorker(options, projectRuntime = {
       }
       if (message.type === "cancel_session") {
         recoveryStore.cancelSession(message.sessionId);
-        await Promise.all(
-          [...activeProcesses].filter(([, active]) => active.sessionId === message.sessionId).map(async ([runId, active]) => {
+        await Promise.all([
+          workerCommandLauncher?.cancelSession(message.sessionId),
+          ...[...activeProcesses].filter(([, active]) => active.sessionId === message.sessionId).map(async ([runId, active]) => {
             cancelledProcessRuns.add(runId);
             closeProcessStdin(active);
             await terminateProcessTree(active.process);
           })
-        );
+        ]);
         sendWorkerMessage(ws, { type: "cancel_session_ack", cancellationId: message.cancellationId });
         return;
       }
@@ -6244,7 +6372,7 @@ async function startWorker(options, projectRuntime = {
           });
           return;
         }
-        if (activeProcesses.size > 0 || activePtys.size > 0 || workspaceSyncRequestsInFlight > 0 || sessionArtifactSyncRequestsInFlight > 0) {
+        if (activeProcesses.size > 0 || (workerCommandLauncher?.activeIds.length ?? 0) > 0 || activePtys.size > 0 || workspaceSyncRequestsInFlight > 0 || sessionArtifactSyncRequestsInFlight > 0) {
           sendWorkerMessage(ws, {
             type: "update_clis_result",
             requestId: message.requestId,
@@ -6277,8 +6405,14 @@ async function startWorker(options, projectRuntime = {
         return;
       }
       if (message.type === "cancel") {
+        let resourceCancellationError;
+        try {
+          await workerCommandLauncher?.cancel(message.runId);
+        } catch (error) {
+          resourceCancellationError = error instanceof Error ? error.message : String(error);
+        }
         const active = activeProcesses.get(message.runId);
-        let cancelled = true;
+        let cancelled = resourceCancellationError === void 0;
         let cancelMessage;
         if (active) {
           try {
@@ -6293,6 +6427,7 @@ async function startWorker(options, projectRuntime = {
           cancelledProcessRuns.add(message.runId);
           cancelMessage = "Cancellation queued before command start";
         }
+        if (resourceCancellationError) cancelMessage += `; job resource termination failed: ${resourceCancellationError}`;
         sendSerializedWorkerMessage(
           ws,
           JSON.stringify({
@@ -6379,13 +6514,16 @@ async function startWorker(options, projectRuntime = {
           });
           return;
         }
-        await reserveWorkspaceCommandAfterCurrentSync(message.target, workspaceSyncSingleFlight, () => {
-          workspaceSyncPriorityPtyTargets.set(message.ptyId, message.target);
-          if (targetMayMutateVisibleWorkspace(message.target)) recordVisibleWorkspaceMutation(message.target);
-        });
+        let resources;
+        let resourcesTransferred = false;
         let releaseWorkspaceMutation;
         let mutationLeaseTransferred = false;
         try {
+          resources = await acquireWorkerCommandLaunch(ws, message, assertMessageAdmission);
+          await reserveWorkspaceCommandAfterCurrentSync(message.target, workspaceSyncSingleFlight, () => {
+            workspaceSyncPriorityPtyTargets.set(message.ptyId, message.target);
+            if (targetMayMutateVisibleWorkspace(message.target)) recordVisibleWorkspaceMutation(message.target);
+          });
           releaseWorkspaceMutation = await acquireWorkspaceCommandMutation(message.target, workspaceSyncSingleFlight);
           if (workspaceOperationsAreFenced(deferredWorkspaceConfiguration, activeWorkspaceIncidentId)) {
             throw new Error(
@@ -6396,6 +6534,7 @@ async function startWorker(options, projectRuntime = {
           process.stdout.write(`[r5d-worker] pty ${message.ptyId}: ${describeWorkerSessionTarget(message.target)}
 `);
           await openPty({
+            resources,
             ws,
             message,
             resolvedTarget,
@@ -6408,6 +6547,7 @@ async function startWorker(options, projectRuntime = {
               }
             } : {}
           });
+          resourcesTransferred = true;
           mutationLeaseTransferred = releaseWorkspaceMutation !== void 0;
         } catch (error) {
           sendWorkerMessageFromCurrentSource(ws, {
@@ -6418,6 +6558,7 @@ async function startWorker(options, projectRuntime = {
           });
         } finally {
           workspaceSyncPriorityPtyTargets.delete(message.ptyId);
+          if (!resourcesTransferred) await releaseWorkerCommandLaunch(resources);
           if (!mutationLeaseTransferred) releaseWorkspaceMutation?.();
         }
         return;
@@ -6447,7 +6588,7 @@ async function startWorker(options, projectRuntime = {
       }
       if (message.type === "pty_close") {
         const activePty = activePtys.get(message.ptyId);
-        closePty(message);
+        await closePty(message);
         if (activePty && targetMayMutateVisibleWorkspace(activePty.target)) {
           markWorkspaceDirty({ type: "process_terminal", detail: `pty ${message.ptyId} closed` }, true, activePty.target);
         }
@@ -6477,9 +6618,11 @@ async function startWorker(options, projectRuntime = {
           return;
         }
         let result;
+        let resources;
         let targetReserved = false;
         const hasWorkspaceEffect = workerCommandHasWorkspaceEffect(message);
         try {
+          resources = await acquireWorkerCommandLaunch(ws, message, assertMessageAdmission);
           if (hasWorkspaceEffect) {
             await reserveWorkspaceCommandAfterCurrentSync(message.target, workspaceSyncSingleFlight, () => {
               workspaceSyncPriorityProcessTargets.set(message.runId, message.target);
@@ -6497,6 +6640,7 @@ async function startWorker(options, projectRuntime = {
             process.stdout.write(`[r5d-worker] exec ${message.runId}: ${message.argv.join(" ")}
 `);
             return await executeCommand({
+              resources,
               message,
               resolvedTarget,
               baseUrl,
@@ -6532,6 +6676,8 @@ async function startWorker(options, projectRuntime = {
           };
         } finally {
           if (targetReserved) workspaceSyncPriorityProcessTargets.delete(message.runId);
+          await releaseWorkerCommandLaunch(resources);
+          cancelledProcessRuns.delete(message.runId);
         }
         if (infrastructureStoppedRunIds.has(message.runId)) {
           recoveryStore.unknown(message.requestId);
@@ -6563,12 +6709,14 @@ async function startWorker(options, projectRuntime = {
           runId: message.runId
         });
         const hasWorkspaceEffect = workerCommandHasWorkspaceEffect(message);
+        let resources;
         const runCommand = async () => {
           try {
             const resolvedTarget = resolveMessageTarget(message.target);
             process.stdout.write(`[r5d-worker] exec_start ${message.runId}: ${message.argv.join(" ")}
 `);
             await executeStreamingCommand({
+              resources,
               ws,
               message,
               resolvedTarget,
@@ -6602,6 +6750,7 @@ async function startWorker(options, projectRuntime = {
         const execution = (async () => {
           let targetReserved = false;
           try {
+            resources = await acquireWorkerCommandLaunch(ws, message, assertMessageAdmission);
             if (hasWorkspaceEffect) {
               await reserveWorkspaceCommandAfterCurrentSync(message.target, workspaceSyncSingleFlight, () => {
                 workspaceSyncPriorityProcessTargets.set(message.runId, message.target);
@@ -6617,6 +6766,8 @@ async function startWorker(options, projectRuntime = {
             });
           } finally {
             if (targetReserved) workspaceSyncPriorityProcessTargets.delete(message.runId);
+            await releaseWorkerCommandLaunch(resources);
+            cancelledProcessRuns.delete(message.runId);
           }
         })();
         void execution.catch((error) => {
@@ -6747,11 +6898,10 @@ async function startWorker(options, projectRuntime = {
     const reason = event.reason ? `: ${event.reason}` : "";
     process.stderr.write(`[r5d-worker] disconnected (${event.code}${reason})
 `);
-    if (reloadAfterClose) {
-      process.exit(WORKER_RELOAD_EXIT_CODE);
-    }
-    if (shutdownAfterClose) {
-      process.exit(0);
+    if (launcherFailureInProgress) return;
+    if (reloadAfterClose || shutdownAfterClose) {
+      void exitWorkerRuntime(reloadAfterClose ? WORKER_RELOAD_EXIT_CODE : 0);
+      return;
     }
     const reconnect = () => {
       const delay = workerReconnectDelayMs(Math.random, workerReconnectAttempt++);
@@ -6809,10 +6959,10 @@ async function main() {
     try {
       await startWorker(parsed.options);
     } catch (error) {
-      if (error instanceof WorkerServerUnavailableError) {
+      if (error instanceof WorkerServerUnavailableError || error instanceof CommandLauncherUnavailableError) {
         process.stderr.write(`[r5d-worker] ${error.message}
 `);
-        process.exit(WORKER_RECONNECT_EXIT_CODE);
+        await exitWorkerRuntime(WORKER_RECONNECT_EXIT_CODE);
       }
       throw error;
     }
@@ -6830,10 +6980,10 @@ function isCliEntrypoint() {
   return typeof require === "function" && typeof module !== "undefined" && require.main === module;
 }
 if (isCliEntrypoint()) {
-  main().catch((error) => {
+  main().catch(async (error) => {
     process.stderr.write(`[r5d-worker] ${error instanceof Error ? error.message : String(error)}
 `);
-    process.exit(1);
+    await exitWorkerRuntime(1);
   });
 }
 export {

package/dist/mjs/package.json

@@ -1,5 +1,5 @@
 {
   "name": "@ricsam/r5d-worker",
-  "version": "0.0.110",
+  "version": "0.0.112",
   "type": "module"
 }

package/dist/mjs/supervisor.mjs

@@ -22,7 +22,7 @@ function workerReconnectDelayMs(random = Math.random, attempt = 0) {
   return floor + Math.floor(sample * (ceiling - floor));
 }
 async function defaultRuntimeReaper(runtime) {
-  await terminateProcessTree(runtime);
+  await terminateProcessTree(runtime, { graceMs: 6e3 });
 }
 function isRetryableWorkerServerStatus(status) {
   return status === 408 || status === 429 || status >= 500;