agent-relay-runner 0.129.1 → 0.129.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-relay-runner",
-  "version": "0.129.1",
+  "version": "0.129.3",
   "description": "Unified provider lifecycle runner for Agent Relay",
   "type": "module",
   "bin": {

package/plugins/claude/.claude-plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "agent-relay-runner",
   "description": "Thin Agent Relay runner bridge for Claude Code",
-  "version": "0.129.1",
+  "version": "0.129.3",
   "agentRelayContracts": {
     "providerPluginProtocol": 1
   }

package/src/adapters/codex-launch-config.ts

@@ -0,0 +1,143 @@
+// Codex launch/config-arg construction: the `-c` override builders, CLI command
+// resolution (shim detection), and the tool-output token limit. Extracted from
+// codex.ts (file-size ratchet, epic #291); codex.ts re-exports these so existing
+// consumers + tests keep their import path.
+import { accessSync, constants, realpathSync } from "node:fs";
+import { basename, join } from "node:path";
+import { isRecord } from "agent-relay-sdk";
+import type { RunnerSpawnConfig } from "../adapter";
+import { tomlString } from "../relay-mcp";
+
+export const DEFAULT_CODEX_TOOL_OUTPUT_TOKEN_LIMIT = 12_000;
+
+// #1248 — default background-terminal poll window (300000ms/5min) is too short for
+// our long-running commands (tests/releases run 5+ min), forcing agents to poll
+// instead of just waiting. Verified against installed codex-cli's source (rust-v0.144.0):
+// `background_terminal_max_timeout` is a top-level Config field — it replaced the older
+// `background_terminal_timeout` key — and IS correctly threaded through: ConfigManager
+// replays these launch `-c` overrides for every thread (config/mod.rs,
+// app-server/src/config_manager.rs), so `UnifiedExecProcessManager` (unified_exec/mod.rs)
+// is built with this raised ceiling and the `write_stdin` handler clamps empty-poll
+// yield_time_ms against it (unified_exec/process_manager.rs). The config value alone does
+// NOT make agents wait longer, though: codex's `write_stdin` tool schema
+// (tools/handlers/shell_spec.rs::create_write_stdin_tool) ships a hardcoded description —
+// "empty polls wait 5000-300000 ms by default" — that never reflects this override, so a
+// model sees no signal it may request more and keeps polling at ~300000ms. That's why a
+// live probe still saw ~300.001s polls after this override landed. codexRelayContextBlock()
+// (codex.ts) tells Codex agents explicitly that they may request yield_time_ms up to this value.
+const CODEX_BACKGROUND_TERMINAL_MAX_TIMEOUT_MS = 900000;
+
+export const CODEX_BACKGROUND_TERMINAL_HINT =
+  `This Codex session's background-terminal poll window is configured to ${CODEX_BACKGROUND_TERMINAL_MAX_TIMEOUT_MS}ms ` +
+  "(15 min), well above the 300000ms ceiling the write_stdin tool's own description advertises. " +
+  "When polling a long-running background command with write_stdin, pass a larger " +
+  `yield_time_ms (up to ${CODEX_BACKGROUND_TERMINAL_MAX_TIMEOUT_MS}) instead of repeatedly ` +
+  "polling at the tool's documented default — it will be honored.";
+
+export function codexModelConfigArgs(model?: string, effort?: string): string[] {
+  const args: string[] = [];
+  if (model) args.push("-c", `model=${tomlString(model)}`);
+  if (effort) args.push("-c", `model_reasoning_effort=${tomlString(effort)}`);
+  return args;
+}
+
+export function codexApprovalConfigArgs(approvalMode?: string): string[] {
+  const sandboxMode = approvalMode === "open"
+    ? "danger-full-access"
+    : approvalMode === "read-only"
+      ? "read-only"
+      : "workspace-write";
+  return [
+    "-c",
+    'approval_policy="never"',
+    "-c",
+    `sandbox_mode=${tomlString(sandboxMode)}`,
+  ];
+}
+
+// #1248 — see CODEX_BACKGROUND_TERMINAL_MAX_TIMEOUT_MS above for why this override
+// is correct but insufficient on its own.
+export function codexManagedConfigArgs(): string[] {
+  return [
+    "-c",
+    "check_for_update_on_startup=false",
+    "-c",
+    `background_terminal_max_timeout=${CODEX_BACKGROUND_TERMINAL_MAX_TIMEOUT_MS}`,
+  ];
+}
+
+export function isCodexCliCommand(command: string): boolean {
+  return basename(command) === "codex";
+}
+
+export function resolveRealCodexCommand(command: string, pathValue = process.env.PATH || ""): string {
+  const candidates: string[] = [];
+  if (command.includes("/")) candidates.push(command);
+  for (const dir of pathValue.split(":").filter(Boolean)) candidates.push(join(dir, "codex"));
+
+  const seen = new Set<string>();
+  for (const candidate of candidates) {
+    if (seen.has(candidate)) continue;
+    seen.add(candidate);
+    if (!isExecutableFile(candidate)) continue;
+    const real = realPath(candidate);
+    if (isAgentRelayCodexShim(candidate) || isAgentRelayCodexShim(real)) continue;
+    return candidate;
+  }
+  return command;
+}
+
+function isExecutableFile(path: string): boolean {
+  try {
+    accessSync(path, constants.X_OK);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function realPath(path: string): string {
+  try {
+    return realpathSync(path);
+  } catch {
+    return path;
+  }
+}
+
+function isAgentRelayCodexShim(path: string): boolean {
+  return path.includes("/.agent-relay/codex/bin/codex");
+}
+
+export function codexAppServerConfigArgs(...argLists: string[][]): string[] {
+  const result: string[] = [];
+  for (const args of argLists) {
+    for (let i = 0; i < args.length; i++) {
+      const arg = args[i];
+      if (arg === "-c" || arg === "--config" || arg === "--enable" || arg === "--disable") {
+        const value = args[i + 1];
+        if (value !== undefined) {
+          result.push(arg, value);
+          i++;
+        }
+      } else if (arg?.startsWith("--config=") || arg?.startsWith("--enable=") || arg?.startsWith("--disable=")) {
+        result.push(arg);
+      }
+    }
+  }
+  return result;
+}
+
+export function codexToolOutputTokenLimitConfigArgs(config: Pick<RunnerSpawnConfig, "agentProfile">): string[] {
+  const configured = codexToolOutputTokenLimit(config.agentProfile?.providerOptions);
+  if (configured === null) return [];
+  const limit = configured ?? DEFAULT_CODEX_TOOL_OUTPUT_TOKEN_LIMIT;
+  return ["-c", `tool_output_token_limit=${limit}`];
+}
+
+function codexToolOutputTokenLimit(providerOptions: unknown): number | null | undefined {
+  if (!isRecord(providerOptions)) return undefined;
+  const codex = providerOptions.codex;
+  if (!isRecord(codex)) return undefined;
+  const limit = codex.toolOutputTokenLimit;
+  return typeof limit === "number" || limit === null ? limit : undefined;
+}

package/src/adapters/codex.ts

@@ -1,5 +1,3 @@
-import { accessSync, constants, existsSync, realpathSync } from "node:fs";
-import { basename, join, resolve } from "node:path";
 import type { ContextState, InteractivePrompt, LivenessSignal, Message, MessageTurnEndReason, ProviderState } from "agent-relay-sdk";
 import { isRecord, stringValue } from "agent-relay-sdk";
 import { isPidAlive, killPid, processTreePids, processTreePidsFromTable, waitForPidsExit } from "agent-relay-sdk/process-utils";
@@ -14,15 +12,28 @@ import { codexTokenUsageContext, isContextState, latestCodexContext } from "./co
 import { computeLivenessSignal, type LivenessInputs } from "../liveness";
 
 /** Relay context prepended to a Codex agent's first turn: the standard relay
- * blurb plus, when running in an isolated workspace, the deps caveat (#159). */
+ * blurb plus, when running in an isolated workspace, the deps caveat (#159),
+ * plus the #1248 background-terminal poll-ceiling hint. */
 function codexRelayContextBlock(): string {
-  return [relayContext(), workspaceDepsNoteFromEnv()].filter(Boolean).join("\n\n");
+  return [relayContext(), workspaceDepsNoteFromEnv(), CODEX_BACKGROUND_TERMINAL_HINT].filter(Boolean).join("\n\n");
 }
 import { prepareCodexProfileHome, profileUsesHostProviderGlobals } from "../profile-home";
 import { CodexAppClient, type ClientEvent } from "./codex-client";
+import { CODEX_BACKGROUND_TERMINAL_HINT, codexApprovalConfigArgs, codexAppServerConfigArgs, codexManagedConfigArgs, codexModelConfigArgs, codexToolOutputTokenLimitConfigArgs, isCodexCliCommand, resolveRealCodexCommand } from "./codex-launch-config";
 
 export { processTreePidsFromTable };
-export const DEFAULT_CODEX_TOOL_OUTPUT_TOKEN_LIMIT = 12_000;
+// Launch/config-arg construction lives in ./codex-launch-config; re-exported so
+// codex.ts consumers + tests keep their import path.
+export {
+  codexApprovalConfigArgs,
+  codexAppServerConfigArgs,
+  codexManagedConfigArgs,
+  codexModelConfigArgs,
+  codexToolOutputTokenLimitConfigArgs,
+  DEFAULT_CODEX_TOOL_OUTPUT_TOKEN_LIMIT,
+  isCodexCliCommand,
+  resolveRealCodexCommand,
+} from "./codex-launch-config";
 
 type PendingCodexApproval = {
   id: string;
@@ -1073,85 +1084,6 @@ function activeFlags(value: unknown): string[] {
   return value.activeFlags.filter((flag): flag is string => typeof flag === "string" && flag.length > 0);
 }
 
-export function codexModelConfigArgs(model?: string, effort?: string): string[] {
-  const args: string[] = [];
-  if (model) args.push("-c", `model=${tomlString(model)}`);
-  if (effort) args.push("-c", `model_reasoning_effort=${tomlString(effort)}`);
-  return args;
-}
-
-export function codexApprovalConfigArgs(approvalMode?: string): string[] {
-  const sandboxMode = approvalMode === "open"
-    ? "danger-full-access"
-    : approvalMode === "read-only"
-      ? "read-only"
-      : "workspace-write";
-  return [
-    "-c",
-    'approval_policy="never"',
-    "-c",
-    `sandbox_mode=${tomlString(sandboxMode)}`,
-  ];
-}
-
-// #1248 — default background-terminal poll window (300000ms/5min) is too short for
-// our long-running commands (tests/releases run 5+ min), forcing agents to poll
-// instead of just waiting. Verified against the installed codex-cli binary (0.144.0):
-// `background_terminal_max_timeout` is a top-level (global) Config field — it replaced
-// the older `background_terminal_timeout` key, which no longer appears in that build.
-const CODEX_BACKGROUND_TERMINAL_MAX_TIMEOUT_MS = 900000;
-
-export function codexManagedConfigArgs(): string[] {
-  return [
-    "-c",
-    "check_for_update_on_startup=false",
-    "-c",
-    `background_terminal_max_timeout=${CODEX_BACKGROUND_TERMINAL_MAX_TIMEOUT_MS}`,
-  ];
-}
-
-export function isCodexCliCommand(command: string): boolean {
-  return basename(command) === "codex";
-}
-
-export function resolveRealCodexCommand(command: string, pathValue = process.env.PATH || ""): string {
-  const candidates: string[] = [];
-  if (command.includes("/")) candidates.push(command);
-  for (const dir of pathValue.split(":").filter(Boolean)) candidates.push(join(dir, "codex"));
-
-  const seen = new Set<string>();
-  for (const candidate of candidates) {
-    if (seen.has(candidate)) continue;
-    seen.add(candidate);
-    if (!isExecutableFile(candidate)) continue;
-    const real = realPath(candidate);
-    if (isAgentRelayCodexShim(candidate) || isAgentRelayCodexShim(real)) continue;
-    return candidate;
-  }
-  return command;
-}
-
-function isExecutableFile(path: string): boolean {
-  try {
-    accessSync(path, constants.X_OK);
-    return true;
-  } catch {
-    return false;
-  }
-}
-
-function realPath(path: string): string {
-  try {
-    return realpathSync(path);
-  } catch {
-    return path;
-  }
-}
-
-function isAgentRelayCodexShim(path: string): boolean {
-  return path.includes("/.agent-relay/codex/bin/codex");
-}
-
 // #557 — bundledCodexSkillDirs + bundledSkillConfigArgs now live in ../launch-assembly
 // (the single launch-arg home, shared with the assembler); re-exported so existing
 // codex.ts consumers + tests keep their import path. tomlString likewise from ../relay-mcp.
@@ -1161,41 +1093,6 @@ export { tomlString };
 // so codex.ts consumers + tests keep their import path.
 export { codexTokenUsageContext, codexRolloutContextFromText } from "./codex-rollout-context";
 
-export function codexAppServerConfigArgs(...argLists: string[][]): string[] {
-  const result: string[] = [];
-  for (const args of argLists) {
-    for (let i = 0; i < args.length; i++) {
-      const arg = args[i];
-      if (arg === "-c" || arg === "--config" || arg === "--enable" || arg === "--disable") {
-        const value = args[i + 1];
-        if (value !== undefined) {
-          result.push(arg, value);
-          i++;
-        }
-      } else if (arg?.startsWith("--config=") || arg?.startsWith("--enable=") || arg?.startsWith("--disable=")) {
-        result.push(arg);
-      }
-    }
-  }
-  return result;
-}
-
-export function codexToolOutputTokenLimitConfigArgs(config: Pick<RunnerSpawnConfig, "agentProfile">): string[] {
-  const configured = codexToolOutputTokenLimit(config.agentProfile?.providerOptions);
-  if (configured === null) return [];
-  const limit = configured ?? DEFAULT_CODEX_TOOL_OUTPUT_TOKEN_LIMIT;
-  return ["-c", `tool_output_token_limit=${limit}`];
-}
-
-function codexToolOutputTokenLimit(providerOptions: unknown): number | null | undefined {
-  if (!isRecord(providerOptions)) return undefined;
-  const codex = providerOptions.codex;
-  if (!isRecord(codex)) return undefined;
-  const limit = codex.toolOutputTokenLimit;
-  return typeof limit === "number" || limit === null ? limit : undefined;
-}
-
-
 async function connectWithRetry(client: CodexAppClient, attempts = 40): Promise<void> {
   // Give the freshly-spawned app-server a moment to bind its socket before the
   // first attempt, so we don't spend the retry budget on guaranteed refusals.

package/src/runner-core.ts

@@ -1053,8 +1053,10 @@ export class AgentRunner {
         return;
       }
 
+      let restartResumeId: string | undefined;
       if (this.shouldStopUnexpectedProviderExit(diagnostics)) {
-        const hasResumeId = providerExitResumeId(diagnostics) !== undefined;
+        const resumeId = providerExitResumeId(diagnostics);
+        const hasResumeId = resumeId !== undefined;
         if (!hasResumeId) {
           const recoveredStatus = await this.recoveredStatusAfterUndeterminedProviderExit();
           if (recoveredStatus) {
@@ -1081,6 +1083,17 @@ export class AgentRunner {
         }
         if (!hasResumeId && this.shouldRestartRecoverableProviderExit(diagnostics)) {
           logger.warn("lifecycle", `${this.options.provider} exit signal is early/recoverable; allowing fresh restart`);
+        } else if (resumeId && this.shouldResumeWithContextProviderExit(diagnostics, recent.length)) {
+          // #1258 (L2): a long-lived agent killed via tmux-session-ended (external SIGTERM, reboot,
+          // OOM, admin cleanup) used to be stranded permanently offline even though the provider CLI printed
+          // a resumable session id. Now, when Layer 3 captured a valid resume id, auto-resume WITH
+          // context instead of surfacing manual recovery. Crash-loop protection reuses the existing
+          // unexpected-exit window: `recent` is capped at MAX_RAPID_UNEXPECTED_EXITS within
+          // UNEXPECTED_EXIT_WINDOW_MS, and the fall-through path below applies backoff + a hard
+          // give-up — so a genuinely broken agent stops resurrecting once the window fills, while a
+          // resume that survives past the window resets the counter (no permanent lockout).
+          restartResumeId = resumeId;
+          logger.warn("lifecycle", `${this.options.provider} exited via tmux; auto-resuming with captured context (resume id ${resumeId})`);
         } else {
           logger.warn("lifecycle", `${this.options.provider} exited; leaving agent offline for manual recovery`);
           this.publishRunnerTimelineEvent({
@@ -1141,18 +1154,21 @@ export class AgentRunner {
       }
 
       const delayMs = Math.min(10_000, Math.max(500, 500 * recent.length));
-      logger.warn("lifecycle", `provider session exited unexpectedly after ${Math.round(runtimeMs / 1000)}s; restarting in ${delayMs}ms`);
+      const resuming = restartResumeId !== undefined;
+      logger.warn("lifecycle", `provider session exited unexpectedly after ${Math.round(runtimeMs / 1000)}s; ${resuming ? "resuming with context" : "restarting"} in ${delayMs}ms`);
       this.publishRunnerTimelineEvent({
         status: "provider.restart_decision",
         id: `provider-restart-decision-${this.providerSessionId}-${now}`,
         timestamp: Date.now(),
         title: "Provider restart scheduled",
-        body: `runner will start a fresh ${this.options.provider} provider in ${delayMs}ms`,
+        body: resuming
+          ? `runner will resume ${this.options.provider} with captured context in ${delayMs}ms`
+          : `runner will start a fresh ${this.options.provider} provider in ${delayMs}ms`,
         icon: "ti-refresh",
         metadata: {
           eventType: "provider.restart_decision",
-          decision: "restart-fresh",
-          reason: "unexpected-headless-terminal-exit",
+          decision: resuming ? "resume-with-context" : "restart-fresh",
+          reason: resuming ? "tmux-exit-auto-resume" : "unexpected-headless-terminal-exit",
           delayMs,
           rapidExitCount: recent.length,
           rapidExitWindowMs: UNEXPECTED_EXIT_WINDOW_MS,
@@ -1162,7 +1178,7 @@ export class AgentRunner {
       await Bun.sleep(delayMs);
       if (this.stopped || this.exitCommandInProgress) return;
       try {
-        await this.restartProvider();
+        await this.restartProvider(restartResumeId ? { resumeId: restartResumeId } : undefined);
         this.publishStatus();
         this.scheduleDrain();
       } catch (error) {
@@ -1186,6 +1202,15 @@ export class AgentRunner {
       && diagnostics.runtimeMs < RAPID_EXIT_MS;
   }
 
+  // #1258 (L2): auto-resume-with-context gate for a tmux-session-ended death at any point in an
+  // agent's life (not just the <30s early-flake window). Requires a captured resume id (the caller
+  // only reaches this when one exists) and bounds crash loops via the shared unexpected-exit window:
+  // once `recent` exceeds MAX_RAPID_UNEXPECTED_EXITS the agent is left offline for manual recovery.
+  private shouldResumeWithContextProviderExit(diagnostics: Record<string, unknown>, recentUnexpectedExits: number): boolean {
+    return diagnostics.exitSource === "tmux-session-ended"
+      && recentUnexpectedExits <= MAX_RAPID_UNEXPECTED_EXITS;
+  }
+
   private async recoveredStatusAfterUndeterminedProviderExit(): Promise<"busy" | "idle" | undefined> {
     if (!this.process || !this.options.adapter.probeActivity) return undefined;
     const deadline = Date.now() + UNDETERMINED_PROVIDER_EXIT_CONFIRM_MS;

package/src/runner-helpers.ts

@@ -168,6 +168,17 @@ export function runnerShouldRestartUnexpectedProviderExit(
     && input.hasTerminalSession;
 }
 
+// #1258 (L3): claude prints its resume banner ANSI-dimmed — the raw log bytes are
+// `\x1b[2mclaude --resume <uuid>\x1b[22m`, so `claude` is immediately preceded by the SGR
+// escape's trailing `m`. A leading `\b` in the extraction regex finds no word boundary
+// (`m`→`c` are both word chars) and yields zero matches, stranding the resume id as null.
+// Strip CSI escape sequences before matching so the banner reads as plain text.
+const ANSI_ESCAPE_RE = /\x1b\[[0-9;?]*[ -/]*[@-~]/g;
+
+function stripAnsi(text: string): string {
+  return text.replace(ANSI_ESCAPE_RE, "");
+}
+
 function resumeLogExtractionRe(provider: string): RegExp | undefined {
   const manifest = getManifest(provider);
   const resume = manifest?.resume as { supported: boolean; idPattern?: string } | undefined;
@@ -178,8 +189,9 @@ function resumeLogExtractionRe(provider: string): RegExp | undefined {
 export function latestResumeIdFromText(provider: string, text: string): string | undefined {
   const re = resumeLogExtractionRe(provider);
   if (!re) return undefined;
+  const cleaned = stripAnsi(text);
   let latest: string | undefined;
-  for (let match = re.exec(text); match; match = re.exec(text)) {
+  for (let match = re.exec(cleaned); match; match = re.exec(cleaned)) {
     latest = match[1];
   }
   return latest;