@pushpalsdev/cli 1.1.26 → 1.1.28

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pushpalsdev/cli",
-  "version": "1.1.26",
+  "version": "1.1.28",
   "description": "PushPals terminal CLI for LocalBuddy -> RemoteBuddy orchestration",
   "license": "MIT",
   "repository": {

package/runtime/sandbox/.pushpals-remotebuddy-fallback.js

@@ -8408,6 +8408,7 @@ ${detail}`.toLowerCase();
     "codex cli is mandatory in this backend"
   ].some((needle) => text.includes(needle));
 }
+var CODEX_STARTUP_STALL_WORKER_EXIT_CODE = 87;
 function asAutonomyComponentArea2(value) {
   return normalizeAutonomyComponentArea(value) ?? undefined;
 }
@@ -8950,6 +8951,7 @@ class RemoteBuddyOrchestrator {
   workerpalsEnvFile;
   workerpalsEntrypoint;
   workerpalsUnavailableReason;
+  workerDockerFallbackActivated = false;
   statusHeartbeatMs;
   fetchFailureLogsOnJobFailure;
   executionBudgetInteractiveMs;
@@ -9956,6 +9958,25 @@ Please reply with the missing details and I will enqueue a follow-up request.` :
       entrypoint: this.workerpalsEntrypoint
     });
   }
+  maybeFallbackFromDockerAfterWorkerExit(workerId, code) {
+    if (code !== CODEX_STARTUP_STALL_WORKER_EXIT_CODE)
+      return false;
+    if (!this.spawnWorkerDocker)
+      return false;
+    if (this.workerDockerFallbackActivated)
+      return false;
+    if (parseEnabledFlag(process.env.REMOTEBUDDY_DISABLE_WORKERPAL_DIRECT_FALLBACK, false)) {
+      console.warn(`[RemoteBuddy] WorkerPal ${workerId} exited after a Docker Codex startup stall, but direct WorkerPal fallback is disabled.`);
+      return false;
+    }
+    this.workerDockerFallbackActivated = true;
+    this.spawnWorkerDocker = false;
+    this.spawnWorkerRequireDocker = false;
+    this.workerSpawnCooldownUntil = 0;
+    this.workerpalsUnavailableReason = "Docker-backed WorkerPal Codex startup stalled; falling back to direct isolated-worktree WorkerPal.";
+    console.warn(`[RemoteBuddy] WorkerPal ${workerId} exited after a Docker Codex startup stall; falling back to direct isolated-worktree WorkerPal for future spawns.`);
+    return true;
+  }
   async spawnWorker() {
     if (this.workerSpawnInFlight) {
       return await this.workerSpawnInFlight;
@@ -9983,6 +10004,9 @@ Please reply with the missing details and I will enqueue a follow-up request.` :
         this.managedWorkers.set(workerId, child);
         child.exited.then((code) => {
           this.managedWorkers.delete(workerId);
+          if (this.maybeFallbackFromDockerAfterWorkerExit(workerId, code)) {
+            this.ensureAutoscaledWorkerCapacity("docker codex startup fallback");
+          }
           console.warn(`[RemoteBuddy] WorkerPal process ${workerId} exited with code ${code}`);
         });
         const ready = await this.waitForOnlineWorker(this.workerStartupTimeoutMs, workerId);

package/runtime/sandbox/apps/workerpals/src/backends/openai_codex/openai_codex_executor.py

@@ -106,6 +106,7 @@ _MAX_WRAPPER_BOOTSTRAP_OUTPUT_CHARS = 1_200
 _MAX_WRAPPER_BOOTSTRAP_TOTAL_CHARS = 5_000
 _MAX_CREDIBLE_WRAPPER_LOOP_CHANGED_PATHS = 8
 _MAX_CREDIBLE_WRAPPER_LOOP_TOP_LEVELS = 4
+_MAX_STARTUP_STALL_RECOVERY_ATTEMPTS = 1
 _MAX_NO_EDIT_RECOVERY_ATTEMPTS = 1
 _MAX_ROLLOUT_RECOVERY_ATTEMPTS = 1
 _DEFAULT_NO_EDIT_WATCHDOG_S = 480
@@ -121,6 +122,7 @@ _NARROW_TEST_TASK_ROLLOUT_WATCHDOG_S = 150
 _WEB_REVIEW_ROLLOUT_WATCHDOG_S = 180
 _BACKGROUND_ROLLOUT_WATCHDOG_S = 90
 _NO_PUBLISHABLE_FAILURE_COOLDOWN_MS = 10 * 60 * 1000
+_CODEX_STARTUP_ONLY_EVENT_TYPES = {"thread.started", "turn.started"}
 
 
 def _model_supports_xhigh_reasoning(model: str) -> bool:
@@ -862,6 +864,19 @@ def _build_no_edit_recovery_guidance(trace_excerpt: str, artifact_only_paths: st
     return "\n".join(lines)
 
 
+def _build_startup_stall_recovery_guidance(trace_excerpt: str) -> str:
+    lines = [
+        "Codex startup-stall recovery: the previous Codex subprocess started but emitted no assistant, tool, or reasoning progress before the watchdog.",
+        "Treat this as a fresh execution with a patch-first contract. After at most one narrow read of the hinted owner, make the smallest publishable edit.",
+        "Do not spend this recovery attempt re-reading broad repository topology or validating before an edit exists.",
+        "If the hinted path is absent, choose the nearest existing repo-native owner or test rather than creating unrelated scaffolding.",
+    ]
+    if trace_excerpt:
+        lines.append("Previous Codex event trace excerpt:")
+        lines.append(trace_excerpt)
+    return "\n".join(lines)
+
+
 def _trace_summaries_text(trace: Dict[str, Any]) -> str:
     summaries = trace.get("summaries")
     if not isinstance(summaries, list):
@@ -869,6 +884,36 @@ def _trace_summaries_text(trace: Dict[str, Any]) -> str:
     return "\n".join(str(item or "") for item in summaries[-80:]).lower()
 
 
+def _codex_trace_has_work_progress(trace: Dict[str, Any]) -> bool:
+    if to_int(trace.get("reasoning_events"), 0) > 0:
+        return True
+
+    event_counts = trace.get("event_type_counts")
+    if isinstance(event_counts, dict):
+        for key, value in event_counts.items():
+            event_type = str(key or "").strip()
+            if to_int(value, 0) > 0 and event_type not in _CODEX_STARTUP_ONLY_EVENT_TYPES:
+                return True
+
+    summaries = trace.get("summaries")
+    if isinstance(summaries, list):
+        for item in summaries:
+            summary = str(item or "").strip()
+            if not summary:
+                continue
+            event_type = summary.split("|", 1)[0].strip()
+            if event_type not in _CODEX_STARTUP_ONLY_EVENT_TYPES:
+                return True
+
+    return False
+
+
+def _codex_trace_is_startup_stall(trace: Dict[str, Any]) -> bool:
+    if to_int(trace.get("total_tokens"), 0) > 0:
+        return False
+    return not _codex_trace_has_work_progress(trace)
+
+
 def _detect_offtrack_rollout(trace: Dict[str, Any], artifact_only_paths: str = "") -> str:
     text = _trace_summaries_text(trace)
     if artifact_only_paths:
@@ -1981,6 +2026,7 @@ def _run_codex_task(
     *,
     wrapper_recovery_attempt: int = 0,
     model_compatibility_recovery_attempt: int = 0,
+    startup_stall_recovery_attempt: int = 0,
     no_edit_recovery_attempt: int = 0,
     rollout_recovery_attempt: int = 0,
     model_override: Optional[str] = None,
@@ -2475,6 +2521,7 @@ def _run_codex_task(
                     retry_guidance,
                     wrapper_recovery_attempt=wrapper_recovery_attempt,
                     model_compatibility_recovery_attempt=model_compatibility_recovery_attempt,
+                    startup_stall_recovery_attempt=startup_stall_recovery_attempt,
                     no_edit_recovery_attempt=no_edit_recovery_attempt,
                     rollout_recovery_attempt=rollout_recovery_attempt + 1,
                     model_override=model_override,
@@ -2497,6 +2544,54 @@ def _run_codex_task(
             }
 
         if no_edit_watchdog_fired:
+            startup_stall = _codex_trace_is_startup_stall(stdout_trace)
+            if startup_stall and startup_stall_recovery_attempt < _MAX_STARTUP_STALL_RECOVERY_ATTEMPTS:
+                retry_guidance = [
+                    *supplemental_guidance,
+                    _build_startup_stall_recovery_guidance(trace_excerpt),
+                ]
+                log.warning(
+                    "Codex emitted only startup events before the no-edit watchdog; "
+                    "restarting Codex once before classifying the job terminally."
+                )
+                retry_result = _run_codex_task(
+                    repo,
+                    instruction,
+                    retry_guidance,
+                    wrapper_recovery_attempt=wrapper_recovery_attempt,
+                    model_compatibility_recovery_attempt=model_compatibility_recovery_attempt,
+                    startup_stall_recovery_attempt=startup_stall_recovery_attempt + 1,
+                    no_edit_recovery_attempt=no_edit_recovery_attempt,
+                    rollout_recovery_attempt=rollout_recovery_attempt,
+                    model_override=model_override,
+                    baseline_changes=baseline_snapshot,
+                )
+                retry_result["usage"] = _merge_usage_records(usage, retry_result.get("usage"))
+                if retry_result.get("ok"):
+                    recovered_stdout = str(retry_result.get("stdout") or "").strip()
+                    retry_result["stdout"] = _truncate(
+                        (
+                            "Recovered after the first Codex subprocess stalled before emitting "
+                            f"assistant/tool progress.\n\n{recovered_stdout}"
+                        ).strip()
+                    )
+                return retry_result
+            if startup_stall:
+                detail = (
+                    "Codex subprocess started but did not emit assistant, tool, reasoning, "
+                    "or usage progress before the startup watchdog."
+                )
+                if trace_excerpt:
+                    detail = f"{detail}\n{trace_excerpt}"
+                return {
+                    "ok": False,
+                    "summary": "openai_codex stalled before first response",
+                    "stdout": _truncate(stdout),
+                    "stderr": _truncate(f"{detail}\n{stderr}".strip()),
+                    "exitCode": 124,
+                    "usage": usage,
+                    "cooldownMs": _NO_PUBLISHABLE_FAILURE_COOLDOWN_MS,
+                }
             if no_edit_recovery_attempt < _MAX_NO_EDIT_RECOVERY_ATTEMPTS:
                 retry_guidance = [
                     *supplemental_guidance,
@@ -2511,6 +2606,7 @@ def _run_codex_task(
                     retry_guidance,
                     wrapper_recovery_attempt=wrapper_recovery_attempt,
                     model_compatibility_recovery_attempt=model_compatibility_recovery_attempt,
+                    startup_stall_recovery_attempt=startup_stall_recovery_attempt,
                     no_edit_recovery_attempt=no_edit_recovery_attempt + 1,
                     rollout_recovery_attempt=rollout_recovery_attempt,
                     model_override=model_override,
@@ -2706,6 +2802,7 @@ def _run_codex_task(
                         ],
                         wrapper_recovery_attempt=wrapper_recovery_attempt + 1,
                         model_compatibility_recovery_attempt=model_compatibility_recovery_attempt,
+                        startup_stall_recovery_attempt=startup_stall_recovery_attempt,
                         no_edit_recovery_attempt=no_edit_recovery_attempt,
                         rollout_recovery_attempt=rollout_recovery_attempt,
                         model_override=model_override,
@@ -2820,6 +2917,7 @@ def _run_codex_task(
                     effective_supplemental_guidance,
                     wrapper_recovery_attempt=wrapper_recovery_attempt,
                     model_compatibility_recovery_attempt=model_compatibility_recovery_attempt + 1,
+                    startup_stall_recovery_attempt=startup_stall_recovery_attempt,
                     no_edit_recovery_attempt=no_edit_recovery_attempt,
                     rollout_recovery_attempt=rollout_recovery_attempt,
                     model_override=LEGACY_CODEX_MODEL_FALLBACK,

package/runtime/sandbox/apps/workerpals/src/backends/openai_codex/test_openai_codex_runtime_config.py

@@ -1029,6 +1029,159 @@ class OpenAICodexRuntimeConfigTests(unittest.TestCase):
         self.assertNotIn("broad/noisy", str(result.get("summary") or ""))
         self.assertNotIn("too broad/noisy", str(result.get("stderr") or ""))
 
+    def test_run_codex_task_retries_once_when_codex_stalls_before_first_response(self) -> None:
+        with tempfile.TemporaryDirectory(prefix="pushpals-codex-startup-stall-") as temp_dir:
+            repo = Path(temp_dir) / "repo"
+            repo.mkdir(parents=True, exist_ok=True)
+            (repo / "README.md").write_text("# startup stall repo\n", encoding="utf-8")
+            subprocess.run(["git", "init"], cwd=repo, check=True, capture_output=True, text=True)
+            subprocess.run(
+                ["git", "config", "user.name", "PushPals Test"],
+                cwd=repo,
+                check=True,
+                capture_output=True,
+                text=True,
+            )
+            subprocess.run(
+                ["git", "config", "user.email", "pushpals-tests@example.com"],
+                cwd=repo,
+                check=True,
+                capture_output=True,
+                text=True,
+            )
+            subprocess.run(["git", "add", "README.md"], cwd=repo, check=True, capture_output=True, text=True)
+            subprocess.run(
+                ["git", "commit", "-m", "chore: seed startup stall repo"],
+                cwd=repo,
+                check=True,
+                capture_output=True,
+                text=True,
+            )
+
+            stub_path = Path(temp_dir) / "fake_codex_startup_stall.py"
+            stub_path.write_text(
+                "\n".join(
+                    [
+                        "from pathlib import Path",
+                        "import json",
+                        "import sys",
+                        "import time",
+                        "",
+                        "argv = sys.argv[1:]",
+                        "last_message_path = None",
+                        "for index, arg in enumerate(argv):",
+                        "    if arg == '--output-last-message' and index + 1 < len(argv):",
+                        "        last_message_path = argv[index + 1]",
+                        "        break",
+                        "",
+                        "prompt = sys.stdin.read()",
+                        "if 'Codex startup-stall recovery' in prompt:",
+                        "    Path('src').mkdir(exist_ok=True)",
+                        "    Path('src/startup-stall-recovered.txt').write_text('patched after restart\\n', encoding='utf-8')",
+                        "    if last_message_path:",
+                        "        Path(last_message_path).write_text('Patched after Codex startup-stall recovery.', encoding='utf-8')",
+                        "    print(json.dumps({'type': 'item.completed', 'message': 'Patched after Codex startup-stall recovery.'}), flush=True)",
+                        "    sys.exit(0)",
+                        "",
+                        "print(json.dumps({'type': 'thread.started'}), flush=True)",
+                        "print(json.dumps({'type': 'turn.started'}), flush=True)",
+                        "time.sleep(10)",
+                    ]
+                ),
+                encoding="utf-8",
+            )
+
+            env_overrides = {
+                "PUSHPALS_OPENAI_CODEX_BIN_JSON": json.dumps([sys.executable, str(stub_path)]),
+                "PUSHPALS_OPENAI_CODEX_AUTH_MODE": "api_key",
+                "OPENAI_API_KEY": "pushpals-startup-stall-test-key",
+                "WORKERPALS_OPENAI_CODEX_JSON": "true",
+                "WORKERPALS_OPENAI_CODEX_TIMEOUT_S": "20",
+                "WORKERPALS_OPENAI_CODEX_NO_EDIT_WATCHDOG_S": "1",
+                "WORKERPALS_OPENAI_CODEX_PROGRESS_LOG_INTERVAL_S": "1",
+            }
+            with mock.patch.dict(os.environ, env_overrides, clear=False):
+                result = _run_codex_task(
+                    str(repo),
+                    "Rename one misleading test fixture constant and update the related assertions.",
+                    [],
+                )
+
+        self.assertTrue(result.get("ok"), result)
+        self.assertEqual(result.get("exitCode"), 0)
+        stdout = str(result.get("stdout") or "")
+        self.assertIn("Recovered after the first Codex subprocess stalled", stdout)
+        self.assertIn("Patched after Codex startup-stall recovery", stdout)
+        self.assertIn("src/", stdout)
+
+    def test_run_codex_task_reports_startup_stall_when_restart_also_never_responds(self) -> None:
+        with tempfile.TemporaryDirectory(prefix="pushpals-codex-startup-stall-fail-") as temp_dir:
+            repo = Path(temp_dir) / "repo"
+            repo.mkdir(parents=True, exist_ok=True)
+            (repo / "README.md").write_text("# startup stall failure repo\n", encoding="utf-8")
+            subprocess.run(["git", "init"], cwd=repo, check=True, capture_output=True, text=True)
+            subprocess.run(
+                ["git", "config", "user.name", "PushPals Test"],
+                cwd=repo,
+                check=True,
+                capture_output=True,
+                text=True,
+            )
+            subprocess.run(
+                ["git", "config", "user.email", "pushpals-tests@example.com"],
+                cwd=repo,
+                check=True,
+                capture_output=True,
+                text=True,
+            )
+            subprocess.run(["git", "add", "README.md"], cwd=repo, check=True, capture_output=True, text=True)
+            subprocess.run(
+                ["git", "commit", "-m", "chore: seed startup stall failure repo"],
+                cwd=repo,
+                check=True,
+                capture_output=True,
+                text=True,
+            )
+
+            stub_path = Path(temp_dir) / "fake_codex_startup_stall_fail.py"
+            stub_path.write_text(
+                "\n".join(
+                    [
+                        "import json",
+                        "import sys",
+                        "import time",
+                        "",
+                        "sys.stdin.read()",
+                        "print(json.dumps({'type': 'thread.started'}), flush=True)",
+                        "print(json.dumps({'type': 'turn.started'}), flush=True)",
+                        "time.sleep(10)",
+                    ]
+                ),
+                encoding="utf-8",
+            )
+
+            env_overrides = {
+                "PUSHPALS_OPENAI_CODEX_BIN_JSON": json.dumps([sys.executable, str(stub_path)]),
+                "PUSHPALS_OPENAI_CODEX_AUTH_MODE": "api_key",
+                "OPENAI_API_KEY": "pushpals-startup-stall-fail-test-key",
+                "WORKERPALS_OPENAI_CODEX_JSON": "true",
+                "WORKERPALS_OPENAI_CODEX_TIMEOUT_S": "20",
+                "WORKERPALS_OPENAI_CODEX_NO_EDIT_WATCHDOG_S": "1",
+                "WORKERPALS_OPENAI_CODEX_PROGRESS_LOG_INTERVAL_S": "1",
+            }
+            with mock.patch.dict(os.environ, env_overrides, clear=False):
+                result = _run_codex_task(
+                    str(repo),
+                    "Rename one misleading test fixture constant and update the related assertions.",
+                    [],
+                )
+
+        self.assertFalse(result.get("ok"), result)
+        self.assertEqual(result.get("exitCode"), 124)
+        self.assertEqual(result.get("summary"), "openai_codex stalled before first response")
+        self.assertNotIn("no publishable", str(result.get("summary") or "").lower())
+        self.assertEqual(result.get("cooldownMs"), 600000)
+
     def test_run_codex_task_retries_once_when_no_edit_watchdog_fires(self) -> None:
         with tempfile.TemporaryDirectory(prefix="pushpals-codex-no-edit-watchdog-") as temp_dir:
             repo = Path(temp_dir) / "repo"

package/runtime/sandbox/apps/workerpals/src/common/types.ts

@@ -92,6 +92,7 @@ export interface JobResult {
   stdout?: string;
   stderr?: string;
   exitCode?: number;
+  cooldownMs?: number;
   usage?: JobTokenUsage;
   publishBlocked?: JobPublishBlockedInfo;
   diagnostics?: JobDiagnostics;

package/runtime/sandbox/apps/workerpals/src/docker_executor.ts

@@ -249,7 +249,9 @@ export interface Job {
 }
 
 function compactDockerDiagnosticText(value: unknown, maxChars = 1000): string | null {
-  const text = String(value ?? "").replace(/\s+$/g, "").trim();
+  const text = String(value ?? "")
+    .replace(/\s+$/g, "")
+    .trim();
   if (!text) return null;
   return text.length <= maxChars ? text : text.slice(0, maxChars);
 }
@@ -482,11 +484,11 @@ export class DockerExecutor {
             if (
               retryableFailure &&
               attempt >= this.jobRetryMaxAttempts &&
-              this.failureCooldownMs > 0
+              this.retryExhaustionCooldownMs(result) > 0
             ) {
               return {
                 ...result,
-                cooldownMs: this.failureCooldownMs,
+                cooldownMs: this.retryExhaustionCooldownMs(result),
               };
             }
             return result;
@@ -1279,9 +1281,8 @@ export class DockerExecutor {
       onLog?.("stdout", note);
     }
 
-    const { leadMs: warningLeadMs, delayMs: warningDelayMs } = computeTimeoutWarningWindow(
-      timeoutMs,
-    );
+    const { leadMs: warningLeadMs, delayMs: warningDelayMs } =
+      computeTimeoutWarningWindow(timeoutMs);
     const warningTimer = setTimeout(() => {
       const warning = `[DockerExecutor] Job nearing timeout in warm container (${Math.round(
         warningLeadMs / 1000,
@@ -1424,13 +1425,13 @@ export class DockerExecutor {
     const worktreePrefix = shellSingleQuote(`${containerWorktreePath}/`);
     const command = [
       "set -eu",
-      "linked=\"\"",
+      'linked=""',
       "for name in node_modules; do",
-      "  src=\"/repo/$name\"",
+      '  src="/repo/$name"',
       `  dest=${worktreePrefix}$name`,
-      "  if { [ -e \"$src\" ] || [ -L \"$src\" ]; } && [ ! -e \"$dest\" ] && [ ! -L \"$dest\" ]; then",
-      "    ln -s \"$src\" \"$dest\"",
-      "    linked=\"$linked $name\"",
+      '  if { [ -e "$src" ] || [ -L "$src" ]; } && [ ! -e "$dest" ] && [ ! -L "$dest" ]; then',
+      '    ln -s "$src" "$dest"',
+      '    linked="$linked $name"',
       "  fi",
       "done",
       "printf '%s' \"$linked\"",
@@ -1454,9 +1455,7 @@ export class DockerExecutor {
       .filter(Boolean);
     if (linked.length === 0) return;
 
-    const note = `[DockerExecutor] Linked worktree dependency artifact(s): ${linked.join(
-      ", ",
-    )}`;
+    const note = `[DockerExecutor] Linked worktree dependency artifact(s): ${linked.join(", ")}`;
     console.log(note);
     onLog?.("stdout", note);
   }
@@ -1701,9 +1700,15 @@ export class DockerExecutor {
         stdout,
         stderr: details.join("\n"),
         exitCode,
-        diagnostics: dockerFallbackDiagnostics(summary, context, exitCode, "malformed_structured_result", {
-          sentinelParseError,
-        }),
+        diagnostics: dockerFallbackDiagnostics(
+          summary,
+          context,
+          exitCode,
+          "malformed_structured_result",
+          {
+            sentinelParseError,
+          },
+        ),
       };
     }
 
@@ -1906,8 +1911,15 @@ export class DockerExecutor {
     return this.matchesRetryablePattern(text);
   }
 
+  private retryExhaustionCooldownMs(result: DockerJobResult): number {
+    const resultCooldownMs = readPositiveNumber(result.cooldownMs) ?? 0;
+    return Math.max(this.failureCooldownMs, resultCooldownMs);
+  }
+
   private matchesRetryablePattern(text: string): boolean {
     const transientPatterns: RegExp[] = [
+      /\bstalled before first response\b/i,
+      /\bstartup stall\b/i,
       /warm .*runtime/i,
       /failed to start warm container/i,
       /docker execution error/i,

package/runtime/sandbox/apps/workerpals/src/execute_job.ts

@@ -688,6 +688,9 @@ function inferTerminalFailureClass(result: JobResult, changedPaths: string[]): s
   if (result.ok) return "success";
   const text = `${result.summary ?? ""}\n${result.stderr ?? ""}\n${result.stdout ?? ""}`.toLowerCase();
   const publishableCount = publishableChangedPaths(changedPaths).length;
+  if (text.includes("stalled before first response") || text.includes("startup stall")) {
+    return "codex_startup_stall";
+  }
   if (changedPaths.length > 0 && publishableCount === 0) return "artifact_only_no_publishable_patch";
   if (result.exitCode === 124 || text.includes("timed out") || text.includes("timeout")) return "timeout";
   if (text.includes("validationgate") || text.includes("validation")) return "validation";
@@ -700,6 +703,9 @@ function inferTerminalFailureClass(result: JobResult, changedPaths: string[]): s
 
 function inferTerminalStage(result: JobResult, fallback: string): string {
   const text = `${result.summary ?? ""}\n${result.stderr ?? ""}`.toLowerCase();
+  if (text.includes("stalled before first response") || text.includes("startup stall")) {
+    return "executor_startup";
+  }
   if (text.includes("validationgate") || text.includes("validation")) return "validation";
   if (text.includes("scopegate") || text.includes("scope")) return "scope";
   if (text.includes("criticgate") || text.includes("critic")) return "critic";
@@ -748,7 +754,7 @@ function buildTerminalDiagnostics(args: {
     terminalStage: inferTerminalStage(args.result, args.terminalStage),
     executorBackend: args.executor,
     summary: compactDiagnosticText(args.result.summary, 1_000),
-    watchdogFired: /watchdog|rollout coach/i.test(text),
+    watchdogFired: /watchdog|rollout coach|stalled before first response|startup stall/i.test(text),
     timeoutMs: args.timeoutMs ?? null,
     publishableFileCount: publishable.length,
     artifactOnlyPathCount: artifactOnly.length,

package/runtime/sandbox/apps/workerpals/src/job_runner.ts

@@ -44,6 +44,7 @@ interface JobResult {
   stdout?: string;
   stderr?: string;
   exitCode?: number;
+  cooldownMs?: number;
   commit?: {
     branch: string;
     sha: string;
@@ -115,6 +116,23 @@ echo "password=${token}"
   }
 }
 
+export function buildJobRunnerResult(
+  result: Pick<
+    Awaited<ReturnType<typeof executeJob>>,
+    "ok" | "summary" | "stdout" | "stderr" | "exitCode" | "cooldownMs" | "diagnostics"
+  >,
+): JobResult {
+  return {
+    ok: result.ok,
+    summary: result.summary,
+    stdout: result.stdout,
+    stderr: result.stderr,
+    exitCode: result.exitCode,
+    cooldownMs: result.cooldownMs,
+    diagnostics: result.diagnostics,
+  };
+}
+
 // ─── Main ───────────────────────────────────────────────────────────────────
 
 async function main(): Promise<void> {
@@ -127,8 +145,7 @@ async function main(): Promise<void> {
     process.exit(1);
   }
 
-  const base64Spec =
-    rawSpecArg === "--spec-stdin" ? (await Bun.stdin.text()).trim() : rawSpecArg;
+  const base64Spec = rawSpecArg === "--spec-stdin" ? (await Bun.stdin.text()).trim() : rawSpecArg;
   if (!base64Spec) {
     // eslint-disable-next-line no-console
     console.error("Job spec was empty");
@@ -179,14 +196,7 @@ async function main(): Promise<void> {
       CONFIG,
     );
     // Build result object
-    const jobResult: JobResult = {
-      ok: result.ok,
-      summary: result.summary,
-      stdout: result.stdout,
-      stderr: result.stderr,
-      exitCode: result.exitCode,
-      diagnostics: result.diagnostics,
-    };
+    const jobResult = buildJobRunnerResult(result);
     // Create commit for file-modifying jobs
     if (result.ok && shouldCommit(spec.kind, CONFIG)) {
       log("stdout", `[JobRunner] Job modified files, creating commit...`);
@@ -224,7 +234,8 @@ async function main(): Promise<void> {
         if (commitResult.publishBlocked) {
           jobResult.publishBlocked = commitResult.publishBlocked;
         }
-        jobResult.exitCode = jobResult.exitCode && jobResult.exitCode !== 0 ? jobResult.exitCode : 1;
+        jobResult.exitCode =
+          jobResult.exitCode && jobResult.exitCode !== 0 ? jobResult.exitCode : 1;
         log(
           "stderr",
           commitResult.publishBlocked
@@ -246,8 +257,10 @@ async function main(): Promise<void> {
   }
 }
 
-main().catch((err) => {
-  // eslint-disable-next-line no-console
-  console.error(`[JobRunner] Fatal error: ${err}`);
-  process.exit(1);
-});
+if (import.meta.main) {
+  main().catch((err) => {
+    // eslint-disable-next-line no-console
+    console.error(`[JobRunner] Fatal error: ${err}`);
+    process.exit(1);
+  });
+}

package/runtime/sandbox/apps/workerpals/src/workerpals_main.ts

@@ -67,6 +67,7 @@ type WorkerJobResult = JobResult & {
 
 const DEFAULT_LLM_MODEL = "local-model";
 const CODEX_UNAVAILABLE_WORKER_EXIT_CODE = 86;
+const CODEX_STARTUP_STALL_WORKER_EXIT_CODE = 87;
 const CODEX_UNAVAILABLE_DOCKER_SHUTDOWN_GRACE_MS = 5_000;
 const CODEX_UNAVAILABLE_WORKER_FORCE_EXIT_MS = 4_000;
 const DEFAULT_JOB_PROGRESS_LOG_EVERY_MS = 60_000;
@@ -391,7 +392,9 @@ function inferWorkerJobPhaseFromLogLine(line: string): WorkerJobPhase | null {
   ) {
     return "full validation";
   }
-  if (/creating commit|Publish blocked|publish-blocked|completion ref|enqueueCompletion/i.test(text)) {
+  if (
+    /creating commit|Publish blocked|publish-blocked|completion ref|enqueueCompletion/i.test(text)
+  ) {
     return "publishing";
   }
   if (
@@ -447,11 +450,20 @@ function mergeWorkerDiagnostics(
   };
 }
 
-function inferWorkerTerminalFailureClass(result: JobResult): string {
+function isCodexStartupStallResult(result: JobResult): boolean {
+  const text =
+    `${result.summary ?? ""}\n${result.stderr ?? ""}\n${result.stdout ?? ""}`.toLowerCase();
+  return /stalled before first response|startup stall/.test(text);
+}
+
+export function inferWorkerTerminalFailureClass(result: JobResult): string {
   if (result.ok) return "success";
-  const text = `${result.summary ?? ""}\n${result.stderr ?? ""}\n${result.stdout ?? ""}`.toLowerCase();
+  const text =
+    `${result.summary ?? ""}\n${result.stderr ?? ""}\n${result.stdout ?? ""}`.toLowerCase();
+  if (isCodexStartupStallResult(result)) return "codex_startup_stall";
   if (/timed out|timeout|signal 15|terminated|exit 143|exit 137/.test(text)) return "timeout";
-  if (/no publishable|non-publishable|node_modules/.test(text)) return "artifact_only_no_publishable_patch";
+  if (/no publishable|non-publishable|node_modules/.test(text))
+    return "artifact_only_no_publishable_patch";
   if (/validationgate|validation/.test(text)) return "validation";
   if (/scopegate|scope/.test(text)) return "scope";
   if (/criticgate|critic/.test(text)) return "critic";
@@ -497,11 +509,12 @@ export function shouldRecycleWorkerForHeartbeatDegradation(options: {
   return options.transportStale;
 }
 
-function shouldRecycleWorkerForCodexUnavailableFailure(
+export function shouldRecycleWorkerForCodexUnavailableFailure(
   summary: string,
   stderr?: string | null,
 ): boolean {
   const text = `${summary}\n${stderr ?? ""}`.toLowerCase();
+  if (/stalled before first response|startup stall/.test(text)) return true;
   return [
     "openai_codex cli is not installed",
     "openai_codex chatgpt auth is not ready",
@@ -512,6 +525,12 @@ function shouldRecycleWorkerForCodexUnavailableFailure(
   ].some((needle) => text.includes(needle));
 }
 
+export function workerRecycleExitCodeForResult(result: WorkerJobResult): number {
+  return isCodexStartupStallResult(result)
+    ? CODEX_STARTUP_STALL_WORKER_EXIT_CODE
+    : CODEX_UNAVAILABLE_WORKER_EXIT_CODE;
+}
+
 async function shutdownDockerExecutorBeforeCodexRecycle(
   dockerExecutor: DockerExecutor | null,
 ): Promise<void> {
@@ -754,14 +773,9 @@ async function createIsolatedWorktree(
     .toLowerCase()
     .replace(/[^a-z0-9]+/g, "")
     .slice(0, 8);
-  const nonce = `${Date.now().toString(36).slice(-6)}-${Math.random()
-    .toString(36)
-    .slice(2, 6)}`;
+  const nonce = `${Date.now().toString(36).slice(-6)}-${Math.random().toString(36).slice(2, 6)}`;
 
-  const worktreePath = resolve(
-    worktreeRoot,
-    `job-${safeJobId || "host"}-${nonce}`,
-  );
+  const worktreePath = resolve(worktreeRoot, `job-${safeJobId || "host"}-${nonce}`);
 
   const addResult = await git(repo, ["worktree", "add", "--detach", worktreePath, baseRef]);
   if (!addResult.ok) {
@@ -1752,6 +1766,7 @@ async function workerLoop(
             const jobAttempt =
               Number.isFinite(jobAttemptRaw) && jobAttemptRaw > 0 ? Math.floor(jobAttemptRaw) : 1;
             const llm = workerLlmConfig(CONFIG);
+            const terminalFailureClass = inferWorkerTerminalFailureClass(result);
             result = {
               ...result,
               diagnostics: mergeWorkerDiagnostics(result.diagnostics, {
@@ -1781,12 +1796,15 @@ async function workerLoop(
                   result.ok ? "completed" : result.publishBlocked ? "publish_blocked" : "failed",
                 ),
                 terminal: {
-                  failureClass: inferWorkerTerminalFailureClass(result),
-                  terminalStage: currentJobPhase ?? (result.ok ? "completed" : "worker"),
+                  failureClass: terminalFailureClass,
+                  terminalStage:
+                    terminalFailureClass === "codex_startup_stall"
+                      ? "executor_startup"
+                      : (currentJobPhase ?? (result.ok ? "completed" : "worker")),
                   executorBackend: resolveExecutor(CONFIG),
                   summary: result.summary,
                   watchdogFired:
-                    /watchdog|rollout coach|timed out|timeout|signal 15|terminated|exit 143|exit 137/i.test(
+                    /watchdog|rollout coach|stalled before first response|startup stall|timed out|timeout|signal 15|terminated|exit 143|exit 137/i.test(
                       `${result.summary}\n${result.stderr ?? ""}\n${result.stdout ?? ""}`,
                     ),
                   metadata: {
@@ -1961,12 +1979,15 @@ async function workerLoop(
             clearInterval(busyHeartbeat);
             if (jobProgressTimer) clearInterval(jobProgressTimer);
             if (recycleWorkerAfterJob) {
+              const recycleExitCode = result
+                ? workerRecycleExitCodeForResult(result)
+                : CODEX_UNAVAILABLE_WORKER_EXIT_CODE;
               runtimeState.shutdownRequested = true;
               const forceExitTimer = setTimeout(() => {
                 console.warn(
                   `[WorkerPals] Forcing worker recycle ${CODEX_UNAVAILABLE_WORKER_FORCE_EXIT_MS}ms after Codex backend failure.`,
                 );
-                process.exit(CODEX_UNAVAILABLE_WORKER_EXIT_CODE);
+                process.exit(recycleExitCode);
               }, CODEX_UNAVAILABLE_WORKER_FORCE_EXIT_MS);
               try {
                 await maybeHeartbeat("offline", null, true);
@@ -1983,7 +2004,7 @@ async function workerLoop(
                 await shutdownDockerExecutorBeforeCodexRecycle(dockerExecutor);
               } finally {
                 clearTimeout(forceExitTimer);
-                process.exit(CODEX_UNAVAILABLE_WORKER_EXIT_CODE);
+                process.exit(recycleExitCode);
               }
             }
             if (job.sessionId && result?.cooldownMs && result.cooldownMs > 0) {