@pushpalsdev/cli 1.1.26 → 1.1.27

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pushpalsdev/cli",
-  "version": "1.1.26",
+  "version": "1.1.27",
   "description": "PushPals terminal CLI for LocalBuddy -> RemoteBuddy orchestration",
   "license": "MIT",
   "repository": {

package/runtime/sandbox/apps/workerpals/src/backends/openai_codex/openai_codex_executor.py

@@ -106,6 +106,7 @@ _MAX_WRAPPER_BOOTSTRAP_OUTPUT_CHARS = 1_200
 _MAX_WRAPPER_BOOTSTRAP_TOTAL_CHARS = 5_000
 _MAX_CREDIBLE_WRAPPER_LOOP_CHANGED_PATHS = 8
 _MAX_CREDIBLE_WRAPPER_LOOP_TOP_LEVELS = 4
+_MAX_STARTUP_STALL_RECOVERY_ATTEMPTS = 1
 _MAX_NO_EDIT_RECOVERY_ATTEMPTS = 1
 _MAX_ROLLOUT_RECOVERY_ATTEMPTS = 1
 _DEFAULT_NO_EDIT_WATCHDOG_S = 480
@@ -121,6 +122,7 @@ _NARROW_TEST_TASK_ROLLOUT_WATCHDOG_S = 150
 _WEB_REVIEW_ROLLOUT_WATCHDOG_S = 180
 _BACKGROUND_ROLLOUT_WATCHDOG_S = 90
 _NO_PUBLISHABLE_FAILURE_COOLDOWN_MS = 10 * 60 * 1000
+_CODEX_STARTUP_ONLY_EVENT_TYPES = {"thread.started", "turn.started"}
 
 
 def _model_supports_xhigh_reasoning(model: str) -> bool:
@@ -862,6 +864,19 @@ def _build_no_edit_recovery_guidance(trace_excerpt: str, artifact_only_paths: st
     return "\n".join(lines)
 
 
+def _build_startup_stall_recovery_guidance(trace_excerpt: str) -> str:
+    lines = [
+        "Codex startup-stall recovery: the previous Codex subprocess started but emitted no assistant, tool, or reasoning progress before the watchdog.",
+        "Treat this as a fresh execution with a patch-first contract. After at most one narrow read of the hinted owner, make the smallest publishable edit.",
+        "Do not spend this recovery attempt re-reading broad repository topology or validating before an edit exists.",
+        "If the hinted path is absent, choose the nearest existing repo-native owner or test rather than creating unrelated scaffolding.",
+    ]
+    if trace_excerpt:
+        lines.append("Previous Codex event trace excerpt:")
+        lines.append(trace_excerpt)
+    return "\n".join(lines)
+
+
 def _trace_summaries_text(trace: Dict[str, Any]) -> str:
     summaries = trace.get("summaries")
     if not isinstance(summaries, list):
@@ -869,6 +884,36 @@ def _trace_summaries_text(trace: Dict[str, Any]) -> str:
     return "\n".join(str(item or "") for item in summaries[-80:]).lower()
 
 
+def _codex_trace_has_work_progress(trace: Dict[str, Any]) -> bool:
+    if to_int(trace.get("reasoning_events"), 0) > 0:
+        return True
+
+    event_counts = trace.get("event_type_counts")
+    if isinstance(event_counts, dict):
+        for key, value in event_counts.items():
+            event_type = str(key or "").strip()
+            if to_int(value, 0) > 0 and event_type not in _CODEX_STARTUP_ONLY_EVENT_TYPES:
+                return True
+
+    summaries = trace.get("summaries")
+    if isinstance(summaries, list):
+        for item in summaries:
+            summary = str(item or "").strip()
+            if not summary:
+                continue
+            event_type = summary.split("|", 1)[0].strip()
+            if event_type not in _CODEX_STARTUP_ONLY_EVENT_TYPES:
+                return True
+
+    return False
+
+
+def _codex_trace_is_startup_stall(trace: Dict[str, Any]) -> bool:
+    if to_int(trace.get("total_tokens"), 0) > 0:
+        return False
+    return not _codex_trace_has_work_progress(trace)
+
+
 def _detect_offtrack_rollout(trace: Dict[str, Any], artifact_only_paths: str = "") -> str:
     text = _trace_summaries_text(trace)
     if artifact_only_paths:
@@ -1981,6 +2026,7 @@ def _run_codex_task(
     *,
     wrapper_recovery_attempt: int = 0,
     model_compatibility_recovery_attempt: int = 0,
+    startup_stall_recovery_attempt: int = 0,
     no_edit_recovery_attempt: int = 0,
     rollout_recovery_attempt: int = 0,
     model_override: Optional[str] = None,
@@ -2475,6 +2521,7 @@ def _run_codex_task(
                     retry_guidance,
                     wrapper_recovery_attempt=wrapper_recovery_attempt,
                     model_compatibility_recovery_attempt=model_compatibility_recovery_attempt,
+                    startup_stall_recovery_attempt=startup_stall_recovery_attempt,
                     no_edit_recovery_attempt=no_edit_recovery_attempt,
                     rollout_recovery_attempt=rollout_recovery_attempt + 1,
                     model_override=model_override,
@@ -2497,6 +2544,54 @@ def _run_codex_task(
             }
 
         if no_edit_watchdog_fired:
+            startup_stall = _codex_trace_is_startup_stall(stdout_trace)
+            if startup_stall and startup_stall_recovery_attempt < _MAX_STARTUP_STALL_RECOVERY_ATTEMPTS:
+                retry_guidance = [
+                    *supplemental_guidance,
+                    _build_startup_stall_recovery_guidance(trace_excerpt),
+                ]
+                log.warning(
+                    "Codex emitted only startup events before the no-edit watchdog; "
+                    "restarting Codex once before classifying the job terminally."
+                )
+                retry_result = _run_codex_task(
+                    repo,
+                    instruction,
+                    retry_guidance,
+                    wrapper_recovery_attempt=wrapper_recovery_attempt,
+                    model_compatibility_recovery_attempt=model_compatibility_recovery_attempt,
+                    startup_stall_recovery_attempt=startup_stall_recovery_attempt + 1,
+                    no_edit_recovery_attempt=no_edit_recovery_attempt,
+                    rollout_recovery_attempt=rollout_recovery_attempt,
+                    model_override=model_override,
+                    baseline_changes=baseline_snapshot,
+                )
+                retry_result["usage"] = _merge_usage_records(usage, retry_result.get("usage"))
+                if retry_result.get("ok"):
+                    recovered_stdout = str(retry_result.get("stdout") or "").strip()
+                    retry_result["stdout"] = _truncate(
+                        (
+                            "Recovered after the first Codex subprocess stalled before emitting "
+                            f"assistant/tool progress.\n\n{recovered_stdout}"
+                        ).strip()
+                    )
+                return retry_result
+            if startup_stall:
+                detail = (
+                    "Codex subprocess started but did not emit assistant, tool, reasoning, "
+                    "or usage progress before the startup watchdog."
+                )
+                if trace_excerpt:
+                    detail = f"{detail}\n{trace_excerpt}"
+                return {
+                    "ok": False,
+                    "summary": "openai_codex stalled before first response",
+                    "stdout": _truncate(stdout),
+                    "stderr": _truncate(f"{detail}\n{stderr}".strip()),
+                    "exitCode": 124,
+                    "usage": usage,
+                    "cooldownMs": _NO_PUBLISHABLE_FAILURE_COOLDOWN_MS,
+                }
             if no_edit_recovery_attempt < _MAX_NO_EDIT_RECOVERY_ATTEMPTS:
                 retry_guidance = [
                     *supplemental_guidance,
@@ -2511,6 +2606,7 @@ def _run_codex_task(
                     retry_guidance,
                     wrapper_recovery_attempt=wrapper_recovery_attempt,
                     model_compatibility_recovery_attempt=model_compatibility_recovery_attempt,
+                    startup_stall_recovery_attempt=startup_stall_recovery_attempt,
                     no_edit_recovery_attempt=no_edit_recovery_attempt + 1,
                     rollout_recovery_attempt=rollout_recovery_attempt,
                     model_override=model_override,
@@ -2706,6 +2802,7 @@ def _run_codex_task(
                         ],
                         wrapper_recovery_attempt=wrapper_recovery_attempt + 1,
                         model_compatibility_recovery_attempt=model_compatibility_recovery_attempt,
+                        startup_stall_recovery_attempt=startup_stall_recovery_attempt,
                         no_edit_recovery_attempt=no_edit_recovery_attempt,
                         rollout_recovery_attempt=rollout_recovery_attempt,
                         model_override=model_override,
@@ -2820,6 +2917,7 @@ def _run_codex_task(
                     effective_supplemental_guidance,
                     wrapper_recovery_attempt=wrapper_recovery_attempt,
                     model_compatibility_recovery_attempt=model_compatibility_recovery_attempt + 1,
+                    startup_stall_recovery_attempt=startup_stall_recovery_attempt,
                     no_edit_recovery_attempt=no_edit_recovery_attempt,
                     rollout_recovery_attempt=rollout_recovery_attempt,
                     model_override=LEGACY_CODEX_MODEL_FALLBACK,

package/runtime/sandbox/apps/workerpals/src/backends/openai_codex/test_openai_codex_runtime_config.py

@@ -1029,6 +1029,159 @@ class OpenAICodexRuntimeConfigTests(unittest.TestCase):
         self.assertNotIn("broad/noisy", str(result.get("summary") or ""))
         self.assertNotIn("too broad/noisy", str(result.get("stderr") or ""))
 
+    def test_run_codex_task_retries_once_when_codex_stalls_before_first_response(self) -> None:
+        with tempfile.TemporaryDirectory(prefix="pushpals-codex-startup-stall-") as temp_dir:
+            repo = Path(temp_dir) / "repo"
+            repo.mkdir(parents=True, exist_ok=True)
+            (repo / "README.md").write_text("# startup stall repo\n", encoding="utf-8")
+            subprocess.run(["git", "init"], cwd=repo, check=True, capture_output=True, text=True)
+            subprocess.run(
+                ["git", "config", "user.name", "PushPals Test"],
+                cwd=repo,
+                check=True,
+                capture_output=True,
+                text=True,
+            )
+            subprocess.run(
+                ["git", "config", "user.email", "pushpals-tests@example.com"],
+                cwd=repo,
+                check=True,
+                capture_output=True,
+                text=True,
+            )
+            subprocess.run(["git", "add", "README.md"], cwd=repo, check=True, capture_output=True, text=True)
+            subprocess.run(
+                ["git", "commit", "-m", "chore: seed startup stall repo"],
+                cwd=repo,
+                check=True,
+                capture_output=True,
+                text=True,
+            )
+
+            stub_path = Path(temp_dir) / "fake_codex_startup_stall.py"
+            stub_path.write_text(
+                "\n".join(
+                    [
+                        "from pathlib import Path",
+                        "import json",
+                        "import sys",
+                        "import time",
+                        "",
+                        "argv = sys.argv[1:]",
+                        "last_message_path = None",
+                        "for index, arg in enumerate(argv):",
+                        "    if arg == '--output-last-message' and index + 1 < len(argv):",
+                        "        last_message_path = argv[index + 1]",
+                        "        break",
+                        "",
+                        "prompt = sys.stdin.read()",
+                        "if 'Codex startup-stall recovery' in prompt:",
+                        "    Path('src').mkdir(exist_ok=True)",
+                        "    Path('src/startup-stall-recovered.txt').write_text('patched after restart\\n', encoding='utf-8')",
+                        "    if last_message_path:",
+                        "        Path(last_message_path).write_text('Patched after Codex startup-stall recovery.', encoding='utf-8')",
+                        "    print(json.dumps({'type': 'item.completed', 'message': 'Patched after Codex startup-stall recovery.'}), flush=True)",
+                        "    sys.exit(0)",
+                        "",
+                        "print(json.dumps({'type': 'thread.started'}), flush=True)",
+                        "print(json.dumps({'type': 'turn.started'}), flush=True)",
+                        "time.sleep(10)",
+                    ]
+                ),
+                encoding="utf-8",
+            )
+
+            env_overrides = {
+                "PUSHPALS_OPENAI_CODEX_BIN_JSON": json.dumps([sys.executable, str(stub_path)]),
+                "PUSHPALS_OPENAI_CODEX_AUTH_MODE": "api_key",
+                "OPENAI_API_KEY": "pushpals-startup-stall-test-key",
+                "WORKERPALS_OPENAI_CODEX_JSON": "true",
+                "WORKERPALS_OPENAI_CODEX_TIMEOUT_S": "20",
+                "WORKERPALS_OPENAI_CODEX_NO_EDIT_WATCHDOG_S": "1",
+                "WORKERPALS_OPENAI_CODEX_PROGRESS_LOG_INTERVAL_S": "1",
+            }
+            with mock.patch.dict(os.environ, env_overrides, clear=False):
+                result = _run_codex_task(
+                    str(repo),
+                    "Rename one misleading test fixture constant and update the related assertions.",
+                    [],
+                )
+
+        self.assertTrue(result.get("ok"), result)
+        self.assertEqual(result.get("exitCode"), 0)
+        stdout = str(result.get("stdout") or "")
+        self.assertIn("Recovered after the first Codex subprocess stalled", stdout)
+        self.assertIn("Patched after Codex startup-stall recovery", stdout)
+        self.assertIn("src/", stdout)
+
+    def test_run_codex_task_reports_startup_stall_when_restart_also_never_responds(self) -> None:
+        with tempfile.TemporaryDirectory(prefix="pushpals-codex-startup-stall-fail-") as temp_dir:
+            repo = Path(temp_dir) / "repo"
+            repo.mkdir(parents=True, exist_ok=True)
+            (repo / "README.md").write_text("# startup stall failure repo\n", encoding="utf-8")
+            subprocess.run(["git", "init"], cwd=repo, check=True, capture_output=True, text=True)
+            subprocess.run(
+                ["git", "config", "user.name", "PushPals Test"],
+                cwd=repo,
+                check=True,
+                capture_output=True,
+                text=True,
+            )
+            subprocess.run(
+                ["git", "config", "user.email", "pushpals-tests@example.com"],
+                cwd=repo,
+                check=True,
+                capture_output=True,
+                text=True,
+            )
+            subprocess.run(["git", "add", "README.md"], cwd=repo, check=True, capture_output=True, text=True)
+            subprocess.run(
+                ["git", "commit", "-m", "chore: seed startup stall failure repo"],
+                cwd=repo,
+                check=True,
+                capture_output=True,
+                text=True,
+            )
+
+            stub_path = Path(temp_dir) / "fake_codex_startup_stall_fail.py"
+            stub_path.write_text(
+                "\n".join(
+                    [
+                        "import json",
+                        "import sys",
+                        "import time",
+                        "",
+                        "sys.stdin.read()",
+                        "print(json.dumps({'type': 'thread.started'}), flush=True)",
+                        "print(json.dumps({'type': 'turn.started'}), flush=True)",
+                        "time.sleep(10)",
+                    ]
+                ),
+                encoding="utf-8",
+            )
+
+            env_overrides = {
+                "PUSHPALS_OPENAI_CODEX_BIN_JSON": json.dumps([sys.executable, str(stub_path)]),
+                "PUSHPALS_OPENAI_CODEX_AUTH_MODE": "api_key",
+                "OPENAI_API_KEY": "pushpals-startup-stall-fail-test-key",
+                "WORKERPALS_OPENAI_CODEX_JSON": "true",
+                "WORKERPALS_OPENAI_CODEX_TIMEOUT_S": "20",
+                "WORKERPALS_OPENAI_CODEX_NO_EDIT_WATCHDOG_S": "1",
+                "WORKERPALS_OPENAI_CODEX_PROGRESS_LOG_INTERVAL_S": "1",
+            }
+            with mock.patch.dict(os.environ, env_overrides, clear=False):
+                result = _run_codex_task(
+                    str(repo),
+                    "Rename one misleading test fixture constant and update the related assertions.",
+                    [],
+                )
+
+        self.assertFalse(result.get("ok"), result)
+        self.assertEqual(result.get("exitCode"), 124)
+        self.assertEqual(result.get("summary"), "openai_codex stalled before first response")
+        self.assertNotIn("no publishable", str(result.get("summary") or "").lower())
+        self.assertEqual(result.get("cooldownMs"), 600000)
+
     def test_run_codex_task_retries_once_when_no_edit_watchdog_fires(self) -> None:
         with tempfile.TemporaryDirectory(prefix="pushpals-codex-no-edit-watchdog-") as temp_dir:
             repo = Path(temp_dir) / "repo"

package/runtime/sandbox/apps/workerpals/src/workerpals_main.ts

@@ -450,7 +450,7 @@ function mergeWorkerDiagnostics(
 function inferWorkerTerminalFailureClass(result: JobResult): string {
   if (result.ok) return "success";
   const text = `${result.summary ?? ""}\n${result.stderr ?? ""}\n${result.stdout ?? ""}`.toLowerCase();
-  if (/timed out|timeout|signal 15|terminated|exit 143|exit 137/.test(text)) return "timeout";
+  if (/timed out|timeout|signal 15|terminated|exit 143|exit 137|stalled before first response|startup stall/.test(text)) return "timeout";
   if (/no publishable|non-publishable|node_modules/.test(text)) return "artifact_only_no_publishable_patch";
   if (/validationgate|validation/.test(text)) return "validation";
   if (/scopegate|scope/.test(text)) return "scope";