@pushpalsdev/cli 1.1.32 → 1.1.35

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pushpalsdev/cli",
-  "version": "1.1.32",
+  "version": "1.1.35",
   "description": "PushPals terminal CLI for LocalBuddy -> RemoteBuddy orchestration",
   "license": "MIT",
   "repository": {

package/runtime/sandbox/apps/workerpals/src/backends/openai_codex/openai_codex_executor.py

@@ -116,6 +116,8 @@ _WEB_REVIEW_NO_EDIT_WATCHDOG_S = 240
 _BACKGROUND_NO_EDIT_WATCHDOG_S = 120
 _NO_EDIT_RECOVERY_WATCHDOG_S = 90
 _DEFAULT_NO_EDIT_RECHECK_S = 120
+_DEFAULT_STARTUP_STALL_WATCHDOG_S = 210
+_RECOVERY_STARTUP_STALL_WATCHDOG_S = 150
 _DEFAULT_ROLLOUT_WATCHDOG_S = 300
 _SMALL_TASK_ROLLOUT_WATCHDOG_S = 240
 _NARROW_TEST_TASK_ROLLOUT_WATCHDOG_S = 150
@@ -755,6 +757,44 @@ def _resolve_no_edit_recheck_seconds(communicate_timeout_s: Optional[int]) -> in
     return max(1, min(_DEFAULT_NO_EDIT_RECHECK_S, upper))
 
 
+def _resolve_startup_stall_watchdog_seconds(
+    communicate_timeout_s: Optional[int],
+    recovery_attempt: int = 0,
+) -> Optional[int]:
+    if not communicate_timeout_s:
+        return None
+
+    raw = os.environ.get("WORKERPALS_OPENAI_CODEX_STARTUP_STALL_WATCHDOG_S", "").strip()
+    if raw:
+        if raw == "0":
+            return None
+        parsed = _to_positive_int(raw)
+        if parsed is None:
+            log.info(
+                "Invalid WORKERPALS_OPENAI_CODEX_STARTUP_STALL_WATCHDOG_S="
+                f"{raw!r}; using default startup-stall watchdog."
+            )
+        else:
+            return max(1, min(parsed, max(1, communicate_timeout_s - 1)))
+
+    default_s = (
+        _RECOVERY_STARTUP_STALL_WATCHDOG_S
+        if recovery_attempt > 0
+        else _DEFAULT_STARTUP_STALL_WATCHDOG_S
+    )
+    floor_s = 60
+    return max(floor_s, min(default_s, max(floor_s, communicate_timeout_s - 1)))
+
+
+def _startup_stall_recovery_model(current_model: str) -> str:
+    normalized = str(current_model or "").strip()
+    if not normalized:
+        return LEGACY_CODEX_MODEL_FALLBACK
+    if normalized.lower() == LEGACY_CODEX_MODEL_FALLBACK.lower():
+        return normalized
+    return LEGACY_CODEX_MODEL_FALLBACK
+
+
 def _looks_like_web_review_prompt(prompt: str) -> bool:
     text = str(prompt or "").lower()
     return "repo-native web review" in text or "web review path" in text
@@ -2337,6 +2377,15 @@ def _run_codex_task(
                 else None
             )
             no_edit_recheck_s = _resolve_no_edit_recheck_seconds(communicate_timeout_s)
+            startup_stall_watchdog_s = _resolve_startup_stall_watchdog_seconds(
+                communicate_timeout_s,
+                recovery_attempt=startup_stall_recovery_attempt,
+            )
+            startup_stall_deadline = (
+                started_at + float(startup_stall_watchdog_s)
+                if startup_stall_watchdog_s is not None
+                else None
+            )
             rollout_watchdog_s = (
                 _resolve_rollout_watchdog_seconds(
                     prompt,
@@ -2364,9 +2413,50 @@ def _run_codex_task(
                     _terminate_active_child()
                     break
 
+                if startup_stall_deadline is not None and now >= startup_stall_deadline:
+                    with trace_lock:
+                        live_trace = dict(stdout_trace_state)
+                        summaries = stdout_trace_state.get("summaries")
+                        if isinstance(summaries, list):
+                            live_trace["summaries"] = list(summaries)
+                    if _codex_trace_is_startup_stall(live_trace):
+                        changed_paths, _, effective_paths = _codex_changed_paths(repo, baseline_snapshot)
+                        if not effective_paths:
+                            no_edit_artifact_only_paths = _describe_non_publishable_paths(
+                                changed_paths,
+                                baseline_snapshot,
+                            )
+                            no_edit_watchdog_fired = True
+                            elapsed_s = int(max(0.0, now - started_at))
+                            log.info(
+                                f"Startup-stall watchdog fired after {elapsed_s}s with no assistant/tool progress."
+                            )
+                            _terminate_active_child()
+                            break
+                    startup_stall_deadline = None
+
                 if no_edit_deadline is not None and now >= no_edit_deadline:
                     changed_paths, _, effective_paths = _codex_changed_paths(repo, baseline_snapshot)
                     if not effective_paths:
+                        with trace_lock:
+                            live_trace = dict(stdout_trace_state)
+                            summaries = stdout_trace_state.get("summaries")
+                            if isinstance(summaries, list):
+                                live_trace["summaries"] = list(summaries)
+                        startup_only = _codex_trace_is_startup_stall(live_trace)
+                        if (
+                            startup_only
+                            and startup_stall_deadline is not None
+                            and now < startup_stall_deadline
+                        ):
+                            no_edit_deadline = startup_stall_deadline
+                            remaining_s = int(max(1.0, startup_stall_deadline - now))
+                            log.info(
+                                "No-edit watchdog observed only Codex startup events; "
+                                f"allowing {remaining_s}s for first assistant/tool progress "
+                                "before startup-stall recovery."
+                            )
+                            continue
                         no_edit_artifact_only_paths = _describe_non_publishable_paths(
                             changed_paths,
                             baseline_snapshot,
@@ -2377,9 +2467,15 @@ def _run_codex_task(
                             if no_edit_artifact_only_paths
                             else ""
                         )
-                        log.info(
-                            f"No-edit watchdog fired after {int(no_edit_watchdog_s or 0)}s with no publishable file changes.{artifact_detail} Retrying with patch-first guidance."
-                        )
+                        if startup_only:
+                            elapsed_s = int(max(0.0, now - started_at))
+                            log.info(
+                                f"Startup-stall watchdog fired after {elapsed_s}s with no assistant/tool progress."
+                            )
+                        else:
+                            log.info(
+                                f"No-edit watchdog fired after {int(no_edit_watchdog_s or 0)}s with no publishable file changes.{artifact_detail} Retrying with patch-first guidance."
+                            )
                         _terminate_active_child()
                         break
                     no_edit_deadline = now + float(no_edit_recheck_s)
@@ -2550,9 +2646,15 @@ def _run_codex_task(
                     *supplemental_guidance,
                     _build_startup_stall_recovery_guidance(trace_excerpt),
                 ]
+                recovery_model = _startup_stall_recovery_model(model)
+                recovery_detail = (
+                    f" using fallback model {recovery_model!r}"
+                    if recovery_model and recovery_model != model
+                    else ""
+                )
                 log.warning(
                     "Codex emitted only startup events before the no-edit watchdog; "
-                    "restarting Codex once before classifying the job terminally."
+                    f"restarting Codex once{recovery_detail} before classifying the job terminally."
                 )
                 retry_result = _run_codex_task(
                     repo,
@@ -2563,7 +2665,7 @@ def _run_codex_task(
                     startup_stall_recovery_attempt=startup_stall_recovery_attempt + 1,
                     no_edit_recovery_attempt=no_edit_recovery_attempt,
                     rollout_recovery_attempt=rollout_recovery_attempt,
-                    model_override=model_override,
+                    model_override=recovery_model or model_override,
                     baseline_changes=baseline_snapshot,
                 )
                 retry_result["usage"] = _merge_usage_records(usage, retry_result.get("usage"))

package/runtime/sandbox/apps/workerpals/src/backends/openai_codex/test_openai_codex_runtime_config.py

@@ -49,6 +49,7 @@ from openai_codex_executor import (
     _resolve_codex_command_prefix,
     _resolve_no_edit_watchdog_seconds,
     _resolve_rollout_watchdog_seconds,
+    _resolve_startup_stall_watchdog_seconds,
     _unwrap_shell_wrapper_command,
     _usage_from_trace_or_estimate,
 )
@@ -350,6 +351,85 @@ class OpenAICodexRuntimeConfigTests(unittest.TestCase):
             self.assertIn("Home shell startup is assertable", guidance)
             self.assertIn("bun run web:e2e", guidance)
 
+    def test_parse_payload_accepts_file_backed_payload_transport(self) -> None:
+        with tempfile.TemporaryDirectory(prefix="pushpals-payload-file-") as temp_dir:
+            repo = Path(temp_dir) / "repo"
+            repo.mkdir(parents=True, exist_ok=True)
+            payload = {
+                "kind": "task.execute",
+                "repo": str(repo),
+                "params": {"instruction": "Make one small publishable change"},
+            }
+            encoded = base64.b64encode(json.dumps(payload).encode("utf-8")).decode("ascii")
+            payload_file = Path(temp_dir) / "payload.b64"
+            payload_file.write_text(encoded, encoding="utf-8")
+
+            task = parse_task_execute_payload(
+                ["executor", "--payload-file", str(payload_file)],
+                logger=Logger("[test]"),
+            )
+
+            self.assertEqual(task.kind, "task.execute")
+            self.assertEqual(task.repo, str(repo.resolve()))
+            self.assertEqual(task.instruction, "Make one small publishable change")
+
+    def test_parse_payload_accepts_positional_payload_file_path(self) -> None:
+        with tempfile.TemporaryDirectory(prefix="pushpals-payload-file-positional-") as temp_dir:
+            repo = Path(temp_dir) / "repo"
+            repo.mkdir(parents=True, exist_ok=True)
+            payload = {
+                "kind": "task.execute",
+                "repo": str(repo),
+                "params": {"instruction": "Recover from a direct-worker payload handoff"},
+            }
+            encoded = base64.b64encode(json.dumps(payload).encode("utf-8")).decode("ascii")
+            payload_file = Path(temp_dir) / "payload.b64"
+            payload_file.write_text(encoded, encoding="utf-8")
+
+            task = parse_task_execute_payload(
+                ["executor", str(payload_file)],
+                logger=Logger("[test]"),
+            )
+
+            self.assertEqual(task.kind, "task.execute")
+            self.assertEqual(task.repo, str(repo.resolve()))
+            self.assertEqual(task.instruction, "Recover from a direct-worker payload handoff")
+
+    def test_parse_payload_accepts_unpadded_base64_payload(self) -> None:
+        with tempfile.TemporaryDirectory(prefix="pushpals-payload-unpadded-") as temp_dir:
+            repo = Path(temp_dir) / "repo"
+            repo.mkdir(parents=True, exist_ok=True)
+            payload = {
+                "kind": "task.execute",
+                "repo": str(repo),
+                "params": {"instruction": "Accept wrapper-normalized payload padding"},
+            }
+            encoded = base64.b64encode(json.dumps(payload).encode("utf-8")).decode("ascii")
+            unpadded = encoded.rstrip("=")
+
+            task = parse_task_execute_payload(["executor", unpadded], logger=Logger("[test]"))
+
+            self.assertEqual(task.kind, "task.execute")
+            self.assertEqual(task.repo, str(repo.resolve()))
+            self.assertEqual(task.instruction, "Accept wrapper-normalized payload padding")
+
+    def test_parse_payload_accepts_raw_json_payload(self) -> None:
+        with tempfile.TemporaryDirectory(prefix="pushpals-payload-raw-json-") as temp_dir:
+            repo = Path(temp_dir) / "repo"
+            repo.mkdir(parents=True, exist_ok=True)
+            payload = {
+                "kind": "task.execute",
+                "repo": str(repo),
+                "params": {"instruction": "Accept raw JSON from a recovery wrapper"},
+            }
+            raw_json = json.dumps(payload)
+
+            task = parse_task_execute_payload(["executor", raw_json], logger=Logger("[test]"))
+
+            self.assertEqual(task.kind, "task.execute")
+            self.assertEqual(task.repo, str(repo.resolve()))
+            self.assertEqual(task.instruction, "Accept raw JSON from a recovery wrapper")
+
     def test_parse_payload_prefers_helper_tests_for_visual_derivation_tasks(self) -> None:
         with tempfile.TemporaryDirectory(prefix="pushpals-visual-guidance-") as temp_dir:
             repo = Path(temp_dir) / "repo"
@@ -1069,13 +1149,16 @@ class OpenAICodexRuntimeConfigTests(unittest.TestCase):
                         "",
                         "argv = sys.argv[1:]",
                         "last_message_path = None",
+                        "model = ''",
                         "for index, arg in enumerate(argv):",
                         "    if arg == '--output-last-message' and index + 1 < len(argv):",
                         "        last_message_path = argv[index + 1]",
+                        "    if arg == '-m' and index + 1 < len(argv):",
+                        "        model = argv[index + 1]",
                         "        break",
                         "",
                         "prompt = sys.stdin.read()",
-                        "if 'Codex startup-stall recovery' in prompt:",
+                        "if 'Codex startup-stall recovery' in prompt and model == 'gpt-5.4':",
                         "    Path('src').mkdir(exist_ok=True)",
                         "    Path('src/startup-stall-recovered.txt').write_text('patched after restart\\n', encoding='utf-8')",
                         "    if last_message_path:",
@@ -1097,7 +1180,8 @@ class OpenAICodexRuntimeConfigTests(unittest.TestCase):
                 "OPENAI_API_KEY": "pushpals-startup-stall-test-key",
                 "WORKERPALS_OPENAI_CODEX_JSON": "true",
                 "WORKERPALS_OPENAI_CODEX_TIMEOUT_S": "20",
-                "WORKERPALS_OPENAI_CODEX_NO_EDIT_WATCHDOG_S": "1",
+                "WORKERPALS_OPENAI_CODEX_NO_EDIT_WATCHDOG_S": "0",
+                "WORKERPALS_OPENAI_CODEX_STARTUP_STALL_WATCHDOG_S": "1",
                 "WORKERPALS_OPENAI_CODEX_PROGRESS_LOG_INTERVAL_S": "1",
             }
             with mock.patch.dict(os.environ, env_overrides, clear=False):
@@ -1167,6 +1251,7 @@ class OpenAICodexRuntimeConfigTests(unittest.TestCase):
                 "WORKERPALS_OPENAI_CODEX_JSON": "true",
                 "WORKERPALS_OPENAI_CODEX_TIMEOUT_S": "20",
                 "WORKERPALS_OPENAI_CODEX_NO_EDIT_WATCHDOG_S": "1",
+                "WORKERPALS_OPENAI_CODEX_STARTUP_STALL_WATCHDOG_S": "1",
                 "WORKERPALS_OPENAI_CODEX_PROGRESS_LOG_INTERVAL_S": "1",
             }
             with mock.patch.dict(os.environ, env_overrides, clear=False):
@@ -1565,6 +1650,31 @@ class OpenAICodexRuntimeConfigTests(unittest.TestCase):
 
         self.assertEqual(watchdog_s, 180)
 
+    def test_startup_stall_watchdog_allows_slower_first_response_than_no_edit_watchdog(self) -> None:
+        with mock.patch.dict(
+            os.environ,
+            {"WORKERPALS_OPENAI_CODEX_STARTUP_STALL_WATCHDOG_S": ""},
+            clear=False,
+        ):
+            watchdog_s = _resolve_startup_stall_watchdog_seconds(1200)
+            recovery_watchdog_s = _resolve_startup_stall_watchdog_seconds(
+                1200,
+                recovery_attempt=1,
+            )
+
+        self.assertEqual(watchdog_s, 210)
+        self.assertEqual(recovery_watchdog_s, 150)
+
+    def test_explicit_startup_stall_watchdog_override_is_bounded(self) -> None:
+        with mock.patch.dict(
+            os.environ,
+            {"WORKERPALS_OPENAI_CODEX_STARTUP_STALL_WATCHDOG_S": "500"},
+            clear=False,
+        ):
+            watchdog_s = _resolve_startup_stall_watchdog_seconds(120)
+
+        self.assertEqual(watchdog_s, 119)
+
     def test_narrow_contract_regression_with_required_e2e_uses_fast_no_edit_watchdog(self) -> None:
         prompt = (
             "Harden the opportunity graph contract around autonomous delivery-loop failure signals. "

package/runtime/sandbox/apps/workerpals/src/backends/openhands_task_execute.ts

@@ -17,6 +17,10 @@ import {
   filterResultLines,
 } from "../common/execution_utils.js";
 import { buildWorkerSandboxWritableEnv } from "../common/sandbox_env.js";
+import {
+  createPythonPayloadTransport,
+  type PythonPayloadTransport,
+} from "../common/python_payload_transport.js";
 import { computeTimeoutWarningWindow } from "../timeout_policy.js";
 
 // ---- Script path (resolved relative to this file) ----------------------------
@@ -274,6 +278,7 @@ export async function executeWithOpenHands(
   let timeoutTimer: ReturnType<typeof setTimeout> | null = null;
   let stuckNudgeStartTimer: ReturnType<typeof setTimeout> | null = null;
   let stuckNudgeTimer: ReturnType<typeof setInterval> | null = null;
+  let payloadTransport: PythonPayloadTransport | null = null;
   const outputPolicy = {
     maxOutputChars: runtimeConfig.workerpals.outputMaxChars,
     maxOutputLines: runtimeConfig.workerpals.outputMaxLines,
@@ -282,7 +287,8 @@ export async function executeWithOpenHands(
   };
 
   try {
-    const proc = Bun.spawn([pythonBin, scriptPath, payload], {
+    payloadTransport = createPythonPayloadTransport(payload);
+    const proc = Bun.spawn([pythonBin, scriptPath, ...payloadTransport.args], {
       cwd: repo,
       stdout: "pipe",
       stderr: "pipe",
@@ -623,5 +629,6 @@ export async function executeWithOpenHands(
     if (stuckNudgeTimer) {
       clearInterval(stuckNudgeTimer);
     }
+    payloadTransport?.cleanup();
   }
 }

package/runtime/sandbox/apps/workerpals/src/backends/shared/executor_base.py

@@ -155,14 +155,60 @@ def fail(summary: str, stderr: Optional[str] = None, exit_code: int = 1) -> int:
     return exit_code
 
 
-def decode_payload(raw: str) -> Dict[str, Any]:
-    decoded = base64.b64decode(raw).decode("utf-8")
-    payload = json.loads(decoded)
+def _parse_payload_json(raw: str) -> Dict[str, Any]:
+    payload = json.loads(raw)
     if not isinstance(payload, dict):
         raise ValueError("payload must be a JSON object")
     return payload
 
 
+def decode_payload(raw: str) -> Dict[str, Any]:
+    stripped = str(raw or "").strip()
+    if not stripped:
+        raise ValueError("empty job payload")
+
+    # Direct workers normally receive a file-backed base64 payload, but this
+    # parser intentionally accepts the safe adjacent encodings too. That keeps
+    # executor startup resilient if an outer wrapper normalizes padding, uses
+    # url-safe base64, or hands through raw JSON during recovery.
+    if stripped.startswith("{"):
+        return _parse_payload_json(stripped)
+
+    compact = "".join(stripped.split())
+    padded = compact + ("=" * ((4 - len(compact) % 4) % 4))
+    decode_errors: List[str] = []
+    for decoder in (base64.b64decode, base64.urlsafe_b64decode):
+        try:
+            decoded = decoder(padded).decode("utf-8")
+            return _parse_payload_json(decoded)
+        except Exception as exc:
+            decode_errors.append(str(exc))
+
+    detail = "; ".join(error for error in decode_errors if error) or "unknown decode error"
+    raise ValueError(f"invalid base64/JSON job payload: {detail}")
+
+
+def read_encoded_payload_arg(argv: List[str]) -> str:
+    if len(argv) < 2:
+        raise ValueError("missing base64 job payload")
+    mode = argv[1]
+    if mode == "--payload-file":
+        if len(argv) < 3 or not str(argv[2] or "").strip():
+            raise ValueError("missing payload file path")
+        path = Path(str(argv[2])).expanduser()
+        return path.read_text(encoding="utf-8").strip()
+    if mode == "--payload-stdin":
+        return sys.stdin.read().strip()
+    if len(mode) < 4096:
+        try:
+            path = Path(mode).expanduser()
+            if path.is_file():
+                return path.read_text(encoding="utf-8").strip()
+        except OSError:
+            pass
+    return mode
+
+
 def resolve_repo_within_assigned_root(repo: str) -> Tuple[Optional[str], Optional[str]]:
     raw_repo = str(repo or "").strip()
     if not raw_repo:
@@ -968,11 +1014,8 @@ def parse_task_execute_payload(
     don't need to handle them.
     """
     log = logger or Logger("[Executor]")
-    if len(argv) < 2:
-        raise SystemExit(fail("Missing base64 job payload", exit_code=2))
-
     try:
-        payload = decode_payload(argv[1])
+        payload = decode_payload(read_encoded_payload_arg(argv))
     except Exception as exc:
         raise SystemExit(fail(f"Failed to decode job payload: {exc}", exit_code=2))
 

package/runtime/sandbox/apps/workerpals/src/common/generic_python_executor.ts

@@ -19,6 +19,10 @@ import {
   streamLines,
 } from "./execution_utils.js";
 import { buildWorkerSandboxWritableEnv } from "./sandbox_env.js";
+import {
+  createPythonPayloadTransport,
+  type PythonPayloadTransport,
+} from "./python_payload_transport.js";
 
 interface GenericPythonExecutorConfig {
   backendName: string;
@@ -357,7 +361,6 @@ export function createGenericPythonExecutor(
       }),
       "utf-8",
     ).toString("base64");
-    const args = [pythonBin, scriptPath, payloadBase64];
     const childTimeoutMs = resolveGenericPythonExecutorChildTimeoutMs({
       backendName,
       hostTimeoutMs: timeoutMs,
@@ -379,12 +382,15 @@ export function createGenericPythonExecutor(
           )}ms`
         : "";
 
-    onLog?.(
-      "stdout",
-      `[${backendLabel}Executor] Spawning ${backendName} executor (timeout=${timeoutMs}ms; ${timeoutDetail}${childTimeoutDetail})`,
-    );
-
+    let payloadTransport: PythonPayloadTransport | null = null;
     try {
+      payloadTransport = createPythonPayloadTransport(payloadBase64);
+      const args = [pythonBin, scriptPath, ...payloadTransport.args];
+      onLog?.(
+        "stdout",
+        `[${backendLabel}Executor] Spawning ${backendName} executor (timeout=${timeoutMs}ms; ${timeoutDetail}${childTimeoutDetail})`,
+      );
+
       const outputPolicy = {
         maxOutputChars: runtimeConfig.workerpals.outputMaxChars,
         maxOutputLines: runtimeConfig.workerpals.outputMaxLines,
@@ -539,6 +545,8 @@ export function createGenericPythonExecutor(
           "",
         ),
       };
+    } finally {
+      payloadTransport?.cleanup();
     }
   };
 }

package/runtime/sandbox/apps/workerpals/src/common/python_payload_transport.ts

@@ -0,0 +1,26 @@
+import { mkdtempSync, rmSync, writeFileSync } from "fs";
+import { tmpdir } from "os";
+import { join } from "path";
+
+export interface PythonPayloadTransport {
+  args: string[];
+  filePath: string;
+  cleanup: () => void;
+}
+
+export function createPythonPayloadTransport(payloadBase64: string): PythonPayloadTransport {
+  const dir = mkdtempSync(join(tmpdir(), "pushpals-python-payload-"));
+  const filePath = join(dir, "payload.b64");
+  writeFileSync(filePath, payloadBase64, { encoding: "utf8", mode: 0o600 });
+
+  let cleaned = false;
+  return {
+    args: ["--payload-file", filePath],
+    filePath,
+    cleanup: () => {
+      if (cleaned) return;
+      cleaned = true;
+      rmSync(dir, { recursive: true, force: true });
+    },
+  };
+}

package/runtime/sandbox/apps/workerpals/src/docker_executor.ts

@@ -1918,8 +1918,6 @@ export class DockerExecutor {
 
   private matchesRetryablePattern(text: string): boolean {
     const transientPatterns: RegExp[] = [
-      /\bstalled before first response\b/i,
-      /\bstartup stall\b/i,
       /warm .*runtime/i,
       /failed to start warm container/i,
       /docker execution error/i,