@pushpalsdev/cli 1.1.27 → 1.1.28

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pushpalsdev/cli",
-  "version": "1.1.27",
+  "version": "1.1.28",
   "description": "PushPals terminal CLI for LocalBuddy -> RemoteBuddy orchestration",
   "license": "MIT",
   "repository": {

package/runtime/sandbox/.pushpals-remotebuddy-fallback.js

@@ -8408,6 +8408,7 @@ ${detail}`.toLowerCase();
     "codex cli is mandatory in this backend"
   ].some((needle) => text.includes(needle));
 }
+var CODEX_STARTUP_STALL_WORKER_EXIT_CODE = 87;
 function asAutonomyComponentArea2(value) {
   return normalizeAutonomyComponentArea(value) ?? undefined;
 }
@@ -8950,6 +8951,7 @@ class RemoteBuddyOrchestrator {
   workerpalsEnvFile;
   workerpalsEntrypoint;
   workerpalsUnavailableReason;
+  workerDockerFallbackActivated = false;
   statusHeartbeatMs;
   fetchFailureLogsOnJobFailure;
   executionBudgetInteractiveMs;
@@ -9956,6 +9958,25 @@ Please reply with the missing details and I will enqueue a follow-up request.` :
       entrypoint: this.workerpalsEntrypoint
     });
   }
+  maybeFallbackFromDockerAfterWorkerExit(workerId, code) {
+    if (code !== CODEX_STARTUP_STALL_WORKER_EXIT_CODE)
+      return false;
+    if (!this.spawnWorkerDocker)
+      return false;
+    if (this.workerDockerFallbackActivated)
+      return false;
+    if (parseEnabledFlag(process.env.REMOTEBUDDY_DISABLE_WORKERPAL_DIRECT_FALLBACK, false)) {
+      console.warn(`[RemoteBuddy] WorkerPal ${workerId} exited after a Docker Codex startup stall, but direct WorkerPal fallback is disabled.`);
+      return false;
+    }
+    this.workerDockerFallbackActivated = true;
+    this.spawnWorkerDocker = false;
+    this.spawnWorkerRequireDocker = false;
+    this.workerSpawnCooldownUntil = 0;
+    this.workerpalsUnavailableReason = "Docker-backed WorkerPal Codex startup stalled; falling back to direct isolated-worktree WorkerPal.";
+    console.warn(`[RemoteBuddy] WorkerPal ${workerId} exited after a Docker Codex startup stall; falling back to direct isolated-worktree WorkerPal for future spawns.`);
+    return true;
+  }
   async spawnWorker() {
     if (this.workerSpawnInFlight) {
       return await this.workerSpawnInFlight;
@@ -9983,6 +10004,9 @@ Please reply with the missing details and I will enqueue a follow-up request.` :
         this.managedWorkers.set(workerId, child);
         child.exited.then((code) => {
           this.managedWorkers.delete(workerId);
+          if (this.maybeFallbackFromDockerAfterWorkerExit(workerId, code)) {
+            this.ensureAutoscaledWorkerCapacity("docker codex startup fallback");
+          }
           console.warn(`[RemoteBuddy] WorkerPal process ${workerId} exited with code ${code}`);
         });
         const ready = await this.waitForOnlineWorker(this.workerStartupTimeoutMs, workerId);

package/runtime/sandbox/apps/workerpals/src/common/types.ts

@@ -92,6 +92,7 @@ export interface JobResult {
   stdout?: string;
   stderr?: string;
   exitCode?: number;
+  cooldownMs?: number;
   usage?: JobTokenUsage;
   publishBlocked?: JobPublishBlockedInfo;
   diagnostics?: JobDiagnostics;

package/runtime/sandbox/apps/workerpals/src/docker_executor.ts

@@ -249,7 +249,9 @@ export interface Job {
 }
 
 function compactDockerDiagnosticText(value: unknown, maxChars = 1000): string | null {
-  const text = String(value ?? "").replace(/\s+$/g, "").trim();
+  const text = String(value ?? "")
+    .replace(/\s+$/g, "")
+    .trim();
   if (!text) return null;
   return text.length <= maxChars ? text : text.slice(0, maxChars);
 }
@@ -482,11 +484,11 @@ export class DockerExecutor {
             if (
               retryableFailure &&
               attempt >= this.jobRetryMaxAttempts &&
-              this.failureCooldownMs > 0
+              this.retryExhaustionCooldownMs(result) > 0
             ) {
               return {
                 ...result,
-                cooldownMs: this.failureCooldownMs,
+                cooldownMs: this.retryExhaustionCooldownMs(result),
               };
             }
             return result;
@@ -1279,9 +1281,8 @@ export class DockerExecutor {
       onLog?.("stdout", note);
     }
 
-    const { leadMs: warningLeadMs, delayMs: warningDelayMs } = computeTimeoutWarningWindow(
-      timeoutMs,
-    );
+    const { leadMs: warningLeadMs, delayMs: warningDelayMs } =
+      computeTimeoutWarningWindow(timeoutMs);
     const warningTimer = setTimeout(() => {
       const warning = `[DockerExecutor] Job nearing timeout in warm container (${Math.round(
         warningLeadMs / 1000,
@@ -1424,13 +1425,13 @@ export class DockerExecutor {
     const worktreePrefix = shellSingleQuote(`${containerWorktreePath}/`);
     const command = [
       "set -eu",
-      "linked=\"\"",
+      'linked=""',
       "for name in node_modules; do",
-      "  src=\"/repo/$name\"",
+      '  src="/repo/$name"',
       `  dest=${worktreePrefix}$name`,
-      "  if { [ -e \"$src\" ] || [ -L \"$src\" ]; } && [ ! -e \"$dest\" ] && [ ! -L \"$dest\" ]; then",
-      "    ln -s \"$src\" \"$dest\"",
-      "    linked=\"$linked $name\"",
+      '  if { [ -e "$src" ] || [ -L "$src" ]; } && [ ! -e "$dest" ] && [ ! -L "$dest" ]; then',
+      '    ln -s "$src" "$dest"',
+      '    linked="$linked $name"',
       "  fi",
       "done",
       "printf '%s' \"$linked\"",
@@ -1454,9 +1455,7 @@ export class DockerExecutor {
       .filter(Boolean);
     if (linked.length === 0) return;
 
-    const note = `[DockerExecutor] Linked worktree dependency artifact(s): ${linked.join(
-      ", ",
-    )}`;
+    const note = `[DockerExecutor] Linked worktree dependency artifact(s): ${linked.join(", ")}`;
     console.log(note);
     onLog?.("stdout", note);
   }
@@ -1701,9 +1700,15 @@ export class DockerExecutor {
         stdout,
         stderr: details.join("\n"),
         exitCode,
-        diagnostics: dockerFallbackDiagnostics(summary, context, exitCode, "malformed_structured_result", {
-          sentinelParseError,
-        }),
+        diagnostics: dockerFallbackDiagnostics(
+          summary,
+          context,
+          exitCode,
+          "malformed_structured_result",
+          {
+            sentinelParseError,
+          },
+        ),
       };
     }
 
@@ -1906,8 +1911,15 @@ export class DockerExecutor {
     return this.matchesRetryablePattern(text);
   }
 
+  private retryExhaustionCooldownMs(result: DockerJobResult): number {
+    const resultCooldownMs = readPositiveNumber(result.cooldownMs) ?? 0;
+    return Math.max(this.failureCooldownMs, resultCooldownMs);
+  }
+
   private matchesRetryablePattern(text: string): boolean {
     const transientPatterns: RegExp[] = [
+      /\bstalled before first response\b/i,
+      /\bstartup stall\b/i,
       /warm .*runtime/i,
       /failed to start warm container/i,
       /docker execution error/i,

package/runtime/sandbox/apps/workerpals/src/execute_job.ts

@@ -688,6 +688,9 @@ function inferTerminalFailureClass(result: JobResult, changedPaths: string[]): s
   if (result.ok) return "success";
   const text = `${result.summary ?? ""}\n${result.stderr ?? ""}\n${result.stdout ?? ""}`.toLowerCase();
   const publishableCount = publishableChangedPaths(changedPaths).length;
+  if (text.includes("stalled before first response") || text.includes("startup stall")) {
+    return "codex_startup_stall";
+  }
   if (changedPaths.length > 0 && publishableCount === 0) return "artifact_only_no_publishable_patch";
   if (result.exitCode === 124 || text.includes("timed out") || text.includes("timeout")) return "timeout";
   if (text.includes("validationgate") || text.includes("validation")) return "validation";
@@ -700,6 +703,9 @@ function inferTerminalFailureClass(result: JobResult, changedPaths: string[]): s
 
 function inferTerminalStage(result: JobResult, fallback: string): string {
   const text = `${result.summary ?? ""}\n${result.stderr ?? ""}`.toLowerCase();
+  if (text.includes("stalled before first response") || text.includes("startup stall")) {
+    return "executor_startup";
+  }
   if (text.includes("validationgate") || text.includes("validation")) return "validation";
   if (text.includes("scopegate") || text.includes("scope")) return "scope";
   if (text.includes("criticgate") || text.includes("critic")) return "critic";
@@ -748,7 +754,7 @@ function buildTerminalDiagnostics(args: {
     terminalStage: inferTerminalStage(args.result, args.terminalStage),
     executorBackend: args.executor,
     summary: compactDiagnosticText(args.result.summary, 1_000),
-    watchdogFired: /watchdog|rollout coach/i.test(text),
+    watchdogFired: /watchdog|rollout coach|stalled before first response|startup stall/i.test(text),
     timeoutMs: args.timeoutMs ?? null,
     publishableFileCount: publishable.length,
     artifactOnlyPathCount: artifactOnly.length,

package/runtime/sandbox/apps/workerpals/src/job_runner.ts

@@ -44,6 +44,7 @@ interface JobResult {
   stdout?: string;
   stderr?: string;
   exitCode?: number;
+  cooldownMs?: number;
   commit?: {
     branch: string;
     sha: string;
@@ -115,6 +116,23 @@ echo "password=${token}"
   }
 }
 
+export function buildJobRunnerResult(
+  result: Pick<
+    Awaited<ReturnType<typeof executeJob>>,
+    "ok" | "summary" | "stdout" | "stderr" | "exitCode" | "cooldownMs" | "diagnostics"
+  >,
+): JobResult {
+  return {
+    ok: result.ok,
+    summary: result.summary,
+    stdout: result.stdout,
+    stderr: result.stderr,
+    exitCode: result.exitCode,
+    cooldownMs: result.cooldownMs,
+    diagnostics: result.diagnostics,
+  };
+}
+
 // ─── Main ───────────────────────────────────────────────────────────────────
 
 async function main(): Promise<void> {
@@ -127,8 +145,7 @@ async function main(): Promise<void> {
     process.exit(1);
   }
 
-  const base64Spec =
-    rawSpecArg === "--spec-stdin" ? (await Bun.stdin.text()).trim() : rawSpecArg;
+  const base64Spec = rawSpecArg === "--spec-stdin" ? (await Bun.stdin.text()).trim() : rawSpecArg;
   if (!base64Spec) {
     // eslint-disable-next-line no-console
     console.error("Job spec was empty");
@@ -179,14 +196,7 @@ async function main(): Promise<void> {
       CONFIG,
     );
     // Build result object
-    const jobResult: JobResult = {
-      ok: result.ok,
-      summary: result.summary,
-      stdout: result.stdout,
-      stderr: result.stderr,
-      exitCode: result.exitCode,
-      diagnostics: result.diagnostics,
-    };
+    const jobResult = buildJobRunnerResult(result);
     // Create commit for file-modifying jobs
     if (result.ok && shouldCommit(spec.kind, CONFIG)) {
       log("stdout", `[JobRunner] Job modified files, creating commit...`);
@@ -224,7 +234,8 @@ async function main(): Promise<void> {
         if (commitResult.publishBlocked) {
           jobResult.publishBlocked = commitResult.publishBlocked;
         }
-        jobResult.exitCode = jobResult.exitCode && jobResult.exitCode !== 0 ? jobResult.exitCode : 1;
+        jobResult.exitCode =
+          jobResult.exitCode && jobResult.exitCode !== 0 ? jobResult.exitCode : 1;
         log(
           "stderr",
           commitResult.publishBlocked
@@ -246,8 +257,10 @@ async function main(): Promise<void> {
   }
 }
 
-main().catch((err) => {
-  // eslint-disable-next-line no-console
-  console.error(`[JobRunner] Fatal error: ${err}`);
-  process.exit(1);
-});
+if (import.meta.main) {
+  main().catch((err) => {
+    // eslint-disable-next-line no-console
+    console.error(`[JobRunner] Fatal error: ${err}`);
+    process.exit(1);
+  });
+}

package/runtime/sandbox/apps/workerpals/src/workerpals_main.ts

@@ -67,6 +67,7 @@ type WorkerJobResult = JobResult & {
 
 const DEFAULT_LLM_MODEL = "local-model";
 const CODEX_UNAVAILABLE_WORKER_EXIT_CODE = 86;
+const CODEX_STARTUP_STALL_WORKER_EXIT_CODE = 87;
 const CODEX_UNAVAILABLE_DOCKER_SHUTDOWN_GRACE_MS = 5_000;
 const CODEX_UNAVAILABLE_WORKER_FORCE_EXIT_MS = 4_000;
 const DEFAULT_JOB_PROGRESS_LOG_EVERY_MS = 60_000;
@@ -391,7 +392,9 @@ function inferWorkerJobPhaseFromLogLine(line: string): WorkerJobPhase | null {
   ) {
     return "full validation";
   }
-  if (/creating commit|Publish blocked|publish-blocked|completion ref|enqueueCompletion/i.test(text)) {
+  if (
+    /creating commit|Publish blocked|publish-blocked|completion ref|enqueueCompletion/i.test(text)
+  ) {
     return "publishing";
   }
   if (
@@ -447,11 +450,20 @@ function mergeWorkerDiagnostics(
   };
 }
 
-function inferWorkerTerminalFailureClass(result: JobResult): string {
+function isCodexStartupStallResult(result: JobResult): boolean {
+  const text =
+    `${result.summary ?? ""}\n${result.stderr ?? ""}\n${result.stdout ?? ""}`.toLowerCase();
+  return /stalled before first response|startup stall/.test(text);
+}
+
+export function inferWorkerTerminalFailureClass(result: JobResult): string {
   if (result.ok) return "success";
-  const text = `${result.summary ?? ""}\n${result.stderr ?? ""}\n${result.stdout ?? ""}`.toLowerCase();
-  if (/timed out|timeout|signal 15|terminated|exit 143|exit 137|stalled before first response|startup stall/.test(text)) return "timeout";
-  if (/no publishable|non-publishable|node_modules/.test(text)) return "artifact_only_no_publishable_patch";
+  const text =
+    `${result.summary ?? ""}\n${result.stderr ?? ""}\n${result.stdout ?? ""}`.toLowerCase();
+  if (isCodexStartupStallResult(result)) return "codex_startup_stall";
+  if (/timed out|timeout|signal 15|terminated|exit 143|exit 137/.test(text)) return "timeout";
+  if (/no publishable|non-publishable|node_modules/.test(text))
+    return "artifact_only_no_publishable_patch";
   if (/validationgate|validation/.test(text)) return "validation";
   if (/scopegate|scope/.test(text)) return "scope";
   if (/criticgate|critic/.test(text)) return "critic";
@@ -497,11 +509,12 @@ export function shouldRecycleWorkerForHeartbeatDegradation(options: {
   return options.transportStale;
 }
 
-function shouldRecycleWorkerForCodexUnavailableFailure(
+export function shouldRecycleWorkerForCodexUnavailableFailure(
   summary: string,
   stderr?: string | null,
 ): boolean {
   const text = `${summary}\n${stderr ?? ""}`.toLowerCase();
+  if (/stalled before first response|startup stall/.test(text)) return true;
   return [
     "openai_codex cli is not installed",
     "openai_codex chatgpt auth is not ready",
@@ -512,6 +525,12 @@ function shouldRecycleWorkerForCodexUnavailableFailure(
   ].some((needle) => text.includes(needle));
 }
 
+export function workerRecycleExitCodeForResult(result: WorkerJobResult): number {
+  return isCodexStartupStallResult(result)
+    ? CODEX_STARTUP_STALL_WORKER_EXIT_CODE
+    : CODEX_UNAVAILABLE_WORKER_EXIT_CODE;
+}
+
 async function shutdownDockerExecutorBeforeCodexRecycle(
   dockerExecutor: DockerExecutor | null,
 ): Promise<void> {
@@ -754,14 +773,9 @@ async function createIsolatedWorktree(
     .toLowerCase()
     .replace(/[^a-z0-9]+/g, "")
     .slice(0, 8);
-  const nonce = `${Date.now().toString(36).slice(-6)}-${Math.random()
-    .toString(36)
-    .slice(2, 6)}`;
+  const nonce = `${Date.now().toString(36).slice(-6)}-${Math.random().toString(36).slice(2, 6)}`;
 
-  const worktreePath = resolve(
-    worktreeRoot,
-    `job-${safeJobId || "host"}-${nonce}`,
-  );
+  const worktreePath = resolve(worktreeRoot, `job-${safeJobId || "host"}-${nonce}`);
 
   const addResult = await git(repo, ["worktree", "add", "--detach", worktreePath, baseRef]);
   if (!addResult.ok) {
@@ -1752,6 +1766,7 @@ async function workerLoop(
             const jobAttempt =
               Number.isFinite(jobAttemptRaw) && jobAttemptRaw > 0 ? Math.floor(jobAttemptRaw) : 1;
             const llm = workerLlmConfig(CONFIG);
+            const terminalFailureClass = inferWorkerTerminalFailureClass(result);
             result = {
               ...result,
               diagnostics: mergeWorkerDiagnostics(result.diagnostics, {
@@ -1781,12 +1796,15 @@ async function workerLoop(
                   result.ok ? "completed" : result.publishBlocked ? "publish_blocked" : "failed",
                 ),
                 terminal: {
-                  failureClass: inferWorkerTerminalFailureClass(result),
-                  terminalStage: currentJobPhase ?? (result.ok ? "completed" : "worker"),
+                  failureClass: terminalFailureClass,
+                  terminalStage:
+                    terminalFailureClass === "codex_startup_stall"
+                      ? "executor_startup"
+                      : (currentJobPhase ?? (result.ok ? "completed" : "worker")),
                   executorBackend: resolveExecutor(CONFIG),
                   summary: result.summary,
                   watchdogFired:
-                    /watchdog|rollout coach|timed out|timeout|signal 15|terminated|exit 143|exit 137/i.test(
+                    /watchdog|rollout coach|stalled before first response|startup stall|timed out|timeout|signal 15|terminated|exit 143|exit 137/i.test(
                       `${result.summary}\n${result.stderr ?? ""}\n${result.stdout ?? ""}`,
                     ),
                   metadata: {
@@ -1961,12 +1979,15 @@ async function workerLoop(
             clearInterval(busyHeartbeat);
             if (jobProgressTimer) clearInterval(jobProgressTimer);
             if (recycleWorkerAfterJob) {
+              const recycleExitCode = result
+                ? workerRecycleExitCodeForResult(result)
+                : CODEX_UNAVAILABLE_WORKER_EXIT_CODE;
               runtimeState.shutdownRequested = true;
               const forceExitTimer = setTimeout(() => {
                 console.warn(
                   `[WorkerPals] Forcing worker recycle ${CODEX_UNAVAILABLE_WORKER_FORCE_EXIT_MS}ms after Codex backend failure.`,
                 );
-                process.exit(CODEX_UNAVAILABLE_WORKER_EXIT_CODE);
+                process.exit(recycleExitCode);
               }, CODEX_UNAVAILABLE_WORKER_FORCE_EXIT_MS);
               try {
                 await maybeHeartbeat("offline", null, true);
@@ -1983,7 +2004,7 @@ async function workerLoop(
                 await shutdownDockerExecutorBeforeCodexRecycle(dockerExecutor);
               } finally {
                 clearTimeout(forceExitTimer);
-                process.exit(CODEX_UNAVAILABLE_WORKER_EXIT_CODE);
+                process.exit(recycleExitCode);
               }
             }
             if (job.sessionId && result?.cooldownMs && result.cooldownMs > 0) {