@pushpalsdev/cli 1.1.44 → 1.1.46

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pushpalsdev/cli",
-  "version": "1.1.44",
+  "version": "1.1.46",
   "description": "PushPals terminal CLI for LocalBuddy -> RemoteBuddy orchestration",
   "license": "MIT",
   "repository": {

package/runtime/sandbox/apps/workerpals/src/common/generic_python_executor.ts

@@ -37,6 +37,7 @@ const BACKEND_TIMEOUT_RESULT_GRACE_MS = 30_000;
 const OPENAI_CODEX_MIN_VALIDATION_RESERVE_MS = 240_000;
 const OPENAI_CODEX_MAX_VALIDATION_RESERVE_MS = 720_000;
 const OPENAI_CODEX_MIN_PRIMARY_TURN_BUDGET_MS = 540_000;
+const OPENAI_CODEX_VALIDATION_RESERVE_RATIO = 0.25;
 
 function estimateTokensFromText(text: string): number {
   return Math.max(0, Math.ceil(String(text ?? "").length / 3));
@@ -161,7 +162,10 @@ export function resolveOpenAICodexValidationReserveMs(
       budgetMs,
       Math.max(
         OPENAI_CODEX_MIN_VALIDATION_RESERVE_MS,
-        Math.min(OPENAI_CODEX_MAX_VALIDATION_RESERVE_MS, budgetMs * 0.5),
+        Math.min(
+          OPENAI_CODEX_MAX_VALIDATION_RESERVE_MS,
+          budgetMs * OPENAI_CODEX_VALIDATION_RESERVE_RATIO,
+        ),
       ),
     ),
   );

package/runtime/sandbox/apps/workerpals/src/workerpals_main.ts

@@ -70,6 +70,7 @@ const CODEX_UNAVAILABLE_WORKER_EXIT_CODE = 86;
 const CODEX_STARTUP_STALL_WORKER_EXIT_CODE = 87;
 const CODEX_UNAVAILABLE_DOCKER_SHUTDOWN_GRACE_MS = 5_000;
 const CODEX_UNAVAILABLE_WORKER_FORCE_EXIT_MS = 4_000;
+const CODEX_STARTUP_STALL_DIRECT_RETRY_DEFER_MS = 5_000;
 const DEFAULT_JOB_PROGRESS_LOG_EVERY_MS = 60_000;
 const CONFIG = loadPushPalsConfig();
 const LOG = new Logger("WorkerPals");
@@ -525,6 +526,14 @@ export function shouldRecycleWorkerForCodexUnavailableFailure(
   ].some((needle) => text.includes(needle));
 }
 
+export function shouldDeferDockerCodexStartupStallForDirectRetry(options: {
+  dockerEnabled: boolean;
+  result: JobResult;
+}): boolean {
+  if (!options.dockerEnabled) return false;
+  return isCodexStartupStallResult(options.result);
+}
+
 export function workerRecycleExitCodeForResult(result: WorkerJobResult): number {
   return isCodexStartupStallResult(result)
     ? CODEX_STARTUP_STALL_WORKER_EXIT_CODE
@@ -1282,12 +1291,20 @@ async function deferClaimedJobForMaintenance(
   headers: Record<string, string>,
   jobId: string,
   deferMs: number,
+  options: { targetWorkerId?: string | null; reason?: string } = {},
 ): Promise<{ ok: boolean; availableAt?: string; message?: string }> {
   try {
-    const response = await postJsonWithTimeout(`${opts.server}/jobs/${jobId}/defer`, headers, {
+    const body: Record<string, unknown> = {
       workerId: opts.workerId,
       deferMs,
-    });
+    };
+    if (Object.prototype.hasOwnProperty.call(options, "targetWorkerId")) {
+      body.targetWorkerId = options.targetWorkerId;
+    }
+    if (options.reason) {
+      body.reason = options.reason;
+    }
+    const response = await postJsonWithTimeout(`${opts.server}/jobs/${jobId}/defer`, headers, body);
     const payload = (await response.json().catch(() => ({}))) as {
       ok?: boolean;
       availableAt?: string;
@@ -1818,6 +1835,7 @@ async function workerLoop(
             };
 
             let statusPersistedToServer = false;
+            let deferredForDirectRetry = false;
             if (result.publishBlocked) {
               await reportToolRunForUnsuccessfulJob({
                 opts,
@@ -1872,40 +1890,86 @@ async function workerLoop(
                 `[WorkerPals] Job ${job.id} completed in ${formatDurationMs(jobDurationMs)}: ${result.summary}`,
               );
             } else {
-              await reportToolRunForUnsuccessfulJob({
-                opts,
-                headers,
-                job,
-                result,
-                durationMs: jobDurationMs,
-                phase: job.kind,
-              });
-              const response = await postJsonWithTimeout(
-                `${opts.server}/jobs/${job.id}/fail`,
-                headers,
-                {
-                  message: result.summary,
-                  detail: redactSensitiveText(result.stderr ?? ""),
+              let unsuccessfulToolRunReported = false;
+              const reportUnsuccessfulToolRun = async (phase: string) => {
+                if (unsuccessfulToolRunReported) return;
+                unsuccessfulToolRunReported = true;
+                await reportToolRunForUnsuccessfulJob({
+                  opts,
+                  headers,
+                  job,
+                  result,
                   durationMs: jobDurationMs,
-                  diagnostics: result.diagnostics,
-                },
-              );
-              statusPersistedToServer = response.ok;
-              console.log(
-                `[WorkerPals] Job ${job.id} failed in ${formatDurationMs(jobDurationMs)}: ${result.summary}`,
-              );
-              recycleWorkerAfterJob = shouldRecycleWorkerForCodexUnavailableFailure(
-                result.summary,
-                result.stderr,
-              );
-              if (recycleWorkerAfterJob) {
-                console.error(
-                  `[WorkerPals] Codex backend unavailable for job ${job.id}; terminating this worker for replacement.`,
+                  phase,
+                });
+              };
+              const failCurrentJob = async () => {
+                await reportUnsuccessfulToolRun(job.kind);
+                const response = await postJsonWithTimeout(
+                  `${opts.server}/jobs/${job.id}/fail`,
+                  headers,
+                  {
+                    message: result.summary,
+                    detail: redactSensitiveText(result.stderr ?? ""),
+                    durationMs: jobDurationMs,
+                    diagnostics: result.diagnostics,
+                  },
+                );
+                statusPersistedToServer = response.ok;
+                console.log(
+                  `[WorkerPals] Job ${job.id} failed in ${formatDurationMs(jobDurationMs)}: ${result.summary}`,
                 );
+                recycleWorkerAfterJob = shouldRecycleWorkerForCodexUnavailableFailure(
+                  result.summary,
+                  result.stderr,
+                );
+                if (recycleWorkerAfterJob) {
+                  console.error(
+                    `[WorkerPals] Codex backend unavailable for job ${job.id}; terminating this worker for replacement.`,
+                  );
+                }
+              };
+
+              if (
+                shouldDeferDockerCodexStartupStallForDirectRetry({
+                  dockerEnabled: Boolean(dockerExecutor),
+                  result,
+                })
+              ) {
+                await reportUnsuccessfulToolRun("worker:docker-codex-startup-stall-defer");
+                const deferred = await deferClaimedJobForMaintenance(
+                  opts,
+                  headers,
+                  job.id,
+                  CODEX_STARTUP_STALL_DIRECT_RETRY_DEFER_MS,
+                  {
+                    targetWorkerId: null,
+                    reason: "codex_startup_stall_direct_retry",
+                  },
+                );
+                if (deferred.ok) {
+                  deferredForDirectRetry = true;
+                  statusPersistedToServer = true;
+                  recycleWorkerAfterJob = true;
+                  console.warn(
+                    `[WorkerPals] Deferred job ${job.id} after Docker Codex startup stall until ${
+                      deferred.availableAt ?? "a direct WorkerPal retry"
+                    }; recycling this worker so RemoteBuddy can spawn a direct isolated-worktree WorkerPal.`,
+                  );
+                } else {
+                  console.warn(
+                    `[WorkerPals] Failed to defer Docker Codex startup-stall job ${job.id}; marking failed: ${
+                      deferred.message || "unknown error"
+                    }`,
+                  );
+                  await failCurrentJob();
+                }
+              } else {
+                await failCurrentJob();
               }
             }
 
-            if (job.sessionId) {
+            if (job.sessionId && !deferredForDirectRetry) {
               const jobOrigin = taskExecuteOrigin(parsedParams);
               const responseMode = String(parsedParams.responseMode ?? "")
                 .trim()