agentxchain 2.155.72 → 2.156.0

package/src/commands/step.js

@@ -22,7 +22,7 @@
  */
 
 import chalk from 'chalk';
-import { existsSync, readFileSync } from 'fs';
+import { existsSync, readFileSync, unlinkSync } from 'fs';
 import { join } from 'path';
 import { loadProjectContext, loadProjectState } from '../lib/config.js';
 import {
@@ -76,6 +76,8 @@ import { evaluateApprovalSlaReminders } from '../lib/notification-runner.js';
 import { consumeNextApprovedIntent } from '../lib/intake.js';
 import { failTurnStartup, reconcileStaleTurns } from '../lib/stale-turn-watchdog.js';
 import { isKnownTurnRunningProofStream } from '../lib/dispatch-streams.js';
+import { getDispatchProgressRelativePath } from '../lib/dispatch-progress.js';
+import { checkpointAcceptedTurn } from '../lib/turn-checkpoint.js';
 
 export async function stepCommand(opts) {
   const context = loadProjectContext();
@@ -192,6 +194,7 @@ export async function stepCommand(opts) {
         process.exit(1);
       }
 
+      guardResumeWorkerLiveness(root, targetTurn);
       skipAssignment = true;
       console.log(chalk.yellow(`Resuming active turn: ${targetTurn.turn_id}`));
     } else if (activeCount >= maxConcurrent) {
@@ -272,6 +275,7 @@ export async function stepCommand(opts) {
         process.exit(1);
       }
 
+      guardResumeWorkerLiveness(root, targetTurn);
       console.log(chalk.yellow(`Re-dispatching blocked turn: ${targetTurn.turn_id}`));
       const reactivated = reactivateGovernedRun(root, state, { via: 'step --resume', notificationConfig: config });
       if (!reactivated.ok) {
@@ -999,6 +1003,21 @@ export async function stepCommand(opts) {
     }
 
     printAcceptSummary(acceptResult, config);
+
+    // Auto-checkpoint accepted turn so workspace is clean for next assignment
+    if (!opts.noCheckpoint && existsSync(join(root, '.git'))) {
+      const checkpoint = checkpointAcceptedTurn(root, { turnId: turn.turn_id });
+      if (!checkpoint.ok) {
+        console.log(`  ${chalk.yellow('Checkpoint:')} accepted but checkpoint failed`);
+        console.log(`  ${chalk.dim('Error:')}  ${checkpoint.error}`);
+        console.log(`  ${chalk.dim('Retry:')}  agentxchain checkpoint-turn --turn ${turn.turn_id}`);
+        console.log('');
+        process.exit(1);
+      }
+      if (!checkpoint.skipped) {
+        console.log(`  ${chalk.dim('Checkpoint:')} ${checkpoint.checkpoint_sha}`);
+      }
+    }
   } else {
     // Reject and potentially retry
     console.log(chalk.yellow('Validation failed:'));
@@ -1039,6 +1058,49 @@ export async function stepCommand(opts) {
   }
 }
 
+function guardResumeWorkerLiveness(root, turn) {
+  if (!turn || turn.worker_pid == null) {
+    return;
+  }
+
+  if (isWorkerAlive(turn.worker_pid)) {
+    console.log(chalk.red(`Worker process (PID ${turn.worker_pid}) is still alive.`));
+    console.log(chalk.dim('The previous dispatch appears to still be running.'));
+    console.log(chalk.dim('Wait for it to complete, or kill it first, then retry.'));
+    process.exit(1);
+  }
+
+  console.log(chalk.yellow(`Detected crashed worker (PID ${turn.worker_pid}). Re-dispatching turn ${turn.turn_id}...`));
+  cleanupStaleDispatchProgress(root, turn.turn_id);
+}
+
+function isWorkerAlive(pid) {
+  const numericPid = Number(pid);
+  if (!Number.isInteger(numericPid) || numericPid <= 0) {
+    return false;
+  }
+
+  try {
+    process.kill(numericPid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function cleanupStaleDispatchProgress(root, turnId) {
+  const progressPath = join(root, getDispatchProgressRelativePath(turnId));
+  if (!existsSync(progressPath)) {
+    return;
+  }
+
+  try {
+    unlinkSync(progressPath);
+  } catch {
+    // Best-effort cleanup: resume can still proceed and rewrite fresh progress.
+  }
+}
+
 function printGhostTurnRecovery(ghostTurns) {
   console.log(chalk.red.bold('Ghost turn detected — subprocess never started.'));
   console.log('');

package/src/commands/verify.js

@@ -244,6 +244,7 @@ export async function verifyTurnCommand(turnId, opts = {}) {
       root,
       verification: turnResult.verification,
       timeoutMs,
+      allowCommandExecution: Boolean(opts.execute),
     }),
   };
 

package/src/lib/adapters/local-cli-adapter.js

@@ -34,7 +34,14 @@ import {
   getClaudeSubprocessAuthIssue,
   hasClaudeAuthenticationFailureText,
   hasClaudeNodeIncompatibilityText,
+  hasCodexAuthenticationFailureText,
+  hasRateLimitOutput,
   isClaudeLocalCliRuntime,
+  isCodexLocalCliRuntime,
+  isCursorLocalCliRuntime,
+  isWindsurfLocalCliRuntime,
+  isOpenCodeLocalCliRuntime,
+  parseRateLimitResetMs,
   resolveClaudeCompatibleNodeBinary,
 } from '../claude-local-auth.js';
 
@@ -49,6 +56,12 @@ const DIAGNOSTIC_ENV_KEYS = [
 const DIAGNOSTIC_STDERR_EXCERPT_LIMIT = 800;
 const DEFAULT_STARTUP_WATCHDOG_MS = 180_000;
 const DEFAULT_STARTUP_WATCHDOG_SIGKILL_GRACE_MS = 10_000;
+const DEFAULT_STARTUP_HEARTBEAT_MS = 30_000;
+// RB-6: liveness heartbeat interval. Unlike the startup heartbeat (which stops
+// at first output), this keeps the dispatch-progress `last_activity_at` fresh
+// for the whole lifetime of an alive subprocess, so a long output-silent turn
+// (e.g. one running a slow test suite) is not misclassified as a stalled turn.
+const DEFAULT_LIVENESS_HEARTBEAT_MS = 60_000;
 
 /**
  * Launch a local CLI subprocess for a governed turn.
@@ -97,6 +110,8 @@ export async function dispatchLocalCli(root, state, config, options = {}) {
   }
   const startupWatchdogMs = startupWatchdogOverrideMs ?? resolveStartupWatchdogMs(config, runtime);
   const startupWatchdogKillGraceMs = resolveStartupWatchdogKillGraceMs(options.startupWatchdogKillGraceMs);
+  const startupHeartbeatMs = resolveStartupHeartbeatMs(config, runtime, options.startupHeartbeatMs);
+  const livenessHeartbeatMs = resolveLivenessHeartbeatMs(config, runtime, options.livenessHeartbeatMs);
 
   // Read the dispatch bundle prompt
   const promptPath = join(root, getDispatchPromptPath(turn.turn_id));
@@ -116,8 +131,28 @@ export async function dispatchLocalCli(root, state, config, options = {}) {
     return { ok: false, error: `Cannot resolve CLI command for runtime "${runtimeId}". Expected "command" field in runtime config.` };
   }
 
-  // Compute timeout from deadline or default (20 minutes)
-  const timeoutMs = turn.deadline_at
+  const compatibility = validateLocalCliCommandCompatibility({ command, args, runtimeId });
+  if (!compatibility.ok) {
+    const logs = [];
+    appendDiagnostic(logs, 'command_compatibility_failed', compatibility.diagnostic);
+    return {
+      ok: false,
+      blocked: true,
+      classified: {
+        error_class: compatibility.error_class,
+        recovery: compatibility.recovery,
+      },
+      error: compatibility.error,
+      logs,
+    };
+  }
+
+  // Compute timeout from explicit dispatch deadline, turn deadline, or default (20 minutes).
+  const timeoutMs = options.dispatchTimeoutMs != null
+    ? options.dispatchTimeoutMs
+    : options.dispatchDeadlineAt
+      ? Math.max(0, new Date(options.dispatchDeadlineAt).getTime() - Date.now())
+      : turn.deadline_at
     ? Math.max(0, new Date(turn.deadline_at).getTime() - Date.now())
     : 1200000;
 
@@ -201,6 +236,8 @@ export async function dispatchLocalCli(root, state, config, options = {}) {
     let firstOutputLatencyMs = null;
     let startupWatchdog = null;
     let startupSigkillHandle = null;
+    let startupHeartbeat = null;
+    let livenessHeartbeat = null;
     let startupTimedOut = false;
     let startupFailureType = null;
     let stdoutBytes = 0;
@@ -210,6 +247,7 @@ export async function dispatchLocalCli(root, state, config, options = {}) {
     const settle = (result) => {
       if (settled) return;
       settled = true;
+      clearLivenessHeartbeat();
       resolve(result);
     };
 
@@ -224,6 +262,85 @@ export async function dispatchLocalCli(root, state, config, options = {}) {
       }
     };
 
+    const clearStartupHeartbeat = () => {
+      if (startupHeartbeat) {
+        clearInterval(startupHeartbeat);
+        startupHeartbeat = null;
+      }
+    };
+
+    const clearLivenessHeartbeat = () => {
+      if (livenessHeartbeat) {
+        clearInterval(livenessHeartbeat);
+        livenessHeartbeat = null;
+      }
+    };
+
+    // RB-6: keep `last_activity_at` fresh for the full lifetime of an alive
+    // subprocess (NOT cleared at first output, unlike the startup heartbeat), so
+    // a long output-silent turn — e.g. one running a slow test suite — is not
+    // misclassified as stalled by the stale-turn watchdog. Genuine hangs are
+    // still bounded by the per-turn hard timeout.
+    const armLivenessHeartbeat = () => {
+      if (livenessHeartbeat || !(livenessHeartbeatMs > 0 && Number.isFinite(livenessHeartbeatMs))) {
+        return;
+      }
+      livenessHeartbeat = setInterval(() => {
+        if (settled) {
+          clearLivenessHeartbeat();
+          return;
+        }
+        const payload = {
+          liveness_heartbeat_ms: livenessHeartbeatMs,
+          pid: child.pid ?? null,
+          spawn_confirmed_at: spawnConfirmedAt,
+          elapsed_since_spawn_ms: spawnConfirmedAtMs == null ? null : Math.max(0, Date.now() - spawnConfirmedAtMs),
+          first_output_at: firstOutputAt,
+          stdout_bytes: stdoutBytes,
+          stderr_bytes: stderrBytes,
+        };
+        appendDiagnostic(logs, 'liveness_heartbeat', payload);
+        if (options.onLivenessHeartbeat) {
+          try {
+            options.onLivenessHeartbeat(payload);
+          } catch {}
+        }
+      }, livenessHeartbeatMs);
+      if (typeof livenessHeartbeat.unref === 'function') {
+        livenessHeartbeat.unref();
+      }
+    };
+
+    const armStartupHeartbeat = () => {
+      if (startupHeartbeat || !(startupHeartbeatMs > 0 && Number.isFinite(startupHeartbeatMs))) {
+        return;
+      }
+      startupHeartbeat = setInterval(() => {
+        if (firstOutputAt || settled) {
+          clearStartupHeartbeat();
+          return;
+        }
+        const payload = {
+          startup_heartbeat_ms: startupHeartbeatMs,
+          startup_watchdog_ms: startupWatchdogMs,
+          pid: child.pid ?? null,
+          spawn_confirmed_at: spawnConfirmedAt,
+          elapsed_since_spawn_ms: spawnConfirmedAtMs == null ? null : Math.max(0, Date.now() - spawnConfirmedAtMs),
+          stdout_bytes: stdoutBytes,
+          stderr_bytes: stderrBytes,
+        };
+        appendDiagnostic(logs, 'startup_heartbeat', payload);
+        if (options.onStartupHeartbeat) {
+          try {
+            options.onStartupHeartbeat(payload);
+          } catch {}
+        }
+      }, startupHeartbeatMs);
+      if (typeof startupHeartbeat.unref === 'function') {
+        startupHeartbeat.unref();
+      }
+    };
+
     const armStartupWatchdog = () => {
       if (startupWatchdog || !(startupWatchdogMs > 0 && Number.isFinite(startupWatchdogMs))) {
         return;
@@ -269,6 +386,7 @@ export async function dispatchLocalCli(root, state, config, options = {}) {
       firstOutputStream = stream;
       firstOutputLatencyMs = spawnConfirmedAtMs == null ? null : Math.max(0, Date.now() - spawnConfirmedAtMs);
       clearStartupWatchdog();
+      clearStartupHeartbeat();
       appendDiagnostic(logs, 'first_output', {
         at: firstOutputAt,
         stream,
@@ -296,6 +414,8 @@ export async function dispatchLocalCli(root, state, config, options = {}) {
         } catch {}
       }
       armStartupWatchdog();
+      armStartupHeartbeat();
+      armLivenessHeartbeat();
     });
 
     // Collect stdout/stderr
@@ -369,6 +489,7 @@ export async function dispatchLocalCli(root, state, config, options = {}) {
     const onAbort = () => {
       logs.push('[adapter] Abort signal received. Sending SIGTERM.');
       clearStartupWatchdog();
+      clearStartupHeartbeat();
       clearTimeout(timeoutHandle);
       clearTimeout(sigkillHandle);
       clearTimeout(abortSigkillHandle);
@@ -389,6 +510,7 @@ export async function dispatchLocalCli(root, state, config, options = {}) {
     // Process exit
     child.on('close', (exitCode, killSignal) => {
       clearStartupWatchdog();
+      clearStartupHeartbeat();
       clearTimeout(timeoutHandle);
       clearTimeout(sigkillHandle);
       clearTimeout(abortSigkillHandle);
@@ -453,6 +575,22 @@ export async function dispatchLocalCli(root, state, config, options = {}) {
           error: `Claude local_cli authentication failed. ${recovery}`,
           logs,
         });
+      } else if (isCodexLocalCliRuntime(runtime) && hasCodexAuthFailureOutput(logs)) {
+        const recovery = 'Refresh OpenAI credentials before resuming: export a valid OPENAI_API_KEY, then run agentxchain step --resume.';
+        settle({
+          ok: false,
+          blocked: true,
+          exitCode,
+          timedOut: false,
+          aborted: false,
+          firstOutputAt,
+          classified: {
+            error_class: 'codex_auth_failed',
+            recovery,
+          },
+          error: `Codex local_cli authentication failed. ${recovery}`,
+          logs,
+        });
       } else if (isClaudeLocalCliRuntime(runtime) && hasClaudeNodeRuntimeIncompatibilityOutput(logs)) {
         const recovery = 'Run AgentXchain with Node.js 20.5+ available to the Claude local_cli runtime, then resume continuous mode.';
         settle({
@@ -469,6 +607,26 @@ export async function dispatchLocalCli(root, state, config, options = {}) {
           error: `Claude local_cli runtime is using an incompatible Node.js version. ${recovery}`,
           logs,
         });
+      } else if (hasRateLimitOutput(logs)) {
+        const resetMs = parseRateLimitResetMs(logs.join('\n'));
+        const resetDetail = resetMs ? ` Resets in ${Math.ceil(resetMs / 1000)}s.` : '';
+        const recovery = `Provider rate-limited.${resetDetail} Will retry with backoff.`;
+        settle({
+          ok: false,
+          blocked: true,
+          exitCode,
+          timedOut: false,
+          aborted: false,
+          firstOutputAt,
+          classified: {
+            error_class: 'rate_limited',
+            retryable: true,
+            recovery,
+            reset_ms: resetMs,
+          },
+          error: `Subprocess rate-limited by provider. ${recovery}`,
+          logs,
+        });
       } else if (startupTimedOut) {
         settle({
           ok: false,
@@ -524,6 +682,7 @@ export async function dispatchLocalCli(root, state, config, options = {}) {
 
     child.on('error', (err) => {
       clearStartupWatchdog();
+      clearStartupHeartbeat();
       clearTimeout(timeoutHandle);
       clearTimeout(sigkillHandle);
       clearTimeout(abortSigkillHandle);
@@ -653,6 +812,32 @@ function resolveStartupWatchdogMs(config, runtime) {
   return DEFAULT_STARTUP_WATCHDOG_MS;
 }
 
+function resolveStartupHeartbeatMs(config, runtime, override) {
+  if (Number.isInteger(override) && override > 0) {
+    return override;
+  }
+  if (runtime?.type === 'local_cli' && Number.isInteger(runtime?.startup_heartbeat_ms) && runtime.startup_heartbeat_ms > 0) {
+    return runtime.startup_heartbeat_ms;
+  }
+  if (Number.isInteger(config?.run_loop?.startup_heartbeat_ms) && config.run_loop.startup_heartbeat_ms > 0) {
+    return config.run_loop.startup_heartbeat_ms;
+  }
+  return DEFAULT_STARTUP_HEARTBEAT_MS;
+}
+
+function resolveLivenessHeartbeatMs(config, runtime, override) {
+  if (Number.isInteger(override) && override > 0) {
+    return override;
+  }
+  if (runtime?.type === 'local_cli' && Number.isInteger(runtime?.liveness_heartbeat_ms) && runtime.liveness_heartbeat_ms > 0) {
+    return runtime.liveness_heartbeat_ms;
+  }
+  if (Number.isInteger(config?.run_loop?.liveness_heartbeat_ms) && config.run_loop.liveness_heartbeat_ms > 0) {
+    return config.run_loop.liveness_heartbeat_ms;
+  }
+  return DEFAULT_LIVENESS_HEARTBEAT_MS;
+}
+
 function resolveStartupWatchdogKillGraceMs(value) {
   if (Number.isInteger(value) && value >= 0) {
     return value;
@@ -660,6 +845,137 @@ function resolveStartupWatchdogKillGraceMs(value) {
   return DEFAULT_STARTUP_WATCHDOG_SIGKILL_GRACE_MS;
 }
 
+function validateLocalCliCommandCompatibility({ command, args = [], runtimeId = null }) {
+  const tokens = [command, ...args].filter((token) => typeof token === 'string');
+  const binaryName = command ? command.split('/').filter(Boolean).pop() || command : '';
+  const runtimeShape = { command: tokens };
+  const outputFormatIndex = tokens.findIndex((token) => token === '--output-format');
+  const outputFormatValue = outputFormatIndex >= 0 ? tokens[outputFormatIndex + 1] : null;
+  const usesStreamJson = tokens.includes('--output-format=stream-json')
+    || outputFormatValue === 'stream-json';
+  const usesPrint = tokens.includes('--print') || tokens.includes('-p');
+  const hasVerbose = tokens.includes('--verbose');
+  const usesCodex = isCodexLocalCliRuntime(runtimeShape);
+  const usesCodexExec = tokens.includes('exec');
+  const hasCodexJson = tokens.includes('--json');
+
+  if (binaryName === 'claude' && usesPrint && usesStreamJson && !hasVerbose) {
+    const runtimeLabel = runtimeId ? `Runtime "${runtimeId}"` : 'Claude local_cli runtime';
+    const recovery = `${runtimeLabel} uses "claude --print --output-format stream-json" without "--verbose". Add "--verbose" to the command array before dispatching again.`;
+    return {
+      ok: false,
+      error_class: 'local_cli_command_incompatible',
+      recovery,
+      error: recovery,
+      diagnostic: {
+        runtime_id: runtimeId,
+        binary: binaryName,
+        rule: 'claude_print_stream_json_requires_verbose',
+        has_print: usesPrint,
+        has_stream_json: usesStreamJson,
+        has_verbose: hasVerbose,
+        recovery,
+      },
+    };
+  }
+
+  if (usesCodex && !usesCodexExec) {
+    const runtimeLabel = runtimeId ? `Runtime "${runtimeId}"` : 'Codex local_cli runtime';
+    const recovery = `${runtimeLabel} uses "codex" without the "exec" subcommand. Governed local runs require "codex exec" for non-interactive execution.`;
+    return {
+      ok: false,
+      error_class: 'local_cli_command_incompatible',
+      recovery,
+      error: recovery,
+      diagnostic: {
+        runtime_id: runtimeId,
+        binary: binaryName,
+        rule: 'codex_requires_exec',
+        has_exec: usesCodexExec,
+        recovery,
+      },
+    };
+  }
+
+  if (usesCodex && usesCodexExec && !hasCodexJson) {
+    const runtimeLabel = runtimeId ? `Runtime "${runtimeId}"` : 'Codex local_cli runtime';
+    const recovery = `${runtimeLabel} uses "codex exec" without "--json". Add "--json" so subprocess output is machine-readable diagnostics while turn results remain staged on disk.`;
+    return {
+      ok: false,
+      error_class: 'local_cli_command_incompatible',
+      recovery,
+      error: recovery,
+      diagnostic: {
+        runtime_id: runtimeId,
+        binary: binaryName,
+        rule: 'codex_exec_requires_json',
+        has_exec: usesCodexExec,
+        has_json: hasCodexJson,
+        recovery,
+      },
+    };
+  }
+
+  const usesCursor = isCursorLocalCliRuntime(runtimeShape);
+  if (usesCursor && !tokens.includes('--background-agent') && !tokens.includes('agent')) {
+    const runtimeLabel = runtimeId ? `Runtime "${runtimeId}"` : 'Cursor local_cli runtime';
+    const recovery = `${runtimeLabel} uses "cursor" without a background agent flag. Governed local runs require Cursor's agent or background-agent mode for non-interactive execution.`;
+    return {
+      ok: false,
+      error_class: 'local_cli_command_incompatible',
+      recovery,
+      error: recovery,
+      diagnostic: {
+        runtime_id: runtimeId,
+        binary: binaryName,
+        rule: 'cursor_requires_agent_mode',
+        has_agent_flag: false,
+        recovery,
+      },
+    };
+  }
+
+  const usesWindsurf = isWindsurfLocalCliRuntime(runtimeShape);
+  if (usesWindsurf && !tokens.includes('--agent')) {
+    const runtimeLabel = runtimeId ? `Runtime "${runtimeId}"` : 'Windsurf local_cli runtime';
+    const recovery = `${runtimeLabel} uses "windsurf" without an agent flag. Governed local runs require Windsurf's --agent mode for non-interactive execution.`;
+    return {
+      ok: false,
+      error_class: 'local_cli_command_incompatible',
+      recovery,
+      error: recovery,
+      diagnostic: {
+        runtime_id: runtimeId,
+        binary: binaryName,
+        rule: 'windsurf_requires_agent_mode',
+        has_agent_flag: false,
+        recovery,
+      },
+    };
+  }
+
+  const usesOpenCode = isOpenCodeLocalCliRuntime(runtimeShape);
+  if (usesOpenCode && !tokens.includes('--non-interactive')) {
+    const runtimeLabel = runtimeId ? `Runtime "${runtimeId}"` : 'OpenCode local_cli runtime';
+    const recovery = `${runtimeLabel} uses "opencode" without --non-interactive. Governed local runs require OpenCode's non-interactive mode for unattended execution.`;
+    return {
+      ok: false,
+      error_class: 'local_cli_command_incompatible',
+      recovery,
+      error: recovery,
+      diagnostic: {
+        runtime_id: runtimeId,
+        binary: binaryName,
+        rule: 'opencode_requires_non_interactive',
+        has_non_interactive_flag: false,
+        recovery,
+      },
+    };
+  }
+
+  return { ok: true };
+}
+
 /**
  * Check if the staged result file exists and has meaningful content.
  * Delegates to the shared `hasMeaningfulStagedResult` helper so watchdog,
@@ -687,6 +1003,11 @@ function hasClaudeAuthFailureOutput(logs) {
   return logs.some((line) => hasClaudeAuthenticationFailureText(line));
 }
 
+function hasCodexAuthFailureOutput(logs) {
+  if (!Array.isArray(logs)) return false;
+  return logs.some((line) => hasCodexAuthenticationFailureText(line));
+}
+
 function hasClaudeNodeRuntimeIncompatibilityOutput(logs) {
   if (!Array.isArray(logs)) return false;
   return hasClaudeNodeIncompatibilityText(logs.join('\n'));
@@ -777,3 +1098,6 @@ function appendDiagnosticExcerpt(existing, chunk, limit) {
 export { resolveCommand };
 export { resolvePromptTransport };
 export { resolveStartupWatchdogMs };
+export { resolveStartupHeartbeatMs };
+export { resolveLivenessHeartbeatMs };
+export { validateLocalCliCommandCompatibility };