agentxchain 2.155.73 → 2.157.0

package/src/lib/adapters/local-cli-adapter.js

@@ -35,10 +35,19 @@ import {
   hasClaudeAuthenticationFailureText,
   hasClaudeNodeIncompatibilityText,
   hasCodexAuthenticationFailureText,
+  hasRateLimitOutput,
   isClaudeLocalCliRuntime,
   isCodexLocalCliRuntime,
+  isCursorLocalCliRuntime,
+  isWindsurfLocalCliRuntime,
+  isOpenCodeLocalCliRuntime,
+  parseRateLimitResetMs,
   resolveClaudeCompatibleNodeBinary,
 } from '../claude-local-auth.js';
+import {
+  createStreamJsonCostParser,
+  buildCostFromStreamJson,
+} from '../stream-json-cost-parser.js';
 
 const DIAGNOSTIC_ENV_KEYS = [
   'PATH',
@@ -52,6 +61,11 @@ const DIAGNOSTIC_STDERR_EXCERPT_LIMIT = 800;
 const DEFAULT_STARTUP_WATCHDOG_MS = 180_000;
 const DEFAULT_STARTUP_WATCHDOG_SIGKILL_GRACE_MS = 10_000;
 const DEFAULT_STARTUP_HEARTBEAT_MS = 30_000;
+// RB-6: liveness heartbeat interval. Unlike the startup heartbeat (which stops
+// at first output), this keeps the dispatch-progress `last_activity_at` fresh
+// for the whole lifetime of an alive subprocess, so a long output-silent turn
+// (e.g. one running a slow test suite) is not misclassified as a stalled turn.
+const DEFAULT_LIVENESS_HEARTBEAT_MS = 60_000;
 
 /**
  * Launch a local CLI subprocess for a governed turn.
@@ -101,6 +115,7 @@ export async function dispatchLocalCli(root, state, config, options = {}) {
   const startupWatchdogMs = startupWatchdogOverrideMs ?? resolveStartupWatchdogMs(config, runtime);
   const startupWatchdogKillGraceMs = resolveStartupWatchdogKillGraceMs(options.startupWatchdogKillGraceMs);
   const startupHeartbeatMs = resolveStartupHeartbeatMs(config, runtime, options.startupHeartbeatMs);
+  const livenessHeartbeatMs = resolveLivenessHeartbeatMs(config, runtime, options.livenessHeartbeatMs);
 
   // Read the dispatch bundle prompt
   const promptPath = join(root, getDispatchPromptPath(turn.turn_id));
@@ -136,6 +151,16 @@ export async function dispatchLocalCli(root, state, config, options = {}) {
     };
   }
 
+  // Create stream-json cost parser for Claude runtimes with stream-json output.
+  // Only Claude runtimes emit parseable NDJSON with result events; other runtimes
+  // (Codex, Cursor, Windsurf, OpenCode) have different stdout formats.
+  const tokens = [command, ...args].filter((t) => typeof t === 'string');
+  const usesStreamJson = tokens.includes('--output-format=stream-json')
+    || (tokens.includes('--output-format') && tokens[tokens.indexOf('--output-format') + 1] === 'stream-json');
+  const costParser = isClaudeLocalCliRuntime(runtime) && usesStreamJson
+    ? createStreamJsonCostParser()
+    : null;
+
   // Compute timeout from explicit dispatch deadline, turn deadline, or default (20 minutes).
   const timeoutMs = options.dispatchTimeoutMs != null
     ? options.dispatchTimeoutMs
@@ -226,6 +251,7 @@ export async function dispatchLocalCli(root, state, config, options = {}) {
     let startupWatchdog = null;
     let startupSigkillHandle = null;
     let startupHeartbeat = null;
+    let livenessHeartbeat = null;
     let startupTimedOut = false;
     let startupFailureType = null;
     let stdoutBytes = 0;
@@ -235,6 +261,7 @@ export async function dispatchLocalCli(root, state, config, options = {}) {
     const settle = (result) => {
       if (settled) return;
       settled = true;
+      clearLivenessHeartbeat();
       resolve(result);
     };
 
@@ -256,6 +283,48 @@ export async function dispatchLocalCli(root, state, config, options = {}) {
       }
     };
 
+    const clearLivenessHeartbeat = () => {
+      if (livenessHeartbeat) {
+        clearInterval(livenessHeartbeat);
+        livenessHeartbeat = null;
+      }
+    };
+
+    // RB-6: keep `last_activity_at` fresh for the full lifetime of an alive
+    // subprocess (NOT cleared at first output, unlike the startup heartbeat), so
+    // a long output-silent turn — e.g. one running a slow test suite — is not
+    // misclassified as stalled by the stale-turn watchdog. Genuine hangs are
+    // still bounded by the per-turn hard timeout.
+    const armLivenessHeartbeat = () => {
+      if (livenessHeartbeat || !(livenessHeartbeatMs > 0 && Number.isFinite(livenessHeartbeatMs))) {
+        return;
+      }
+      livenessHeartbeat = setInterval(() => {
+        if (settled) {
+          clearLivenessHeartbeat();
+          return;
+        }
+        const payload = {
+          liveness_heartbeat_ms: livenessHeartbeatMs,
+          pid: child.pid ?? null,
+          spawn_confirmed_at: spawnConfirmedAt,
+          elapsed_since_spawn_ms: spawnConfirmedAtMs == null ? null : Math.max(0, Date.now() - spawnConfirmedAtMs),
+          first_output_at: firstOutputAt,
+          stdout_bytes: stdoutBytes,
+          stderr_bytes: stderrBytes,
+        };
+        appendDiagnostic(logs, 'liveness_heartbeat', payload);
+        if (options.onLivenessHeartbeat) {
+          try {
+            options.onLivenessHeartbeat(payload);
+          } catch {}
+        }
+      }, livenessHeartbeatMs);
+      if (typeof livenessHeartbeat.unref === 'function') {
+        livenessHeartbeat.unref();
+      }
+    };
+
     const armStartupHeartbeat = () => {
       if (startupHeartbeat || !(startupHeartbeatMs > 0 && Number.isFinite(startupHeartbeatMs))) {
         return;
@@ -360,6 +429,7 @@ export async function dispatchLocalCli(root, state, config, options = {}) {
       }
       armStartupWatchdog();
       armStartupHeartbeat();
+      armLivenessHeartbeat();
     });
 
     // Collect stdout/stderr
@@ -369,6 +439,7 @@ export async function dispatchLocalCli(root, state, config, options = {}) {
         stdoutBytes += Buffer.byteLength(text);
         recordFirstOutput('stdout');
         logs.push(text);
+        if (costParser) costParser.push(text);
         if (onStdout) onStdout(text);
       });
     }
@@ -502,6 +573,28 @@ export async function dispatchLocalCli(root, state, config, options = {}) {
       appendDiagnostic(logs, 'process_exit', exitDiagnostic);
 
       if (hasResult) {
+        // Enrich cost from stream-json if available (best-effort, never blocks settle)
+        if (costParser) {
+          try {
+            const parsedCost = costParser.getResult();
+            if (parsedCost) {
+              const stagingPath = join(root, getTurnStagingResultPath(turn.turn_id));
+              const raw = readFileSync(stagingPath, 'utf8');
+              const turnResult = JSON.parse(raw);
+              turnResult.cost = buildCostFromStreamJson(parsedCost, config);
+              writeFileSync(stagingPath, JSON.stringify(turnResult, null, 2) + '\n');
+              appendDiagnostic(logs, 'stream_json_cost_enriched', {
+                input_tokens: parsedCost.input_tokens,
+                output_tokens: parsedCost.output_tokens,
+                cost_usd: parsedCost.cost_usd,
+                model: parsedCost.model,
+                computed_usd: turnResult.cost.usd,
+              });
+            }
+          } catch {
+            // Cost enrichment is best-effort — if it fails, agent-reported cost is preserved
+          }
+        }
         settle({ ok: true, exitCode, timedOut: false, aborted: false, logs, firstOutputAt });
       } else if (isClaudeLocalCliRuntime(runtime) && hasClaudeAuthFailureOutput(logs)) {
         const recovery = 'Refresh Claude credentials before resuming: export a valid ANTHROPIC_API_KEY or CLAUDE_CODE_OAUTH_TOKEN, then run agentxchain step --resume.';
@@ -551,6 +644,26 @@ export async function dispatchLocalCli(root, state, config, options = {}) {
           error: `Claude local_cli runtime is using an incompatible Node.js version. ${recovery}`,
           logs,
         });
+      } else if (hasRateLimitOutput(logs)) {
+        const resetMs = parseRateLimitResetMs(logs.join('\n'));
+        const resetDetail = resetMs ? ` Resets in ${Math.ceil(resetMs / 1000)}s.` : '';
+        const recovery = `Provider rate-limited.${resetDetail} Will retry with backoff.`;
+        settle({
+          ok: false,
+          blocked: true,
+          exitCode,
+          timedOut: false,
+          aborted: false,
+          firstOutputAt,
+          classified: {
+            error_class: 'rate_limited',
+            retryable: true,
+            recovery,
+            reset_ms: resetMs,
+          },
+          error: `Subprocess rate-limited by provider. ${recovery}`,
+          logs,
+        });
       } else if (startupTimedOut) {
         settle({
           ok: false,
@@ -749,6 +862,19 @@ function resolveStartupHeartbeatMs(config, runtime, override) {
   return DEFAULT_STARTUP_HEARTBEAT_MS;
 }
 
+function resolveLivenessHeartbeatMs(config, runtime, override) {
+  if (Number.isInteger(override) && override > 0) {
+    return override;
+  }
+  if (runtime?.type === 'local_cli' && Number.isInteger(runtime?.liveness_heartbeat_ms) && runtime.liveness_heartbeat_ms > 0) {
+    return runtime.liveness_heartbeat_ms;
+  }
+  if (Number.isInteger(config?.run_loop?.liveness_heartbeat_ms) && config.run_loop.liveness_heartbeat_ms > 0) {
+    return config.run_loop.liveness_heartbeat_ms;
+  }
+  return DEFAULT_LIVENESS_HEARTBEAT_MS;
+}
+
 function resolveStartupWatchdogKillGraceMs(value) {
   if (Number.isInteger(value) && value >= 0) {
     return value;
@@ -827,6 +953,63 @@ function validateLocalCliCommandCompatibility({ command, args = [], runtimeId =
     };
   }
 
+  const usesCursor = isCursorLocalCliRuntime(runtimeShape);
+  if (usesCursor && !tokens.includes('--background-agent') && !tokens.includes('agent')) {
+    const runtimeLabel = runtimeId ? `Runtime "${runtimeId}"` : 'Cursor local_cli runtime';
+    const recovery = `${runtimeLabel} uses "cursor" without a background agent flag. Governed local runs require Cursor's agent or background-agent mode for non-interactive execution.`;
+    return {
+      ok: false,
+      error_class: 'local_cli_command_incompatible',
+      recovery,
+      error: recovery,
+      diagnostic: {
+        runtime_id: runtimeId,
+        binary: binaryName,
+        rule: 'cursor_requires_agent_mode',
+        has_agent_flag: false,
+        recovery,
+      },
+    };
+  }
+
+  const usesWindsurf = isWindsurfLocalCliRuntime(runtimeShape);
+  if (usesWindsurf && !tokens.includes('--agent')) {
+    const runtimeLabel = runtimeId ? `Runtime "${runtimeId}"` : 'Windsurf local_cli runtime';
+    const recovery = `${runtimeLabel} uses "windsurf" without an agent flag. Governed local runs require Windsurf's --agent mode for non-interactive execution.`;
+    return {
+      ok: false,
+      error_class: 'local_cli_command_incompatible',
+      recovery,
+      error: recovery,
+      diagnostic: {
+        runtime_id: runtimeId,
+        binary: binaryName,
+        rule: 'windsurf_requires_agent_mode',
+        has_agent_flag: false,
+        recovery,
+      },
+    };
+  }
+
+  const usesOpenCode = isOpenCodeLocalCliRuntime(runtimeShape);
+  if (usesOpenCode && !tokens.includes('--non-interactive')) {
+    const runtimeLabel = runtimeId ? `Runtime "${runtimeId}"` : 'OpenCode local_cli runtime';
+    const recovery = `${runtimeLabel} uses "opencode" without --non-interactive. Governed local runs require OpenCode's non-interactive mode for unattended execution.`;
+    return {
+      ok: false,
+      error_class: 'local_cli_command_incompatible',
+      recovery,
+      error: recovery,
+      diagnostic: {
+        runtime_id: runtimeId,
+        binary: binaryName,
+        rule: 'opencode_requires_non_interactive',
+        has_non_interactive_flag: false,
+        recovery,
+      },
+    };
+  }
+
   return { ok: true };
 }
 
@@ -953,4 +1136,5 @@ export { resolveCommand };
 export { resolvePromptTransport };
 export { resolveStartupWatchdogMs };
 export { resolveStartupHeartbeatMs };
+export { resolveLivenessHeartbeatMs };
 export { validateLocalCliCommandCompatibility };

package/src/lib/api/execution-worker.js

@@ -0,0 +1,192 @@
+/**
+ * Execution Worker — in-process worker that polls the job queue and executes
+ * governed turns via run-loop + api_proxy adapter composition.
+ *
+ * Design rules:
+ *   - Composes run-loop directly (protocol parity invariant)
+ *   - Single turn per job (maxTurns: 1)
+ *   - Heartbeat-based liveness per execution plane spec rule 3
+ *   - Structured execution events per spec rule 6
+ *
+ * @module execution-worker
+ */
+
+import { randomUUID } from 'node:crypto';
+import { existsSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+import { runLoop } from '../run-loop.js';
+import { dispatchApiProxy } from '../adapters/api-proxy-adapter.js';
+import { getTurnStagingResultPath } from '../runner-interface.js';
+
+const DEFAULT_POLL_INTERVAL_MS = 2000;
+const HEARTBEAT_INTERVAL_MS = 30_000;
+
+/**
+ * Create an execution worker.
+ * @param {object} options
+ * @param {string} options.root - project root directory
+ * @param {object} options.config - normalized config
+ * @param {object} options.queue - job queue instance
+ * @param {function} [options.onEvent] - structured event callback
+ * @param {string} [options.workerId] - worker identifier
+ * @param {number} [options.pollIntervalMs=2000] - queue poll interval
+ * @returns {{ start(): void, stop(): void, getStatus(): object }}
+ */
+export function createExecutionWorker(options) {
+  const {
+    root,
+    config,
+    queue,
+    onEvent,
+    workerId = `worker-${randomUUID().slice(0, 8)}`,
+    pollIntervalMs = DEFAULT_POLL_INTERVAL_MS,
+  } = options;
+
+  let running = false;
+  let pollTimer = null;
+  let currentJob = null;
+  let currentAbort = null;
+
+  function emit(type, data = {}) {
+    if (!onEvent) return;
+    try {
+      onEvent({ type, worker_id: workerId, timestamp: new Date().toISOString(), ...data });
+    } catch {
+      // swallow event handler errors
+    }
+  }
+
+  async function executeJob(job, lease) {
+    currentJob = job;
+    const abortController = new AbortController();
+    currentAbort = abortController;
+
+    emit('execution_started', { job_id: job.job_id, run_id: job.run_id, role: job.role });
+
+    // Heartbeat interval
+    const heartbeatTimer = setInterval(() => {
+      const alive = queue.heartbeat(lease.lease_id);
+      if (!alive) {
+        abortController.abort();
+      }
+    }, HEARTBEAT_INTERVAL_MS);
+
+    try {
+      const result = await runLoop(root, config, {
+        selectRole(state) {
+          return job.role;
+        },
+
+        async dispatch(context) {
+          const adapterResult = await dispatchApiProxy(root, context.state, config, {
+            turnId: context.turn.turn_id,
+            signal: abortController.signal,
+          });
+
+          if (!adapterResult.ok) {
+            return { accept: false, reason: adapterResult.error || 'api_proxy dispatch failed' };
+          }
+
+          // api_proxy stages to disk; read back for run-loop acceptance
+          const stagingFile = join(root, getTurnStagingResultPath(context.turn.turn_id));
+          if (!existsSync(stagingFile)) {
+            return { accept: false, reason: 'adapter completed but no staged result found' };
+          }
+
+          let turnResult;
+          try {
+            turnResult = JSON.parse(readFileSync(stagingFile, 'utf8'));
+          } catch (err) {
+            return { accept: false, reason: `failed to parse staged result: ${err.message}` };
+          }
+
+          return { accept: true, turnResult };
+        },
+
+        approveGate() {
+          return true; // auto-approve in hosted mode
+        },
+
+        onEvent(event) {
+          emit('execution_progress', { job_id: job.job_id, event });
+        },
+      }, { maxTurns: 1 });
+
+      clearInterval(heartbeatTimer);
+
+      const success = result.ok;
+      queue.finalize(lease.lease_id, success ? 'completed' : 'failed');
+
+      emit(success ? 'execution_completed' : 'execution_interrupted', {
+        job_id: job.job_id,
+        run_id: job.run_id,
+        result_status: result.stop_reason || (success ? 'completed' : 'failed'),
+        turns_executed: result.turnsExecuted,
+      });
+
+      return result;
+    } catch (err) {
+      clearInterval(heartbeatTimer);
+      queue.finalize(lease.lease_id, 'failed');
+
+      emit('execution_interrupted', {
+        job_id: job.job_id,
+        run_id: job.run_id,
+        error: err.message,
+      });
+
+      return null;
+    } finally {
+      currentJob = null;
+      currentAbort = null;
+    }
+  }
+
+  async function poll() {
+    if (!running) return;
+
+    const claimed = queue.claim(workerId);
+    if (claimed) {
+      await executeJob(claimed.job, claimed.lease);
+    }
+
+    // Expire stale leases periodically
+    queue.expireStaleLeases();
+
+    // Schedule next poll
+    if (running) {
+      pollTimer = setTimeout(poll, pollIntervalMs);
+    }
+  }
+
+  function start() {
+    if (running) return;
+    running = true;
+    emit('worker_started', {});
+    // Start polling immediately
+    pollTimer = setTimeout(poll, 0);
+  }
+
+  function stop() {
+    running = false;
+    if (pollTimer) {
+      clearTimeout(pollTimer);
+      pollTimer = null;
+    }
+    if (currentAbort) {
+      currentAbort.abort();
+    }
+    emit('worker_stopped', {});
+  }
+
+  function getStatus() {
+    return {
+      worker_id: workerId,
+      running,
+      current_job: currentJob ? currentJob.job_id : null,
+    };
+  }
+
+  return { start, stop, getStatus };
+}