agent-relay 3.1.19 → 3.1.21

package/packages/sdk/src/workflows/runner.ts

@@ -13,6 +13,7 @@ import path from 'node:path';
 
 import { parse as parseYaml } from 'yaml';
 import { stripAnsi as stripAnsiFn } from '../pty.js';
+import type { BrokerEvent } from '../protocol.js';
 
 import {
   loadCustomSteps,
@@ -64,6 +65,25 @@ export interface WorkflowDb {
   getStepsByRunId(runId: string): Promise<WorkflowStepRow[]>;
 }
 
+/** Result returned by spawnAndWait / execNonInteractive with optional process exit info. */
+interface SpawnResult {
+  output: string;
+  exitCode?: number;
+  exitSignal?: string;
+}
+
+/** Error carrying exit code/signal from a failed subprocess spawn. */
+class SpawnExitError extends Error {
+  exitCode?: number;
+  exitSignal?: string;
+  constructor(message: string, exitCode?: number, exitSignal?: string | null) {
+    super(message);
+    this.name = 'SpawnExitError';
+    this.exitCode = exitCode;
+    this.exitSignal = exitSignal ?? undefined;
+  }
+}
+
 // ── Events ──────────────────────────────────────────────────────────────────
 
 export type WorkflowEvent =
@@ -71,6 +91,7 @@ export type WorkflowEvent =
   | { type: 'run:completed'; runId: string }
   | { type: 'run:failed'; runId: string; error: string }
   | { type: 'run:cancelled'; runId: string }
+  | { type: 'broker:event'; runId: string; event: BrokerEvent }
   | { type: 'step:started'; runId: string; stepName: string }
   | {
       type: 'step:owner-assigned';
@@ -79,7 +100,7 @@ export type WorkflowEvent =
       ownerName: string;
       specialistName: string;
     }
-  | { type: 'step:completed'; runId: string; stepName: string; output?: string }
+  | { type: 'step:completed'; runId: string; stepName: string; output?: string; exitCode?: number; exitSignal?: string }
   | {
       type: 'step:review-completed';
       runId: string;
@@ -88,7 +109,7 @@ export type WorkflowEvent =
       decision: 'approved' | 'rejected';
     }
   | { type: 'step:owner-timeout'; runId: string; stepName: string; ownerName: string }
-  | { type: 'step:failed'; runId: string; stepName: string; error: string }
+  | { type: 'step:failed'; runId: string; stepName: string; error: string; exitCode?: number; exitSignal?: string }
   | { type: 'step:skipped'; runId: string; stepName: string }
   | { type: 'step:retrying'; runId: string; stepName: string; attempt: number }
   | { type: 'step:nudged'; runId: string; stepName: string; nudgeCount: number }
@@ -223,6 +244,8 @@ export class WorkflowRunner {
 
   // PTY-based output capture: accumulate terminal output per-agent
   private readonly ptyOutputBuffers = new Map<string, string[]>();
+  /** Snapshot of PTY output from the most recent failed attempt, keyed by step name. */
+  private readonly lastFailedStepOutput = new Map<string, string>();
   private readonly ptyListeners = new Map<string, (chunk: string) => void>();
   private readonly ptyLogStreams = new Map<string, WriteStream>();
   /** Path to workers.json so `agents:kill` can find workflow-spawned agents */
@@ -1142,8 +1165,15 @@ export class WorkflowRunner {
     workflowName?: string,
     vars?: VariableContext
   ): Promise<WorkflowRunRow> {
+    // Set up abort controller early so callers can abort() even during setup
+    this.abortController = new AbortController();
+    this.paused = false;
+
     const resolved = vars ? this.resolveVariables(config, vars) : config;
 
+    // Validate config (catches cycles, missing deps, invalid steps, etc.)
+    this.validateConfig(resolved);
+
     // Resolve and validate named paths from the top-level `paths` config
     const pathResult = this.resolvePathDefinitions(resolved.paths, this.cwd);
     if (pathResult.errors.length > 0) {
@@ -1227,6 +1257,10 @@ export class WorkflowRunner {
 
   /** Resume a previously paused or partially completed run. */
   async resume(runId: string, vars?: VariableContext): Promise<WorkflowRunRow> {
+    // Set up abort controller early so callers can abort() even during setup
+    this.abortController = new AbortController();
+    this.paused = false;
+
     const run = await this.db.getRun(runId);
     if (!run) {
       throw new Error(`Run "${runId}" not found`);
@@ -1289,9 +1323,7 @@ export class WorkflowRunner {
     const { run, workflow, config, stepStates, isResume } = input;
     const runId = run.id;
 
-    // Start execution
-    this.abortController = new AbortController();
-    this.paused = false;
+    // Start execution (abortController already set by execute()/resume())
     this.currentConfig = config;
     this.currentRunId = runId;
     this.runStartTime = Date.now();
@@ -1336,15 +1368,22 @@ export class WorkflowRunner {
         config.swarm.channel = channel;
         await this.db.updateRun(runId, { config });
       }
+      const relaycastDisabled =
+        this.relayOptions.env?.AGENT_RELAY_WORKFLOW_DISABLE_RELAYCAST === '1';
+      const requiresBroker =
+        !this.executor &&
+        workflow.steps.some((step) => step.type !== 'deterministic' && step.type !== 'worktree');
       // Skip broker/relay init when an external executor handles agent spawning
-      if (!this.executor) {
-        this.log('Resolving Relaycast API key...');
-        await this.ensureRelaycastApiKey(channel);
-        this.log('API key resolved');
-        if (this.relayApiKeyAutoCreated && this.relayApiKey) {
-          this.log(`Workspace created — follow this run in Relaycast:`);
-          this.log(`  Observer: https://agentrelay.dev/observer?key=${this.relayApiKey}`);
-          this.log(`  Channel: ${channel}`);
+      if (requiresBroker) {
+        if (!relaycastDisabled) {
+          this.log('Resolving Relaycast API key...');
+          await this.ensureRelaycastApiKey(channel);
+          this.log('API key resolved');
+          if (this.relayApiKeyAutoCreated && this.relayApiKey) {
+            this.log(`Workspace created — follow this run in Relaycast:`);
+            this.log(`  Observer: https://agentrelay.dev/observer?key=${this.relayApiKey}`);
+            this.log(`  Channel: ${channel}`);
+          }
         }
 
         this.log('Starting broker...');
@@ -1356,7 +1395,7 @@ export class WorkflowRunner {
         this.relay = new AgentRelay({
           ...this.relayOptions,
           brokerName,
-          channels: [channel],
+          channels: relaycastDisabled ? [] : [channel],
           env: this.getRelayEnv(),
           // Workflows spawn agents across multiple waves; each spawn requires a PTY +
           // Relaycast registration. 60s is too tight when the broker is saturated with
@@ -1412,6 +1451,18 @@ export class WorkflowRunner {
 
         // Wire relay event hooks for rich console logging
         this.relay.onMessageReceived = (msg) => {
+          this.emit({
+            type: 'broker:event',
+            runId,
+            event: {
+              kind: 'relay_inbound',
+              event_id: msg.eventId,
+              from: msg.from,
+              target: msg.to,
+              body: msg.text,
+              thread_id: msg.threadId,
+            } as BrokerEvent,
+          });
           const body = msg.text.length > 120 ? msg.text.slice(0, 117) + '...' : msg.text;
           const fromShort = msg.from.replace(/-[a-f0-9]{6,}$/, '');
           const toShort = msg.to.replace(/-[a-f0-9]{6,}$/, '');
@@ -1429,13 +1480,43 @@ export class WorkflowRunner {
         };
 
         this.relay.onAgentSpawned = (agent) => {
+          this.emit({
+            type: 'broker:event',
+            runId,
+            event: {
+              kind: 'agent_spawned',
+              name: agent.name,
+              runtime: agent.runtime,
+            } as BrokerEvent,
+          });
           // Skip agents already managed by step execution
           if (!this.activeAgentHandles.has(agent.name)) {
             this.log(`[spawned] ${agent.name} (${agent.runtime})`);
           }
         };
 
+        this.relay.onAgentReleased = (agent) => {
+          this.emit({
+            type: 'broker:event',
+            runId,
+            event: {
+              kind: 'agent_released',
+              name: agent.name,
+            } as BrokerEvent,
+          });
+        };
+
         this.relay.onAgentExited = (agent) => {
+          this.emit({
+            type: 'broker:event',
+            runId,
+            event: {
+              kind: 'agent_exited',
+              name: agent.name,
+              code: agent.exitCode,
+              signal: agent.exitSignal,
+            } as BrokerEvent,
+          });
           this.lastActivity.delete(agent.name);
           this.lastIdleLog.delete(agent.name);
           if (!this.activeAgentHandles.has(agent.name)) {
@@ -1443,7 +1524,20 @@ export class WorkflowRunner {
           }
         };
 
+        this.relay.onDeliveryUpdate = (event) => {
+          this.emit({ type: 'broker:event', runId, event });
+        };
+
         this.relay.onAgentIdle = ({ name, idleSecs }) => {
+          this.emit({
+            type: 'broker:event',
+            runId,
+            event: {
+              kind: 'agent_idle',
+              name,
+              idle_secs: idleSecs,
+            } as BrokerEvent,
+          });
           // Only log at 30s multiples to avoid watchdog spam
           const bucket = Math.floor(idleSecs / 30) * 30;
           if (bucket >= 30 && this.lastIdleLog.get(name) !== bucket) {
@@ -1461,20 +1555,22 @@ export class WorkflowRunner {
           console.log(`[broker] ${line}`);
         });
 
-        this.log(`Creating channel: ${channel}...`);
-        if (isResume) {
-          await this.createAndJoinRelaycastChannel(channel);
-        } else {
-          await this.createAndJoinRelaycastChannel(channel, workflow.description);
-        }
-        this.log('Channel ready');
+        if (!relaycastDisabled) {
+          this.log(`Creating channel: ${channel}...`);
+          if (isResume) {
+            await this.createAndJoinRelaycastChannel(channel);
+          } else {
+            await this.createAndJoinRelaycastChannel(channel, workflow.description);
+          }
+          this.log('Channel ready');
 
-        if (isResume) {
-          this.postToChannel(`Workflow **${workflow.name}** resumed — ${pendingCount} pending steps`);
-        } else {
-          this.postToChannel(
-            `Workflow **${workflow.name}** started — ${workflow.steps.length} steps, pattern: ${config.swarm.pattern}`
-          );
+          if (isResume) {
+            this.postToChannel(`Workflow **${workflow.name}** resumed — ${pendingCount} pending steps`);
+          } else {
+            this.postToChannel(
+              `Workflow **${workflow.name}** started — ${workflow.steps.length} steps, pattern: ${config.swarm.pattern}`
+            );
+          }
         }
       }
 
@@ -1491,8 +1587,15 @@ export class WorkflowRunner {
       this.log(`Executing ${workflow.steps.length} steps (pattern: ${config.swarm.pattern})`);
       await this.executeSteps(workflow, stepStates, agentMap, config.errorHandling, runId);
 
+      const errorStrategy =
+        config.errorHandling?.strategy ?? workflow.onError ?? 'fail-fast';
+      const continueOnError =
+        errorStrategy === 'continue' || errorStrategy === 'skip';
       const allCompleted = [...stepStates.values()].every(
-        (s) => s.row.status === 'completed' || s.row.status === 'skipped'
+        (s) =>
+          s.row.status === 'completed' ||
+          s.row.status === 'skipped' ||
+          (continueOnError && s.row.status === 'failed')
       );
 
       if (allCompleted) {
@@ -1517,9 +1620,18 @@ export class WorkflowRunner {
         this.emit({ type: 'run:failed', runId, error: errorMsg });
 
         const outcomes = this.collectOutcomes(stepStates, workflow.steps);
+        const summary = this.trajectory.buildRunSummary(outcomes);
+        const confidence = this.trajectory.computeConfidence(outcomes);
+        const learnings = this.trajectory.extractLearnings(outcomes);
+        const challenges = this.trajectory.extractChallenges(outcomes);
         this.postFailureReport(workflow.name, outcomes, errorMsg);
         this.logRunSummary(workflow.name, outcomes, runId);
-        await this.trajectory.abandon(errorMsg);
+        await this.trajectory.abandon(errorMsg, {
+          summary,
+          confidence,
+          learnings,
+          challenges,
+        });
       }
     } catch (err) {
       const errorMsg = err instanceof Error ? err.message : String(err);
@@ -1528,15 +1640,35 @@ export class WorkflowRunner {
       await this.updateRunStatus(runId, status, errorMsg);
 
       if (status === 'cancelled') {
+        // Mark any pending or in-progress steps as failed due to cancellation
+        for (const [stepName, state] of stepStates) {
+          if (state.row.status === 'pending' || state.row.status === 'running') {
+            state.row.status = 'failed';
+            state.row.error = 'Cancelled';
+            await this.db.updateStep(state.row.id, {
+              status: 'failed',
+              error: 'Cancelled',
+              updatedAt: new Date().toISOString(),
+            });
+            this.emit({ type: 'step:failed', runId, stepName, error: 'Cancelled' });
+          }
+        }
         this.emit({ type: 'run:cancelled', runId });
         this.postToChannel(`Workflow **${workflow.name}** cancelled`);
         await this.trajectory.abandon('Cancelled by user');
       } else {
         this.emit({ type: 'run:failed', runId, error: errorMsg });
         this.postToChannel(`Workflow failed: ${errorMsg}`);
-        await this.trajectory.abandon(errorMsg);
+        const outcomes = this.collectOutcomes(stepStates, workflow.steps);
+        await this.trajectory.abandon(errorMsg, {
+          summary: this.trajectory.buildRunSummary(outcomes),
+          confidence: this.trajectory.computeConfidence(outcomes),
+          learnings: this.trajectory.extractLearnings(outcomes),
+          challenges: this.trajectory.extractChallenges(outcomes),
+        });
       }
     } finally {
+      this.lastFailedStepOutput.clear();
       for (const stream of this.ptyLogStreams.values()) stream.end();
       this.ptyLogStreams.clear();
       this.ptyOutputBuffers.clear();
@@ -1549,9 +1681,11 @@ export class WorkflowRunner {
       if (this.relay) {
         this.relay.onMessageReceived = null;
         this.relay.onAgentSpawned = null;
+        this.relay.onAgentReleased = null;
         this.relay.onAgentExited = null;
         this.relay.onAgentIdle = null;
         this.relay.onWorkerOutput = null;
+        this.relay.onDeliveryUpdate = null;
       }
       this.lastIdleLog.clear();
       this.lastActivity.clear();
@@ -1867,7 +2001,7 @@ export class WorkflowRunner {
   ): Promise<void> {
     // Branch: deterministic steps execute shell commands
     if (this.isDeterministicStep(step)) {
-      return this.executeDeterministicStep(step, stepStates, runId);
+      return this.executeDeterministicStep(step, stepStates, runId, errorHandling);
     }
 
     // Branch: worktree steps set up git worktrees
@@ -1886,167 +2020,199 @@ export class WorkflowRunner {
   private async executeDeterministicStep(
     step: WorkflowStep,
     stepStates: Map<string, StepState>,
-    runId: string
+    runId: string,
+    errorHandling: ErrorHandlingConfig | undefined
   ): Promise<void> {
     const state = stepStates.get(step.name);
     if (!state) throw new Error(`Step state not found: ${step.name}`);
 
-    this.checkAborted();
-
-    // Mark step as running
-    state.row.status = 'running';
-    state.row.startedAt = new Date().toISOString();
-    await this.db.updateStep(state.row.id, {
-      status: 'running',
-      startedAt: state.row.startedAt,
-      updatedAt: new Date().toISOString(),
-    });
-    this.emit({ type: 'step:started', runId, stepName: step.name });
-    this.postToChannel(`**[${step.name}]** Started (deterministic)`);
-
-    // Resolve variables in the command (e.g., {{steps.plan.output}}, {{branch-name}})
-    const stepOutputContext = this.buildStepOutputContext(stepStates, runId);
-    let resolvedCommand = this.interpolateStepTask(step.command ?? '', stepOutputContext);
-
-    // Also resolve simple {{variable}} placeholders (already resolved in top-level config but safe to re-run)
-    resolvedCommand = resolvedCommand.replace(/\{\{([\w][\w.\-]*)\}\}/g, (_match, key: string) => {
-      if (key.startsWith('steps.')) return _match; // Already handled above
-      const value = this.resolveDotPath(key, stepOutputContext);
-      return value !== undefined ? String(value) : _match;
-    });
-
-    // Resolve step workdir (named path reference) for deterministic steps
-    const stepCwd = this.resolveStepWorkdir(step) ?? this.cwd;
+    const maxRetries = step.retries ?? errorHandling?.maxRetries ?? 0;
+    const retryDelay = errorHandling?.retryDelayMs ?? 1000;
+    let lastError: string | undefined;
 
-    try {
-      // Delegate to executor if present
-      if (this.executor?.executeDeterministicStep) {
-        const result = await this.executor.executeDeterministicStep(step, resolvedCommand, stepCwd);
-        const failOnError = step.failOnError !== false;
-        if (failOnError && result.exitCode !== 0) {
-          throw new Error(`Command failed with exit code ${result.exitCode}: ${result.output.slice(0, 500)}`);
-        }
-        const output =
-          step.captureOutput !== false ? result.output : `Command completed (exit code ${result.exitCode})`;
+    for (let attempt = 0; attempt <= maxRetries; attempt += 1) {
+      this.checkAborted();
 
-        // Mark completed
-        state.row.status = 'completed';
-        state.row.output = output;
-        state.row.completedAt = new Date().toISOString();
+      if (attempt > 0) {
+        this.emit({ type: 'step:retrying', runId, stepName: step.name, attempt });
+        this.postToChannel(`**[${step.name}]** Retrying (attempt ${attempt + 1}/${maxRetries + 1})`);
+        state.row.retryCount = attempt;
         await this.db.updateStep(state.row.id, {
-          status: 'completed',
-          output,
-          completedAt: state.row.completedAt,
+          retryCount: attempt,
           updatedAt: new Date().toISOString(),
         });
-        await this.persistStepOutput(runId, step.name, output);
-        this.emit({ type: 'step:completed', runId, stepName: step.name, output });
-        return;
+        await this.delay(retryDelay);
       }
 
-      const output = await new Promise<string>((resolve, reject) => {
-        const child = cpSpawn('sh', ['-c', resolvedCommand], {
-          stdio: 'pipe',
-          cwd: stepCwd,
-          env: { ...process.env },
-        });
+      // Mark step as running
+      state.row.status = 'running';
+      state.row.startedAt = new Date().toISOString();
+      await this.db.updateStep(state.row.id, {
+        status: 'running',
+        startedAt: state.row.startedAt,
+        updatedAt: new Date().toISOString(),
+      });
+      this.emit({ type: 'step:started', runId, stepName: step.name });
+      this.postToChannel(`**[${step.name}]** Started (deterministic)`);
+
+      // Resolve variables in the command (e.g., {{steps.plan.output}}, {{branch-name}})
+      const stepOutputContext = this.buildStepOutputContext(stepStates, runId);
+      let resolvedCommand = this.interpolateStepTask(step.command ?? '', stepOutputContext);
+
+      // Also resolve simple {{variable}} placeholders (already resolved in top-level config but safe to re-run)
+      resolvedCommand = resolvedCommand.replace(/\{\{([\w][\w.\-]*)\}\}/g, (_match, key: string) => {
+        if (key.startsWith('steps.')) return _match; // Already handled above
+        const value = this.resolveDotPath(key, stepOutputContext);
+        return value !== undefined ? String(value) : _match;
+      });
 
-        const stdoutChunks: string[] = [];
-        const stderrChunks: string[] = [];
+      // Resolve step workdir (named path reference) for deterministic steps
+      const stepCwd = this.resolveStepWorkdir(step) ?? this.cwd;
 
-        // Wire abort signal
-        const abortSignal = this.abortController?.signal;
-        let abortHandler: (() => void) | undefined;
-        if (abortSignal && !abortSignal.aborted) {
-          abortHandler = () => {
-            child.kill('SIGTERM');
-            setTimeout(() => child.kill('SIGKILL'), 5000);
-          };
-          abortSignal.addEventListener('abort', abortHandler, { once: true });
-        }
+      try {
+        // Delegate to executor if present
+        if (this.executor?.executeDeterministicStep) {
+          const result = await this.executor.executeDeterministicStep(step, resolvedCommand, stepCwd);
+          const failOnError = step.failOnError !== false;
+          if (failOnError && result.exitCode !== 0) {
+            throw new Error(
+              `Command failed with exit code ${result.exitCode}: ${result.output.slice(0, 500)}`
+            );
+          }
+          const output =
+            step.captureOutput !== false ? result.output : `Command completed (exit code ${result.exitCode})`;
+          if (step.verification) {
+            this.runVerification(step.verification, output, step.name);
+          }
 
-        // Handle timeout
-        let timedOut = false;
-        let timer: ReturnType<typeof setTimeout> | undefined;
-        if (step.timeoutMs) {
-          timer = setTimeout(() => {
-            timedOut = true;
-            child.kill('SIGTERM');
-            setTimeout(() => child.kill('SIGKILL'), 5000);
-          }, step.timeoutMs);
+          // Mark completed
+          state.row.status = 'completed';
+          state.row.output = output;
+          state.row.completedAt = new Date().toISOString();
+          await this.db.updateStep(state.row.id, {
+            status: 'completed',
+            output,
+            completedAt: state.row.completedAt,
+            updatedAt: new Date().toISOString(),
+          });
+          await this.persistStepOutput(runId, step.name, output);
+          this.emit({ type: 'step:completed', runId, stepName: step.name, output });
+          return;
         }
 
-        child.stdout?.on('data', (chunk: Buffer) => {
-          stdoutChunks.push(chunk.toString());
-        });
+        const output = await new Promise<string>((resolve, reject) => {
+          const child = cpSpawn('sh', ['-c', resolvedCommand], {
+            stdio: 'pipe',
+            cwd: stepCwd,
+            env: { ...process.env },
+          });
 
-        child.stderr?.on('data', (chunk: Buffer) => {
-          stderrChunks.push(chunk.toString());
-        });
+          const stdoutChunks: string[] = [];
+          const stderrChunks: string[] = [];
 
-        child.on('close', (code) => {
-          if (timer) clearTimeout(timer);
-          if (abortHandler && abortSignal) {
-            abortSignal.removeEventListener('abort', abortHandler);
+          // Wire abort signal
+          const abortSignal = this.abortController?.signal;
+          let abortHandler: (() => void) | undefined;
+          if (abortSignal && !abortSignal.aborted) {
+            abortHandler = () => {
+              child.kill('SIGTERM');
+              setTimeout(() => child.kill('SIGKILL'), 5000);
+            };
+            abortSignal.addEventListener('abort', abortHandler, { once: true });
           }
 
-          if (abortSignal?.aborted) {
-            reject(new Error(`Step "${step.name}" aborted`));
-            return;
+          // Handle timeout
+          let timedOut = false;
+          let timer: ReturnType<typeof setTimeout> | undefined;
+          if (step.timeoutMs) {
+            timer = setTimeout(() => {
+              timedOut = true;
+              child.kill('SIGTERM');
+              setTimeout(() => child.kill('SIGKILL'), 5000);
+            }, step.timeoutMs);
           }
 
-          if (timedOut) {
-            reject(
-              new Error(`Step "${step.name}" timed out (no step timeout set, check global swarm.timeoutMs)`)
-            );
-            return;
-          }
+          child.stdout?.on('data', (chunk: Buffer) => {
+            stdoutChunks.push(chunk.toString());
+          });
 
-          const stdout = stdoutChunks.join('');
-          const stderr = stderrChunks.join('');
+          child.stderr?.on('data', (chunk: Buffer) => {
+            stderrChunks.push(chunk.toString());
+          });
 
-          // Check exit code unless failOnError is explicitly false
-          const failOnError = step.failOnError !== false;
-          if (failOnError && code !== 0 && code !== null) {
-            reject(
-              new Error(`Command failed with exit code ${code}${stderr ? `: ${stderr.slice(0, 500)}` : ''}`)
-            );
-            return;
-          }
+          child.on('close', (code) => {
+            if (timer) clearTimeout(timer);
+            if (abortHandler && abortSignal) {
+              abortSignal.removeEventListener('abort', abortHandler);
+            }
 
-          resolve(step.captureOutput !== false ? stdout : `Command completed (exit code ${code ?? 0})`);
-        });
+            if (abortSignal?.aborted) {
+              reject(new Error(`Step "${step.name}" aborted`));
+              return;
+            }
 
-        child.on('error', (err) => {
-          if (timer) clearTimeout(timer);
-          if (abortHandler && abortSignal) {
-            abortSignal.removeEventListener('abort', abortHandler);
-          }
-          reject(new Error(`Failed to execute command: ${err.message}`));
+            if (timedOut) {
+              reject(
+                new Error(`Step "${step.name}" timed out (no step timeout set, check global swarm.timeoutMs)`)
+              );
+              return;
+            }
+
+            const stdout = stdoutChunks.join('');
+            const stderr = stderrChunks.join('');
+
+            // Check exit code unless failOnError is explicitly false
+            const failOnError = step.failOnError !== false;
+            if (failOnError && code !== 0 && code !== null) {
+              reject(
+                new Error(
+                  `Command failed with exit code ${code}${stderr ? `: ${stderr.slice(0, 500)}` : ''}`
+                )
+              );
+              return;
+            }
+
+            resolve(step.captureOutput !== false ? stdout : `Command completed (exit code ${code ?? 0})`);
+          });
+
+          child.on('error', (err) => {
+            if (timer) clearTimeout(timer);
+            if (abortHandler && abortSignal) {
+              abortSignal.removeEventListener('abort', abortHandler);
+            }
+            reject(new Error(`Failed to execute command: ${err.message}`));
+          });
         });
-      });
 
-      // Mark completed
-      state.row.status = 'completed';
-      state.row.output = output;
-      state.row.completedAt = new Date().toISOString();
-      await this.db.updateStep(state.row.id, {
-        status: 'completed',
-        output,
-        completedAt: state.row.completedAt,
-        updatedAt: new Date().toISOString(),
-      });
+        if (step.verification) {
+          this.runVerification(step.verification, output, step.name);
+        }
 
-      // Persist step output
-      await this.persistStepOutput(runId, step.name, output);
+        // Mark completed
+        state.row.status = 'completed';
+        state.row.output = output;
+        state.row.completedAt = new Date().toISOString();
+        await this.db.updateStep(state.row.id, {
+          status: 'completed',
+          output,
+          completedAt: state.row.completedAt,
+          updatedAt: new Date().toISOString(),
+        });
 
-      this.emit({ type: 'step:completed', runId, stepName: step.name, output });
-    } catch (err) {
-      const errorMsg = err instanceof Error ? err.message : String(err);
-      this.postToChannel(`**[${step.name}]** Failed: ${errorMsg}`);
-      await this.markStepFailed(state, errorMsg, runId);
-      throw new Error(`Step "${step.name}" failed: ${errorMsg}`);
+        // Persist step output
+        await this.persistStepOutput(runId, step.name, output);
+
+        this.emit({ type: 'step:completed', runId, stepName: step.name, output });
+        return;
+      } catch (err) {
+        lastError = err instanceof Error ? err.message : String(err);
+      }
     }
+
+    const errorMsg = lastError ?? 'Unknown error';
+    this.postToChannel(`**[${step.name}]** Failed: ${errorMsg}`);
+    await this.markStepFailed(state, errorMsg, runId);
+    throw new Error(`Step "${step.name}" failed: ${errorMsg}`);
   }
 
   /**
@@ -2286,10 +2452,16 @@ export class WorkflowRunner {
       this.currentConfig?.swarm?.timeoutMs;
 
     let lastError: string | undefined;
+    let lastExitCode: number | undefined;
+    let lastExitSignal: string | undefined;
 
     for (let attempt = 0; attempt <= maxRetries; attempt++) {
       this.checkAborted();
 
+      // Reset per-attempt exit info so stale values don't leak across retries
+      lastExitCode = undefined;
+      lastExitSignal = undefined;
+
       if (attempt > 0) {
         this.emit({ type: 'step:retrying', runId, stepName: step.name, attempt });
         this.postToChannel(`**[${step.name}]** Retrying (attempt ${attempt + 1}/${maxRetries + 1})`);
@@ -2336,6 +2508,16 @@ export class WorkflowRunner {
         const stepOutputContext = this.buildStepOutputContext(stepStates, runId);
         let resolvedTask = this.interpolateStepTask(step.task ?? '', stepOutputContext);
 
+        // On retry attempts, prepend failure context so the agent knows what went wrong
+        if (attempt > 0 && lastError) {
+          const priorOutput = (this.lastFailedStepOutput.get(step.name) ?? '').slice(-2000);
+          resolvedTask =
+            `[RETRY — Attempt ${attempt + 1}/${maxRetries + 1}]\n` +
+            `Previous attempt failed: ${lastError}\n` +
+            (priorOutput ? `Previous output (last 2000 chars):\n${priorOutput}\n` : '') +
+            `---\n${resolvedTask}`;
+        }
+
         // If this is an interactive agent, append awareness of non-interactive workers
         // so the lead knows not to message them and to use step output chaining instead
         if (specialistDef.interactive !== false || ownerDef.interactive !== false) {
@@ -2378,9 +2560,12 @@ export class WorkflowRunner {
           this.log(`[${step.name}] Spawning owner "${effectiveOwner.name}" (cli: ${effectiveOwner.cli})${step.workdir ? ` [workdir: ${step.workdir}]` : ''}`);
           const resolvedStep = { ...step, task: ownerTask };
           const ownerStartTime = Date.now();
-          const output = this.executor
+          const spawnResult = this.executor
             ? await this.executor.executeAgentStep(resolvedStep, effectiveOwner, ownerTask, timeoutMs)
             : await this.spawnAndWait(effectiveOwner, resolvedStep, timeoutMs);
+          const output = typeof spawnResult === 'string' ? spawnResult : spawnResult.output;
+          lastExitCode = typeof spawnResult === 'string' ? undefined : spawnResult.exitCode;
+          lastExitSignal = typeof spawnResult === 'string' ? undefined : spawnResult.exitSignal;
           ownerElapsed = Date.now() - ownerStartTime;
           this.log(`[${step.name}] Owner "${effectiveOwner.name}" exited`);
           if (usesOwnerFlow) {
@@ -2392,7 +2577,12 @@ export class WorkflowRunner {
 
         // Run verification if configured
         if (step.verification) {
-          this.runVerification(step.verification, specialistOutput, step.name, resolvedTask);
+          this.runVerification(
+            step.verification,
+            specialistOutput,
+            step.name,
+            effectiveOwner.interactive === false ? undefined : resolvedTask
+          );
         }
 
         // Every interactive step gets a review pass; pick a dedicated reviewer when available.
@@ -2425,11 +2615,15 @@ export class WorkflowRunner {
         // Persist step output to disk so it survives restarts and is inspectable
         await this.persistStepOutput(runId, step.name, combinedOutput);
 
-        this.emit({ type: 'step:completed', runId, stepName: step.name, output: combinedOutput });
+        this.emit({ type: 'step:completed', runId, stepName: step.name, output: combinedOutput, exitCode: lastExitCode, exitSignal: lastExitSignal });
         await this.trajectory?.stepCompleted(step, combinedOutput, attempt + 1);
         return;
       } catch (err) {
         lastError = err instanceof Error ? err.message : String(err);
+        if (err instanceof SpawnExitError) {
+          lastExitCode = err.exitCode;
+          lastExitSignal = err.exitSignal;
+        }
         const ownerTimedOut = usesDedicatedOwner
           ? /\bowner timed out\b/i.test(lastError)
           : /\btimed out\b/i.test(lastError) && !lastError.includes(`${step.name}-review`);
@@ -2452,7 +2646,10 @@ export class WorkflowRunner {
       verificationValue,
     });
     this.postToChannel(`**[${step.name}]** Failed: ${lastError ?? 'Unknown error'}`);
-    await this.markStepFailed(state, lastError ?? 'Unknown error', runId);
+    await this.markStepFailed(state, lastError ?? 'Unknown error', runId, {
+      exitCode: lastExitCode,
+      exitSignal: lastExitSignal,
+    });
     throw new Error(
       `Step "${step.name}" failed after ${maxRetries} retries: ${lastError ?? 'Unknown error'}`
     );
@@ -2615,10 +2812,10 @@ export class WorkflowRunner {
 
     const workerSettled = workerPromise.catch(() => undefined);
     workerPromise
-      .then((output) => {
+      .then((result) => {
         workerReleased = true;
         this.postToChannel(`**[${step.name}]** Worker \`${workerRuntimeName}\` exited`);
-        if (step.verification?.type === 'output_contains' && output.includes(step.verification.value)) {
+        if (step.verification?.type === 'output_contains' && result.output.includes(step.verification.value)) {
           this.postToChannel(
             `**[${step.name}]** Verification gate observed: output contains ${JSON.stringify(step.verification.value)}`
           );
@@ -2645,7 +2842,7 @@ export class WorkflowRunner {
     const ownerStartTime = Date.now();
 
     try {
-      const ownerOutput = await this.spawnAndWait(supervised.owner, ownerStep, timeoutMs, {
+      const ownerResultObj = await this.spawnAndWait(supervised.owner, ownerStep, timeoutMs, {
         agentNameSuffix: 'owner',
         onSpawned: ({ actualName }) => {
           this.supervisedRuntimeAgents.set(actualName, {
@@ -2659,10 +2856,11 @@ export class WorkflowRunner {
         },
       });
       const ownerElapsed = Date.now() - ownerStartTime;
+      const ownerOutput = ownerResultObj.output;
       this.log(`[${step.name}] Owner "${supervised.owner.name}" exited`);
       this.assertOwnerCompletionMarker(step, ownerOutput, supervisorTask);
 
-      const specialistOutput = await workerPromise;
+      const specialistOutput = (await workerPromise).output;
       return { specialistOutput, ownerOutput, ownerElapsed };
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
@@ -2909,7 +3107,7 @@ export class WorkflowRunner {
     };
 
     try {
-      reviewOutput = await this.spawnAndWait(reviewerDef, reviewStep, safetyTimeoutMs, {
+      await this.spawnAndWait(reviewerDef, reviewStep, safetyTimeoutMs, {
         onSpawned: ({ agent }) => {
           reviewerHandle = agent;
         },
@@ -3089,7 +3287,7 @@ export class WorkflowRunner {
     agentDef: AgentDefinition,
     step: WorkflowStep,
     timeoutMs?: number
-  ): Promise<string> {
+  ): Promise<SpawnResult> {
     const agentName = `${step.name}-${this.generateShortId()}`;
     const modelArgs = agentDef.constraints?.model ? ['--model', agentDef.constraints.model] : [];
 
@@ -3152,7 +3350,7 @@ export class WorkflowRunner {
     const stderrChunks: string[] = [];
 
     try {
-      const output = await new Promise<string>((resolve, reject) => {
+      const { stdout: output, exitCode, exitSignal } = await new Promise<{ stdout: string; exitCode?: number; exitSignal?: string }>((resolve, reject) => {
         const child = cpSpawn(cmd, args, {
           stdio: ['ignore', 'pipe', 'pipe'],
           cwd: this.resolveAgentCwd(agentDef),
@@ -3217,7 +3415,7 @@ export class WorkflowRunner {
           }, timeoutMs);
         }
 
-        child.on('close', (code) => {
+        child.on('close', (code, signal) => {
           clearInterval(heartbeat);
           if (timer) clearTimeout(timer);
           if (abortHandler && abortSignal) {
@@ -3238,14 +3436,20 @@ export class WorkflowRunner {
           if (code !== 0 && code !== null) {
             const stderr = stderrChunks.join('');
             reject(
-              new Error(
-                `Step "${step.name}" exited with code ${code}${stderr ? `: ${stderr.slice(0, 500)}` : ''}`
+              new SpawnExitError(
+                `Step "${step.name}" exited with code ${code}${stderr ? `: ${stderr.slice(0, 500)}` : ''}`,
+                code,
+                signal
               )
             );
             return;
           }
 
-          resolve(stdout);
+          resolve({
+            stdout,
+            exitCode: code ?? undefined,
+            exitSignal: signal ?? undefined,
+          });
         });
 
         child.on('error', (err) => {
@@ -3258,8 +3462,10 @@ export class WorkflowRunner {
         });
       });
 
-      return output;
+      return { output, exitCode, exitSignal };
     } finally {
+      const combinedOutput = stdoutChunks.join('') + stderrChunks.join('');
+      this.lastFailedStepOutput.set(step.name, combinedOutput);
       stopHeartbeat?.();
       logStream.end();
       this.unregisterWorker(agentName);
@@ -3271,7 +3477,7 @@ export class WorkflowRunner {
     step: WorkflowStep,
     timeoutMs?: number,
     options: SpawnAndWaitOptions = {}
-  ): Promise<string> {
+  ): Promise<SpawnResult> {
     // Branch: non-interactive agents run as simple subprocesses
     if (agentDef.interactive === false) {
       return this.execNonInteractive(agentDef, step, timeoutMs);
@@ -3456,9 +3662,16 @@ export class WorkflowRunner {
           throw new Error(`Step "${step.name}" timed out after ${timeoutMs ?? 'unknown'}ms`);
         }
       }
+
+      if (exitResult === 'force-released') {
+        throw new Error(
+          `Step "${step.name}" failed — agent was force-released after exhausting idle nudges without completing`
+        );
+      }
     } finally {
       // Snapshot PTY chunks before cleanup — we need them for output reading below
       ptyChunks = this.ptyOutputBuffers.get(agentName) ?? [];
+      this.lastFailedStepOutput.set(step.name, ptyChunks.join(''));
 
       // Always clean up PTY resources — prevents fd leaks if spawnPty or waitForExit throws
       stopHeartbeat?.();
@@ -3485,11 +3698,15 @@ export class WorkflowRunner {
         : exitResult === 'timeout'
           ? 'Agent completed (released after idle timeout)'
           : exitResult === 'released'
-            ? 'Agent completed (force-released after idle nudging)'
+            ? 'Agent completed (idle — treated as done)'
             : `Agent exited (${exitResult})`;
     }
 
-    return output;
+    return {
+      output,
+      exitCode: agent?.exitCode,
+      exitSignal: agent?.exitSignal,
+    };
   }
 
   // ── Idle nudging ────────────────────────────────────────────────────────
@@ -3525,7 +3742,7 @@ export class WorkflowRunner {
     agentDef: AgentDefinition,
     step: WorkflowStep,
     timeoutMs?: number
-  ): Promise<'exited' | 'timeout' | 'released'> {
+  ): Promise<'exited' | 'timeout' | 'released' | 'force-released'> {
     const nudgeConfig = this.currentConfig?.swarm.idleNudge;
     if (!nudgeConfig) {
       // Idle = done: race exit against idle. Whichever fires first completes the step.
@@ -3576,7 +3793,7 @@ export class WorkflowRunner {
       }
 
       // Agent is still running after the window expired.
-      if (remaining !== undefined && Date.now() - startTime >= remaining) {
+      if (timeoutMs !== undefined && Date.now() - startTime >= timeoutMs) {
         return 'timeout';
       }
 
@@ -3595,7 +3812,7 @@ export class WorkflowRunner {
       );
       this.emit({ type: 'step:force-released', runId: this.currentRunId ?? '', stepName: step.name });
       await agent.release();
-      return 'released';
+      return 'force-released';
     }
   }
 
@@ -3731,7 +3948,12 @@ export class WorkflowRunner {
     await this.db.updateRun(runId, patch);
   }
 
-  private async markStepFailed(state: StepState, error: string, runId: string): Promise<void> {
+  private async markStepFailed(
+    state: StepState,
+    error: string,
+    runId: string,
+    exitInfo?: { exitCode?: number; exitSignal?: string }
+  ): Promise<void> {
     state.row.status = 'failed';
     state.row.error = error;
     state.row.completedAt = new Date().toISOString();
@@ -3741,7 +3963,14 @@ export class WorkflowRunner {
       completedAt: state.row.completedAt,
       updatedAt: new Date().toISOString(),
     });
-    this.emit({ type: 'step:failed', runId, stepName: state.row.stepName, error });
+    this.emit({
+      type: 'step:failed',
+      runId,
+      stepName: state.row.stepName,
+      error,
+      exitCode: exitInfo?.exitCode,
+      exitSignal: exitInfo?.exitSignal,
+    });
   }
 
   private async markDownstreamSkipped(