@the-open-engine/zeroshot 5.4.0

package/src/agent/agent-lifecycle.js

@@ -0,0 +1,982 @@
+// @ts-nocheck
+/**
+ * AgentLifecycle - Agent state machine and lifecycle management
+ *
+ * Provides:
+ * - Agent startup and shutdown
+ * - Message handling and routing
+ * - Trigger action execution (execute_task, stop_cluster)
+ * - Task execution with retry logic
+ * - Liveness monitoring with multi-indicator stuck detection
+ *
+ * State machine: idle → evaluating → building_context → executing → idle
+ */
+
+const { findMatchingTrigger, evaluateTrigger } = require('./agent-trigger-evaluator');
+const { executeHook } = require('./agent-hook-executor');
+const IsolationManager = require('../isolation-manager');
+const crypto = require('crypto');
+const { bufferMessage, scheduleDrain, drainBufferedMessages } = require('../message-buffer');
+const {
+  analyzeProcessHealth,
+  isPlatformSupported,
+  STUCK_THRESHOLD,
+} = require('./agent-stuck-detector');
+const { normalizeProviderName } = require('../../lib/provider-names');
+const { loadSettings } = require('../../lib/settings');
+const { findPlatformMismatchReason } = require('./validation-platform');
+const { calculateRateLimitDelay, isRateLimitError } = require('./rate-limit-backoff');
+
+const DEFAULT_VALIDATOR_IMAGE = 'zeroshot-cluster-base';
+
+class HookExecutionError extends Error {
+  constructor(message, options) {
+    super(message);
+    this.name = 'HookExecutionError';
+    this.hookFailure = true;
+    this.hookRetries = options?.hookRetries;
+    this.originalHookError = options?.originalHookError;
+  }
+}
+
+function resolveValidatorIsolationConfig(agent) {
+  const config = agent.config?.isolation || {};
+  if (config.type && config.type !== 'docker') {
+    return null;
+  }
+
+  return {
+    image: config.image || DEFAULT_VALIDATOR_IMAGE,
+    mounts: config.mounts,
+    noMounts: config.noMounts,
+    containerHome: config.containerHome,
+  };
+}
+
+async function createValidatorIsolation(agent, isolationConfig) {
+  if (!IsolationManager.isDockerAvailable()) {
+    agent._log(`[${agent.id}] Docker not available - cannot retry validator in isolation`);
+    return null;
+  }
+
+  const cluster = agent.cluster || {};
+  const workDir = agent.config?.cwd || cluster.worktree?.path || cluster.cwd || process.cwd();
+  const image = isolationConfig.image;
+  await IsolationManager.ensureImage(image);
+
+  const manager = new IsolationManager({ image });
+  const providerName = normalizeProviderName(
+    (agent._resolveProvider && agent._resolveProvider()) ||
+      cluster.config?.forceProvider ||
+      cluster.config?.defaultProvider ||
+      loadSettings().defaultProvider ||
+      'claude'
+  );
+
+  const isolationClusterId = `${cluster.id}-validators`;
+  const containerId = await manager.createContainer(isolationClusterId, {
+    workDir,
+    image,
+    noMounts: isolationConfig.noMounts,
+    mounts: isolationConfig.mounts,
+    containerHome: isolationConfig.containerHome,
+    provider: providerName,
+    reuseExistingWorkspace: true,
+  });
+
+  const validatorIsolation = {
+    enabled: true,
+    manager,
+    clusterId: isolationClusterId,
+    containerId,
+    image,
+    workDir,
+  };
+
+  cluster.validatorIsolation = validatorIsolation;
+  return validatorIsolation;
+}
+
+async function ensureValidatorIsolation(agent) {
+  const cluster = agent.cluster || {};
+
+  if (agent.isolation?.enabled) {
+    return agent.isolation;
+  }
+
+  if (cluster.validatorIsolation?.enabled) {
+    agent.isolation = cluster.validatorIsolation;
+    return agent.isolation;
+  }
+
+  if (cluster.validatorIsolationPromise) {
+    const isolation = await cluster.validatorIsolationPromise;
+    if (isolation?.enabled) {
+      agent.isolation = isolation;
+    }
+    return agent.isolation || null;
+  }
+
+  const isolationConfig = resolveValidatorIsolationConfig(agent);
+  if (!isolationConfig) {
+    agent._log(`[${agent.id}] Validator isolation config is not docker - skipping fallback`);
+    return null;
+  }
+
+  cluster.validatorIsolationPromise = createValidatorIsolation(agent, isolationConfig);
+
+  try {
+    const isolation = await cluster.validatorIsolationPromise;
+    if (isolation?.enabled) {
+      agent.isolation = isolation;
+      return agent.isolation;
+    }
+    return null;
+  } finally {
+    cluster.validatorIsolationPromise = null;
+  }
+}
+
+async function maybeRetryValidatorInDocker(agent, result) {
+  if (agent.role !== 'validator') return null;
+  if (agent.isolation?.enabled) return null;
+  if (agent._validatorIsolationAttemptedIteration === agent.iteration) {
+    return null;
+  }
+
+  const reason = findPlatformMismatchReason(result?.result || {});
+  if (!reason) return null;
+
+  const isolation = await ensureValidatorIsolation(agent);
+  if (!isolation) {
+    return null;
+  }
+
+  agent._validatorIsolationAttemptedIteration = agent.iteration;
+  agent._log(`[${agent.id}] Platform mismatch detected - retrying validator in Docker isolation`);
+  return reason;
+}
+
+/**
+ * Start the agent (begin listening for triggers)
+ * @param {AgentWrapper} agent - Agent instance
+ */
+function start(agent) {
+  if (agent.running) {
+    throw new Error(`Agent ${agent.id} is already running`);
+  }
+
+  agent.running = true;
+  agent.state = 'idle';
+
+  // Subscribe to all messages for this cluster
+  agent.unsubscribe = agent.messageBus.subscribe((message) => {
+    if (message.cluster_id === agent.cluster.id) {
+      handleMessage(agent, message).catch((error) => {
+        // FATAL: Message handling failed - crash loud
+        console.error(`\n${'='.repeat(80)}`);
+        console.error(`🔴 FATAL: Agent ${agent.id} message handler crashed`);
+        console.error(`${'='.repeat(80)}`);
+        console.error(`Topic: ${message.topic}`);
+        console.error(`Error: ${error.message}`);
+        console.error(`Stack: ${error.stack}`);
+        console.error(`${'='.repeat(80)}\n`);
+        // Re-throw to crash the process - DO NOT SILENTLY CONTINUE
+        throw error;
+      });
+    }
+  });
+
+  agent._log(`Agent ${agent.id} started (role: ${agent.role})`);
+  agent._publishLifecycle('STARTED', {
+    triggers: agent.config.triggers?.map((t) => t.topic) || [],
+  });
+}
+
+/**
+ * Stop the agent
+ * Waits for any in-flight execution to complete before returning.
+ * @param {AgentWrapper} agent - Agent instance
+ * @returns {Promise<void>}
+ */
+async function stop(agent) {
+  if (!agent.running) {
+    return;
+  }
+
+  agent.running = false;
+  agent.state = 'stopped';
+
+  if (agent.unsubscribe) {
+    agent.unsubscribe();
+    agent.unsubscribe = null;
+  }
+
+  // Kill current task if any
+  if (agent.currentTask) {
+    agent._killTask();
+  }
+
+  // Wait for in-flight execution to complete (up to 5 seconds)
+  // This prevents write-after-close race conditions
+  if (agent._currentExecution) {
+    try {
+      await Promise.race([
+        agent._currentExecution,
+        new Promise((resolve) => setTimeout(resolve, 5000)),
+      ]);
+    } catch {
+      // Ignore errors from cancelled execution
+    }
+    agent._currentExecution = null;
+  }
+
+  agent._log(`Agent ${agent.id} stopped`);
+}
+
+/**
+ * Handle incoming message
+ * @param {AgentWrapper} agent - Agent instance
+ * @param {Object} message - Incoming message
+ */
+async function handleMessage(agent, message) {
+  if (!agent._bufferedMessages) {
+    agent._bufferedMessages = [];
+  }
+
+  // Check if any trigger matches FIRST (before state check)
+  const matchingTrigger = findMatchingTrigger({
+    triggers: agent.config.triggers,
+    message,
+  });
+
+  if (!matchingTrigger) {
+    return; // No trigger for this message type
+  }
+
+  // Now check state - LOG if we're dropping a message we SHOULD handle
+  if (!agent.running) {
+    console.warn(`[${agent.id}] ⚠️ DROPPING message (not running): ${message.topic}`);
+    return;
+  }
+  if (agent.state !== 'idle') {
+    // IMPORTANT: Never drop a message that matches a trigger.
+    // Dropping validation/coordinator signals can wedge clusters in "running" state.
+    bufferMessage(agent, message);
+    console.warn(
+      `[${agent.id}] ⏸️ BUFFERING message (busy, state=${agent.state}): ${message.topic}`
+    );
+    scheduleDrain(
+      agent,
+      () => drainBufferedMessages(agent, (next) => handleMessage(agent, next), { label: 'Agent' }),
+      { label: 'Agent' }
+    );
+    return;
+  }
+
+  // Evaluate trigger logic
+  agent.state = 'evaluating_logic';
+
+  const agentContext = {
+    id: agent.id,
+    role: agent.role,
+    iteration: agent.iteration,
+    cluster_id: agent.cluster.id,
+  };
+
+  const shouldExecute = evaluateTrigger({
+    trigger: matchingTrigger,
+    message,
+    agent: agentContext,
+    logicEngine: agent.logicEngine,
+  });
+
+  if (!shouldExecute) {
+    agent.state = 'idle';
+    return;
+  }
+
+  // Execute trigger action (lifecycle event published inside for execute_task)
+  // Track execution so stop() can wait for it
+  const executionPromise = executeTriggerAction(agent, matchingTrigger, message);
+  agent._currentExecution = executionPromise;
+  try {
+    await executionPromise;
+  } finally {
+    // Clear only if this is still our execution (not replaced by another)
+    if (agent._currentExecution === executionPromise) {
+      agent._currentExecution = null;
+    }
+  }
+}
+
+/**
+ * Execute trigger action
+ * @param {AgentWrapper} agent - Agent instance
+ * @param {Object} trigger - Matched trigger config
+ * @param {Object} message - Triggering message
+ */
+async function executeTriggerAction(agent, trigger, message) {
+  const action = trigger.action || 'execute_task';
+
+  if (action === 'execute_task') {
+    await executeTask(agent, message);
+  } else if (action === 'stop_cluster') {
+    // Publish CLUSTER_COMPLETE message to signal successful completion
+    agent._publish({
+      topic: 'CLUSTER_COMPLETE',
+      receiver: 'system',
+      content: {
+        text: 'All validation passed. Cluster completing successfully.',
+        data: {
+          reason: 'all_validators_approved',
+          timestamp: Date.now(),
+        },
+      },
+    });
+    agent.state = 'completed';
+    agent._log(`Agent ${agent.id}: Cluster completion triggered`);
+  } else {
+    console.warn(`Unknown action: ${action}`);
+    agent.state = 'idle';
+  }
+}
+
+/**
+ * Execute task with built context
+ * Default: uses settings.maxRetries (default 3) for exponential backoff retries.
+ * Rate limit errors (429, capacity exhausted) use longer delays (30s base).
+ * Override via agent config `maxRetries` to change retry behavior.
+ * @param {AgentWrapper} agent - Agent instance
+ * @param {Object} triggeringMessage - Message that triggered execution
+ */
+function handleMaxIterations(agent) {
+  if (agent.iteration < agent.maxIterations) {
+    return false;
+  }
+
+  agent._log(`[Agent ${agent.id}] Hit max iterations (${agent.maxIterations}), stopping cluster`);
+  agent._publishLifecycle('MAX_ITERATIONS_REACHED', {
+    iteration: agent.iteration,
+    maxIterations: agent.maxIterations,
+  });
+  // Publish failure message - orchestrator watches for this and auto-stops
+  agent._publish({
+    topic: 'CLUSTER_FAILED',
+    receiver: 'system',
+    content: {
+      text: `Agent ${agent.id} hit max iterations limit (${agent.maxIterations}). Stopping cluster.`,
+      data: {
+        reason: 'max_iterations',
+        iteration: agent.iteration,
+        maxIterations: agent.maxIterations,
+      },
+    },
+  });
+  agent.state = 'failed';
+  return true;
+}
+
+function logInputContext(agent, context) {
+  if (agent.quiet) {
+    return;
+  }
+
+  console.log(`
+${'='.repeat(80)}`);
+  console.log(`📥 INPUT CONTEXT - Agent: ${agent.id} (Iteration: ${agent.iteration})`);
+  console.log(`${'='.repeat(80)}`);
+  console.log(context);
+  console.log(`${'='.repeat(80)}
+`);
+}
+
+async function applyValidatorJitter(agent) {
+  // LOCK CONTENTION FIX: Add random jitter for validators to prevent thundering herd
+  // When multiple validators wake on the same trigger (e.g., IMPLEMENTATION_READY),
+  // they all try to spawn Claude CLI at the same time. Claude CLI uses a lock file
+  // per workspace, so only one can run. Adding jitter staggers their starts.
+  // SKIP in testMode - tests use mocks and don't need jitter
+  if (agent.role !== 'validator' || agent.testMode) {
+    return;
+  }
+
+  const jitterMs = crypto.randomInt(0, 15000); // 0-15 seconds
+  if (!agent.quiet) {
+    agent._log(
+      `[Agent ${agent.id}] Adding ${Math.round(jitterMs / 1000)}s jitter to prevent lock contention`
+    );
+  }
+  await new Promise((resolve) => setTimeout(resolve, jitterMs));
+}
+
+function publishTaskStarted(agent, triggeringMessage) {
+  const modelSpec = agent._resolveModelSpec ? agent._resolveModelSpec() : null;
+  agent._publishLifecycle('TASK_STARTED', {
+    iteration: agent.iteration,
+    model: agent._selectModel(),
+    provider: agent._resolveProvider ? agent._resolveProvider() : 'claude',
+    modelSpec,
+    triggeredBy: triggeringMessage.topic,
+    triggerFrom: triggeringMessage.sender,
+  });
+}
+
+function attachResultMetadata(agent, result) {
+  // Add task ID to result for debugging and hooks
+  result.taskId = agent.currentTaskId;
+  result.agentId = agent.id;
+  result.iteration = agent.iteration;
+}
+
+function publishTaskCompleted(agent, result) {
+  agent._publishLifecycle('TASK_COMPLETED', {
+    iteration: agent.iteration,
+    success: true,
+    taskId: agent.currentTaskId,
+    tokenUsage: result.tokenUsage || null,
+  });
+}
+
+function publishTokenUsage(agent, result) {
+  // Publish TOKEN_USAGE event for aggregation and tracking
+  // CRITICAL: Include taskId for causal linking - allows consumers to group
+  // messages by task regardless of interleaved timing from async hooks
+  if (!result.tokenUsage) {
+    return;
+  }
+
+  agent.messageBus.publish({
+    cluster_id: agent.cluster.id,
+    topic: 'TOKEN_USAGE',
+    sender: agent.id,
+    content: {
+      text: `${agent.id} used ${result.tokenUsage.inputTokens} input + ${result.tokenUsage.outputTokens} output tokens`,
+      data: {
+        agentId: agent.id,
+        role: agent.role,
+        model: agent._selectModel(),
+        iteration: agent.iteration,
+        taskId: agent.currentTaskId, // Causal linking for message ordering
+        ...result.tokenUsage,
+      },
+    },
+  });
+}
+
+async function executeOnCompleteHookWithRetry(agent, triggeringMessage, result) {
+  // Execute onComplete hook WITH RETRY
+  // Hook failure shouldn't retry the entire task - just the hook
+  const hookMaxRetries = 3;
+  const hookBaseDelay = 1000;
+  let hookSuccess = false;
+
+  for (let hookAttempt = 1; hookAttempt <= hookMaxRetries && !hookSuccess; hookAttempt++) {
+    try {
+      await executeHook({
+        hook: agent.config.hooks?.onComplete,
+        agent: agent,
+        message: triggeringMessage,
+        result: result,
+        messageBus: agent.messageBus,
+        cluster: agent.cluster,
+        orchestrator: agent.orchestrator,
+      });
+      hookSuccess = true;
+    } catch (hookError) {
+      console.error(`
+${'='.repeat(80)}`);
+      console.error(
+        `🔴 HOOK EXECUTION FAILED - AGENT: ${agent.id} (Attempt ${hookAttempt}/${hookMaxRetries})`
+      );
+      console.error(`${'='.repeat(80)}`);
+      console.error(`Error: ${hookError.message}`);
+
+      if (hookAttempt < hookMaxRetries) {
+        const delay = hookBaseDelay * Math.pow(2, hookAttempt - 1);
+        console.error(`Will retry hook in ${delay}ms...`);
+        console.error(`${'='.repeat(80)}
+`);
+        await new Promise((resolve) => setTimeout(resolve, delay));
+      } else {
+        console.error(`${'='.repeat(80)}
+`);
+        // All hook retries exhausted - FAIL THE CLUSTER (do NOT rerun the whole task).
+        // Retrying the task wastes tokens and cannot fix a deterministic hook/config bug.
+        throw new HookExecutionError(
+          `Hook execution failed after ${hookMaxRetries} attempts. ` +
+            `Task completed successfully but hook could not publish result. ` +
+            `Original error: ${hookError.message}`,
+          { hookRetries: hookMaxRetries, originalHookError: hookError.message }
+        );
+      }
+    }
+  }
+}
+
+async function runTaskAttempt(agent, triggeringMessage) {
+  // Execute onStart hook
+  await executeHook({
+    hook: agent.config.hooks?.onStart,
+    agent: agent,
+    message: triggeringMessage,
+    result: undefined,
+    messageBus: agent.messageBus,
+    cluster: agent.cluster,
+    orchestrator: agent.orchestrator,
+  });
+
+  // Check max iterations limit BEFORE incrementing (prevents infinite rejection loops)
+  if (handleMaxIterations(agent)) {
+    return;
+  }
+
+  // Increment iteration BEFORE building context so worker knows current iteration
+  agent.iteration++;
+
+  // Build context
+  agent.state = 'building_context';
+  const context = agent._buildContext(triggeringMessage);
+
+  // Log input context (helps debug what each agent sees)
+  logInputContext(agent, context);
+
+  // Spawn provider task
+  agent.state = 'executing_task';
+  await applyValidatorJitter(agent);
+  publishTaskStarted(agent, triggeringMessage);
+
+  const result = await agent._spawnClaudeTask(context);
+  attachResultMetadata(agent, result);
+
+  // Check if task execution failed
+  if (!result.success) {
+    throw new Error(result.error || 'Task execution failed');
+  }
+
+  const fallbackReason = await maybeRetryValidatorInDocker(agent, result);
+  if (fallbackReason) {
+    throw new Error(
+      `Validator platform mismatch detected (${fallbackReason}). Retrying in Docker isolation.`
+    );
+  }
+
+  // Set state to idle BEFORE publishing lifecycle event
+  // (so lifecycle message includes correct state)
+  agent.state = 'idle';
+
+  // Track completion time for context filtering (used by "since: last_task_end")
+  agent.lastTaskEndTime = Date.now();
+
+  publishTaskCompleted(agent, result);
+  publishTokenUsage(agent, result);
+  await executeOnCompleteHookWithRetry(agent, triggeringMessage, result);
+}
+
+function logTaskAttemptFailure(agent, attempt, maxRetries, error) {
+  // Log attempt failure
+  console.error(`
+${'='.repeat(80)}`);
+  console.error(`🔴 TASK EXECUTION FAILED - AGENT: ${agent.id} (Attempt ${attempt}/${maxRetries})`);
+  console.error(`${'='.repeat(80)}`);
+  console.error(`Error: ${error.message}`);
+}
+
+async function handleLockContention() {
+  // Lock contention - add significant jittered delay
+  const lockDelay = 10000 + crypto.randomInt(0, 20000); // 10-30 seconds
+  console.error(
+    `⚠️ Lock contention detected - waiting ${Math.round(lockDelay / 1000)}s before retry`
+  );
+  await new Promise((resolve) => setTimeout(resolve, lockDelay));
+}
+
+async function handleFinalFailure(agent, triggeringMessage, error, maxRetries) {
+  console.error(`
+${'='.repeat(80)}`);
+  console.error(`🔴🔴🔴 MAX RETRIES EXHAUSTED - AGENT: ${agent.id} 🔴🔴🔴`);
+  console.error(`${'='.repeat(80)}`);
+  console.error(`All ${maxRetries} attempts failed`);
+  console.error(`Final error: ${error.message}`);
+  console.error(`Stack: ${error.stack}`);
+  console.error(`${'='.repeat(80)}
+`);
+
+  // CRITICAL FIX: Validator crash = REJECTION (not auto-approval)
+  // Auto-approval on crash allowed broken code to be merged - unacceptable!
+  // If validator crashed 3x, something is fundamentally wrong - REJECT and investigate
+  if (agent.role === 'validator') {
+    console.error(`
+${'='.repeat(80)}`);
+    console.error(`❌ VALIDATOR CRASHED - REJECTING (NOT AUTO-APPROVING)`);
+    console.error(`${'='.repeat(80)}`);
+    console.error(`Validator ${agent.id} crashed ${maxRetries} times`);
+    console.error(`Error: ${error.message}`);
+    console.error(`REJECTING validation - broken code will NOT be merged`);
+    console.error(`Investigation required before retry`);
+    console.error(`${'='.repeat(80)}
+`);
+
+    // Publish REJECTION message (NOT approval!)
+    const hook = agent.config.hooks?.onComplete;
+    if (hook && hook.action === 'publish_message') {
+      agent._publish({
+        topic: hook.config.topic,
+        receiver: hook.config.receiver || 'broadcast',
+        content: {
+          text: `REJECTED: Validator crashed ${maxRetries} times - ${error.message}`,
+          data: {
+            approved: false, // REJECT!
+            crashedAfterRetries: true,
+            errors: JSON.stringify([
+              `VALIDATOR CRASHED ${maxRetries}x: ${error.message}`,
+              `Validation could not be performed - REJECTING to prevent broken code merge`,
+              `Investigation required before retry`,
+            ]),
+            attempts: maxRetries,
+            requiresInvestigation: true,
+          },
+        },
+      });
+    }
+
+    agent.state = 'error';
+    // Don't return - fall through to publish AGENT_ERROR and save failure info
+    // This allows the cluster to stop and be resumed after investigation
+  }
+
+  // Non-validator agents: publish error and stop
+  agent.state = 'error';
+
+  // Hook failure: fail the whole cluster so it gets stopped + persisted (prevents deadlocked "running" clusters).
+  if (error?.hookFailure) {
+    agent._publish({
+      topic: 'CLUSTER_FAILED',
+      receiver: 'broadcast',
+      content: {
+        text: `Cluster failed: onComplete hook failed for ${agent.id} - ${error.message}`,
+        data: {
+          reason: 'on_complete_hook_failed',
+          agentId: agent.id,
+          role: agent.role,
+          hookRetries: error.hookRetries ?? null,
+          originalHookError: error.originalHookError ?? null,
+          error: error.message,
+        },
+      },
+    });
+  }
+
+  // Save failure info to cluster for resume capability
+  agent.cluster.failureInfo = {
+    agentId: agent.id,
+    taskId: agent.currentTaskId,
+    iteration: agent.iteration,
+    error: error.message,
+    attempts: maxRetries,
+    timestamp: Date.now(),
+  };
+
+  // Publish error to message bus for visibility in logs
+  agent._publish({
+    topic: 'AGENT_ERROR',
+    receiver: 'broadcast',
+    content: {
+      text: `Task execution failed after ${maxRetries} attempts: ${error.message}`,
+      data: {
+        error: error.message,
+        stack: error.stack,
+        hookFailure: error?.hookFailure === true,
+        hookRetries: error?.hookRetries ?? undefined,
+        originalHookError: error?.originalHookError ?? undefined,
+        agent: agent.id,
+        role: agent.role,
+        iteration: agent.iteration,
+        taskId: agent.currentTaskId,
+        attempts: maxRetries,
+        hookFailureContext: error.message.includes('Hook uses result')
+          ? {
+              taskId: agent.currentTaskId || 'UNKNOWN',
+              retrieveLogs: agent.currentTaskId
+                ? `zeroshot task logs ${agent.currentTaskId}`
+                : 'N/A',
+            }
+          : undefined,
+      },
+    },
+    metadata: {
+      triggeringTopic: triggeringMessage.topic,
+    },
+  });
+
+  // Execute onError hook
+  await executeHook({
+    hook: agent.config.hooks?.onError,
+    agent: agent,
+    message: triggeringMessage,
+    result: { error },
+    messageBus: agent.messageBus,
+    cluster: agent.cluster,
+    orchestrator: agent.orchestrator,
+  });
+
+  agent.state = 'idle';
+}
+
+async function scheduleRetry(agent, error, attempt, maxRetries, _baseDelay) {
+  // Use rate-limit-aware backoff (30s+ for 429s, 2s for others)
+  const settings = loadSettings();
+  const delay = calculateRateLimitDelay(error, attempt, settings);
+  const isRateLimit = isRateLimitError(error);
+
+  agent._publishLifecycle('RETRY_SCHEDULED', {
+    attempt,
+    maxRetries,
+    delayMs: delay,
+    error: error.message,
+    isRateLimitError: isRateLimit,
+  });
+
+  const prefix = isRateLimit ? '🔄 Rate limit - ' : '⚠️  ';
+  agent._log(
+    `[${agent.id}] ${prefix}Retrying in ${Math.round(delay / 1000)}s... (${attempt + 1}/${maxRetries})`
+  );
+
+  // Rate-limit-aware backoff
+  await new Promise((resolve) => setTimeout(resolve, delay));
+
+  agent._log(`[${agent.id}] 🔄 Starting retry attempt ${attempt + 1}/${maxRetries}`);
+}
+
+async function handleTaskAttemptFailure({
+  agent,
+  triggeringMessage,
+  error,
+  attempt,
+  maxRetries,
+  baseDelay,
+}) {
+  // LOCK CONTENTION: Add extra jittered delay for lock file errors
+  // This happens when multiple validators try to run Claude CLI in the same workspace
+  const isLockError = error.message && error.message.includes('Lock file');
+
+  logTaskAttemptFailure(agent, attempt, maxRetries, error);
+
+  if (isLockError) {
+    await handleLockContention();
+  } else if (attempt < maxRetries) {
+    console.error(`Will retry in ${baseDelay * Math.pow(2, attempt - 1)}ms...`);
+  }
+  console.error(`${'='.repeat(80)}
+`);
+
+  if (attempt >= maxRetries) {
+    await handleFinalFailure(agent, triggeringMessage, error, maxRetries);
+    return true;
+  }
+
+  await scheduleRetry(agent, error, attempt, maxRetries, baseDelay);
+  return false;
+}
+
+function maybeExtendMaxRetries({
+  error,
+  attempt,
+  maxRetries,
+  sigtermRetryGranted,
+  noMessagesRetryGranted,
+}) {
+  const message = error?.message || '';
+  if (!message || attempt < maxRetries) {
+    return { maxRetries, sigtermRetryGranted, noMessagesRetryGranted };
+  }
+
+  if (message.includes('SIGTERM') && !sigtermRetryGranted) {
+    return { maxRetries: maxRetries + 1, sigtermRetryGranted: true, noMessagesRetryGranted };
+  }
+
+  if (message.toLowerCase().includes('no messages returned') && !noMessagesRetryGranted) {
+    return { maxRetries: maxRetries + 1, sigtermRetryGranted, noMessagesRetryGranted: true };
+  }
+
+  return { maxRetries, sigtermRetryGranted, noMessagesRetryGranted };
+}
+
+/**
+ * Execute claude-zeroshots with built context
+ * Default: uses settings.maxRetries (default 3) for exponential backoff retries.
+ * Override via agent config `maxRetries` to change retry behavior.
+ * @param {AgentWrapper} agent - Agent instance
+ * @param {Object} triggeringMessage - Message that triggered execution
+ */
+async function executeTask(agent, triggeringMessage) {
+  // Early exit if agent was stopped
+  if (!agent.running) {
+    return;
+  }
+
+  // Default: uses settings.maxRetries (default 3)
+  // Override via agent config `maxRetries` to change retry behavior
+  const settings = loadSettings();
+  let maxRetries = agent.config.maxRetries ?? settings.maxRetries ?? 3;
+  const baseDelay = settings.backoffBaseMs ?? 2000;
+  let sigtermRetryGranted = false;
+  let noMessagesRetryGranted = false;
+
+  for (let attempt = 1; attempt <= maxRetries; attempt++) {
+    // Check if agent was stopped between retries
+    if (!agent.running) {
+      return;
+    }
+
+    try {
+      await runTaskAttempt(agent, triggeringMessage);
+      return;
+    } catch (error) {
+      if (!agent.running || agent.state === 'stopped') {
+        agent._log(`[${agent.id}] Task interrupted during shutdown; skipping retry`);
+        return;
+      }
+      if (error?.permanent) {
+        await handleFinalFailure(agent, triggeringMessage, error, attempt);
+        return;
+      }
+      if (error instanceof HookExecutionError) {
+        // Hook failures are deterministic; do not waste tokens retrying the provider task.
+        await handleFinalFailure(agent, triggeringMessage, error, 1);
+        return;
+      }
+      const updated = maybeExtendMaxRetries({
+        error,
+        attempt,
+        maxRetries,
+        sigtermRetryGranted,
+        noMessagesRetryGranted,
+      });
+      maxRetries = updated.maxRetries;
+      sigtermRetryGranted = updated.sigtermRetryGranted;
+      noMessagesRetryGranted = updated.noMessagesRetryGranted;
+      const shouldStop = await handleTaskAttemptFailure({
+        agent,
+        triggeringMessage,
+        error,
+        attempt,
+        maxRetries,
+        baseDelay,
+      });
+      if (shouldStop) {
+        return;
+      }
+    }
+  }
+}
+function startLivenessCheck(agent) {
+  if (agent.livenessCheckInterval) {
+    clearInterval(agent.livenessCheckInterval);
+  }
+
+  // Check if platform supports /proc filesystem (Linux only)
+  if (!isPlatformSupported()) {
+    agent._log(
+      `[${agent.id}] Liveness check disabled: /proc filesystem not available (non-Linux platform)`
+    );
+    return;
+  }
+
+  // Check every 60 seconds (gives time for multi-indicator analysis)
+  const CHECK_INTERVAL_MS = 60 * 1000;
+  const ANALYSIS_SAMPLE_MS = 5000; // Sample CPU/context switches over 5 seconds
+
+  agent.livenessCheckInterval = setInterval(async () => {
+    // Skip if no task running or no PID tracked
+    if (!agent.currentTask || !agent.processPid) {
+      return;
+    }
+
+    // Skip if output is recent (process is clearly active)
+    if (agent.lastOutputTime) {
+      const timeSinceLastOutput = Date.now() - agent.lastOutputTime;
+      if (timeSinceLastOutput < agent.staleDuration) {
+        return; // Output is recent, definitely not stuck
+      }
+    }
+
+    // Output is stale - run multi-indicator analysis to confirm
+    agent._log(
+      `[${agent.id}] Output stale for ${Math.round((Date.now() - (agent.lastOutputTime || 0)) / 1000)}s, running multi-indicator analysis...`
+    );
+
+    try {
+      const analysis = await analyzeProcessHealth(agent.processPid, ANALYSIS_SAMPLE_MS);
+
+      // Process died during analysis
+      if (analysis.isLikelyStuck === null) {
+        agent._log(`[${agent.id}] Process analysis inconclusive: ${analysis.reason}`);
+        return;
+      }
+
+      // Log analysis details for debugging
+      agent._log(
+        `[${agent.id}] Analysis: score=${analysis.stuckScore}/${STUCK_THRESHOLD}, ` +
+          `state=${analysis.state}, wchan=${analysis.wchan}, ` +
+          `CPU=${analysis.cpuPercent}%, ctxSwitches=${analysis.ctxSwitchesDelta}`
+      );
+
+      if (analysis.isLikelyStuck) {
+        agent._log(`⚠️  Agent ${agent.id}: CONFIRMED STUCK (confidence: ${analysis.confidence})`);
+        agent._log(`    ${analysis.analysis}`);
+
+        // CHANGED: Stale detection is informational only - never kills tasks
+        // Publish stale detection event with full analysis (for logging/monitoring)
+        agent._publishLifecycle('AGENT_STALE_WARNING', {
+          timeSinceLastOutput: Date.now() - (agent.lastOutputTime || 0),
+          staleDuration: agent.staleDuration,
+          lastOutputTime: agent.lastOutputTime,
+          // Multi-indicator analysis results
+          stuckScore: analysis.stuckScore,
+          confidence: analysis.confidence,
+          processState: analysis.state,
+          wchan: analysis.wchan,
+          cpuPercent: analysis.cpuPercent,
+          ctxSwitchesDelta: analysis.ctxSwitchesDelta,
+          indicators: analysis.indicators,
+          analysis: analysis.analysis,
+        });
+
+        // Keep monitoring - do NOT stop the agent
+        // User can manually intervene with 'zeroshot resume' if needed
+        // stopLivenessCheck(agent); // REMOVED - keep monitoring
+      } else {
+        agent._log(
+          `[${agent.id}] Process appears WORKING despite stale output (score: ${analysis.stuckScore})`
+        );
+        agent._log(`    ${analysis.analysis}`);
+        // Don't flag as stuck - process is legitimately working
+      }
+    } catch (err) {
+      agent._log(`[${agent.id}] Error during stuck analysis: ${err.message}`);
+      // Don't flag as stuck on analysis error
+    }
+  }, CHECK_INTERVAL_MS);
+}
+
+/**
+ * Stop liveness monitoring
+ * @param {AgentWrapper} agent - Agent instance
+ */
+function stopLivenessCheck(agent) {
+  if (agent.livenessCheckInterval) {
+    clearInterval(agent.livenessCheckInterval);
+    agent.livenessCheckInterval = null;
+  }
+}
+
+module.exports = {
+  start,
+  stop,
+  handleMessage,
+  executeTriggerAction,
+  executeTask,
+  startLivenessCheck,
+  stopLivenessCheck,
+};