@archal/cli 0.9.0 → 0.9.5

package/harnesses/_lib/harness-runner.mjs

@@ -1,354 +0,0 @@
-/**
- * Shared harness scaffolding for bundled agent files.
- *
- * Extracts the common init sequence and run-loop structure that all 4
- * bundled harnesses (naive, zero-shot, hardened, react) duplicate.
- *
- * Usage:
- *   const ctx = await createHarnessContext('react');
- *   await runAgentLoop(ctx, { ... });
- */
-import { collectTwinUrls, discoverAllTools } from './rest-client.mjs';
-import {
-  detectProvider,
-  resolveApiKey,
-  formatToolsForProvider,
-  buildInitialMessages,
-  appendAssistantResponse,
-  appendToolResults,
-  appendUserInstruction,
-  callLlmWithMessages,
-  parseToolCalls,
-  getResponseText,
-  getThinkingContent,
-  getStopReason,
-  withRetry,
-} from './providers.mjs';
-import { createLogger } from './logging.mjs';
-import { writeMetrics } from './metrics.mjs';
-import { createAgentTrace } from './agent-trace.mjs';
-
-// ── Context creation ──────────────────────────────────────────────────
-
-/**
- * @typedef {object} HarnessContext
- * @property {string} harnessName
- * @property {string} task
- * @property {string} model
- * @property {string} provider
- * @property {string} apiKey
- * @property {import('./logging.mjs').Logger} log
- * @property {Record<string, string>} twinUrls
- * @property {Array<{ name: string, description: string, inputSchema: object }>} allTools
- * @property {Record<string, { twinName: string, baseUrl: string, originalName: string }>} toolToTwin
- */
-
-/**
- * Create the full harness context: validate env vars, detect provider,
- * resolve API key, collect twin URLs, and discover tools.
- *
- * Exits with code 1 on missing env vars or unreachable twins.
- *
- * @param {string} harnessName
- * @returns {Promise<HarnessContext>}
- */
-export async function createHarnessContext(harnessName) {
-  const task = (process.env['ARCHAL_ENGINE_TASK'] || '').trim();
-  const model = process.env['ARCHAL_ENGINE_MODEL'];
-
-  if (!task) { console.error('ARCHAL_ENGINE_TASK not set or empty'); process.exit(1); }
-  if (!model) { console.error('ARCHAL_ENGINE_MODEL not set'); process.exit(1); }
-
-  const provider = detectProvider(model);
-  const apiKey = resolveApiKey(provider);
-  const log = createLogger({ harness: harnessName, model, provider });
-
-  const twinUrls = collectTwinUrls();
-  if (Object.keys(twinUrls).length === 0) {
-    console.error(`[${harnessName}] No twin URLs found. Check ARCHAL_TWIN_NAMES and ARCHAL_<TWIN>_URL env vars.`);
-    process.exit(1);
-  }
-
-  const { tools: allTools, toolToTwin } = await discoverAllTools(twinUrls);
-  if (allTools.length === 0) {
-    console.error(`[${harnessName}] No tools discovered from twins. Twin endpoints may be unreachable.`);
-    process.exit(1);
-  }
-
-  return { harnessName, task, model, provider, apiKey, log, twinUrls, allTools, toolToTwin };
-}
-
-// ── Run loop ──────────────────────────────────────────────────────────
-
-/**
- * @typedef {object} RunLoopOptions
- * @property {string} systemPrompt - System prompt text (empty string for none)
- * @property {number} maxSteps - Maximum iteration count
- * @property {boolean} [useRetry=false] - Wrap LLM calls in withRetry
- * @property {number} [retryCount=4] - Max retries when useRetry is true
- * @property {boolean} [useTrace=false] - Record agent trace
- * @property {number} [maxConsecutiveErrors=0] - Bail threshold (0 = no limit)
- * @property {number} [maxInitialNoToolRecoveries=0] - Reprompt attempts when model doesn't call tools initially
- * @property {(ctx: HarnessContext, state: RunState) => Array} [selectTools] -
- *   Per-step tool selection function. Receives context and current state,
- *   returns the MCP tools array for this step. Default: use all tools.
- * @property {(ctx: HarnessContext, state: RunState, stepResult: StepResult) => 'continue' | 'break' | void} [onBeforeToolExecution] -
- *   Hook called after parsing tool calls but before executing them.
- *   Return 'continue' to skip tool execution and loop, 'break' to stop.
- * @property {(provider: string, messages: Array|object) => Array|object} [initMessages] -
- *   Optional post-init hook to modify the initial messages array before the
- *   run loop starts (e.g. to prepend a triage instruction).
- * @property {(ctx: HarnessContext, state: RunState, stepResult: StepResult) => void} [onAfterToolExecution] -
- *   Hook called after tool results are appended. Return value is ignored.
- * @property {(ctx: HarnessContext, state: RunState, stepResult: StepResult) => 'continue' | void} [onNoToolCalls] -
- *   Hook called when the model responds without tool calls. Return
- *   'continue' to add instructions and continue the loop.
- * @property {(tc: { name: string, arguments: object }) => void} [onToolSuccess] -
- *   Called after each successful tool call.
- */
-
-/**
- * @typedef {object} RunState
- * Mutable state tracked across loop iterations.
- * @property {Array|object} messages
- * @property {number} stepsCompleted
- * @property {number} totalInputTokens
- * @property {number} totalOutputTokens
- * @property {number} totalToolCalls
- * @property {number} totalToolErrors
- * @property {number} consecutiveErrors
- * @property {number} initialNoToolRecoveries
- * @property {string} exitReason
- * @property {import('./agent-trace.mjs').ReturnType<typeof createAgentTrace>|null} agentTrace
- */
-
-/**
- * @typedef {object} StepResult
- * @property {number} step - 1-indexed step number
- * @property {object} response - Raw LLM response wrapper
- * @property {Array|null} toolCalls - Parsed tool calls or null
- * @property {string|null} thinking - Model thinking content
- * @property {string|null} text - Model text content
- * @property {number} iterDurationMs
- * @property {string|null} stopReason
- */
-
-/**
- * Run the agent loop with shared metrics, logging, and tool execution.
- *
- * @param {HarnessContext} ctx
- * @param {RunLoopOptions} opts
- */
-export async function runAgentLoop(ctx, opts) {
-  const {
-    systemPrompt,
-    maxSteps,
-    useRetry = false,
-    retryCount = 4,
-    useTrace = false,
-    maxConsecutiveErrors = 0,
-    maxInitialNoToolRecoveries = 0,
-    selectTools,
-    onBeforeToolExecution,
-    onAfterToolExecution,
-    onNoToolCalls,
-    onToolSuccess,
-  } = opts;
-
-  const { harnessName, task, model, provider, apiKey, log, allTools, toolToTwin } = ctx;
-
-  let messages = buildInitialMessages(provider, systemPrompt, task, model);
-
-  // Allow callers to modify initial messages (e.g. react's triage instruction)
-  if (opts.initMessages) {
-    messages = opts.initMessages(provider, messages);
-  }
-
-  const state = {
-    messages,
-    stepsCompleted: 0,
-    totalInputTokens: 0,
-    totalOutputTokens: 0,
-    totalToolCalls: 0,
-    totalToolErrors: 0,
-    consecutiveErrors: 0,
-    initialNoToolRecoveries: 0,
-    exitReason: 'max_steps',
-    agentTrace: useTrace ? createAgentTrace() : null,
-  };
-
-  const runStart = Date.now();
-
-  log.info('run_start', { task: task.slice(0, 200), maxSteps });
-
-  try {
-    for (let step = 0; step < maxSteps; step++) {
-      state.stepsCompleted = step + 1;
-      const iterStart = Date.now();
-
-      // Select tools for this step (default: all tools)
-      const stepTools = selectTools ? selectTools(ctx, state) : allTools;
-      const providerTools = formatToolsForProvider(provider, stepTools);
-
-      // Call the LLM (optionally with retry)
-      log.llmCall(step + 1);
-      let response;
-      try {
-        const llmCall = () => callLlmWithMessages(provider, model, apiKey, state.messages, providerTools);
-        response = useRetry ? await withRetry(llmCall, retryCount) : await llmCall();
-      } catch (err) {
-        const msg = err?.message ?? String(err);
-        log.error('llm_call_failed', { step: step + 1, error: msg });
-        process.stderr.write(`[${harnessName}] LLM API error: ${msg.slice(0, 500)}\n`);
-        state.exitReason = 'llm_error';
-        break;
-      }
-
-      const iterDurationMs = Date.now() - iterStart;
-      state.totalInputTokens += response.usage.inputTokens;
-      state.totalOutputTokens += response.usage.outputTokens;
-
-      const toolCalls = parseToolCalls(provider, response);
-      const hasToolCalls = !!toolCalls;
-      const stopReason = getStopReason(provider, response);
-      log.llmResponse(step + 1, iterDurationMs, hasToolCalls, stopReason);
-      log.tokenUsage(step + 1, response.usage, {
-        inputTokens: state.totalInputTokens,
-        outputTokens: state.totalOutputTokens,
-      });
-
-      const thinking = getThinkingContent(provider, response);
-      const text = getResponseText(provider, response);
-
-      state.messages = appendAssistantResponse(provider, state.messages, response);
-
-      /** @type {StepResult} */
-      const stepResult = { step: step + 1, response, toolCalls, thinking, text, iterDurationMs, stopReason };
-
-      if (!toolCalls) {
-        // Record trace for no-tool-call steps
-        if (state.agentTrace) {
-          state.agentTrace.addStep({ step: step + 1, thinking, text, toolCalls: [], durationMs: iterDurationMs });
-        }
-        if (text) {
-          process.stderr.write(`[${harnessName}] Step ${step + 1}: ${text.slice(0, 200)}\n`);
-        }
-
-        // Initial no-tool recovery (reprompt)
-        const shouldRecoverInitial = state.totalToolCalls === 0
-          && maxInitialNoToolRecoveries > 0
-          && state.initialNoToolRecoveries < maxInitialNoToolRecoveries;
-        if (shouldRecoverInitial) {
-          state.initialNoToolRecoveries++;
-          state.messages = appendUserInstruction(
-            provider,
-            state.messages,
-            'You must use tools to make progress. ' +
-              'On your next response, call at least one relevant tool before giving any summary or conclusion. ' +
-              'Start by gathering concrete evidence from the systems, then execute the required actions.',
-          );
-          log.info('no_tool_calls_reprompt', {
-            step: step + 1,
-            attempt: state.initialNoToolRecoveries,
-          });
-          continue;
-        }
-
-        // Harness-specific no-tool-call handling
-        if (onNoToolCalls) {
-          const directive = onNoToolCalls(ctx, state, stepResult);
-          if (directive === 'continue') continue;
-        }
-
-        state.exitReason = state.totalToolCalls === 0 ? 'no_tool_calls' : 'completed';
-        break;
-      }
-
-      state.initialNoToolRecoveries = 0;
-
-      // Pre-execution hook (e.g. react's repo content guard)
-      if (onBeforeToolExecution) {
-        const directive = onBeforeToolExecution(ctx, state, stepResult);
-        if (directive === 'continue') continue;
-        if (directive === 'break') break;
-      }
-
-      // Execute tool calls
-      const { executeToolCalls } = await import('./tool-executor.mjs');
-      const { results, bailout } = await executeToolCalls(toolCalls, {
-        toolToTwin,
-        harnessName,
-        step: step + 1,
-        log,
-        counters: state,
-        maxConsecutiveErrors,
-        onSuccess: onToolSuccess,
-      });
-
-      // Record trace
-      if (state.agentTrace) {
-        state.agentTrace.addStep({
-          step: step + 1,
-          thinking,
-          text,
-          toolCalls: toolCalls.map((tc) => ({ name: tc.name, arguments: tc.arguments })),
-          durationMs: iterDurationMs,
-        });
-      }
-
-      if (bailout) {
-        state.exitReason = 'consecutive_errors';
-        break;
-      }
-
-      // Append tool results to conversation
-      state.messages = appendToolResults(provider, state.messages, toolCalls, results);
-
-      // Post-execution hook
-      if (onAfterToolExecution) {
-        onAfterToolExecution(ctx, state, stepResult);
-      }
-    }
-  } finally {
-    const totalTimeMs = Date.now() - runStart;
-
-    log.summary({
-      iterations: state.stepsCompleted,
-      totalInputTokens: state.totalInputTokens,
-      totalOutputTokens: state.totalOutputTokens,
-      totalTimeMs,
-      toolCallCount: state.totalToolCalls,
-      toolErrorCount: state.totalToolErrors,
-      exitReason: state.exitReason,
-    });
-
-    writeMetrics({
-      inputTokens: state.totalInputTokens,
-      outputTokens: state.totalOutputTokens,
-      llmCallCount: state.stepsCompleted,
-      toolCallCount: state.totalToolCalls,
-      toolErrorCount: state.totalToolErrors,
-      totalTimeMs,
-      exitReason: state.exitReason,
-      provider,
-      model,
-    });
-
-    if (state.agentTrace) {
-      state.agentTrace.flush();
-    }
-
-    process.stderr.write(
-      `\n[${harnessName}] Summary: ${state.stepsCompleted} iterations, ${state.totalToolCalls} tool calls ` +
-      `(${state.totalToolErrors} errors), ${state.totalInputTokens} input tokens, ` +
-      `${state.totalOutputTokens} output tokens, ${(totalTimeMs / 1000).toFixed(1)}s total\n`
-    );
-
-    if (state.exitReason === 'llm_error') {
-      process.exit(1);
-    }
-  }
-}
-
-// Re-export for convenience — harnesses that need to build custom initial messages
-export { appendUserInstruction };