@archal/cli 0.7.12 → 0.9.0

package/README.md

@@ -20,19 +20,22 @@ archal init
 archal run scenario.md \
   --engine-endpoint "https://gateway.openclaw.ai/v1/responses" \
   --engine-token "$OPENCLAW_GATEWAY_TOKEN" \
-  --engine-model "openclaw:main"
+  --agent-model "openclaw:main"
 ```
 
 ## Commands
 
 | Command | Description |
 | --- | --- |
-| `archal init` | Scaffold a new archal project with example scenarios |
 | `archal run <scenario>` | Execute a scenario against hosted twins and evaluate results |
-| `archal login` | Authenticate via browser auth (hosted endpoint by default) |
+| `archal demo --model <model>` | Compare bundled harnesses on a scenario |
+| `archal batch run <suite>` | Run a configured suite as a hosted batch |
+| `archal batch compare <suite>` | Compare multiple named batch variants on one suite |
+| `archal batch list` | List suites and batch definitions from `.archal.json` |
+| `archal init` | Scaffold a new Archal project with example scenarios |
+| `archal login` | Authenticate via browser auth or token |
 | `archal whoami` | Show current auth/account status |
-| `archal twins` | Manage selected twins for your plan |
-| `archal trace` | Inspect run traces |
+| `archal logout` | Remove stored credentials |
 | `archal config` | View or update CLI configuration |
 
 ## Environment Variables
@@ -50,10 +53,10 @@ archal run scenario.md \
 | `ARCHAL_ENGINE_TIMEOUT` | Default API engine timeout in seconds |
 | `ARCHAL_ENGINE_TWIN_URLS` | Default path to remote twin URL overrides |
 | `ARCHAL_HARNESS_DIR` | Default harness directory for local mode |
-| `OPENCLAW_URL` | OpenClaw gateway URL (fallback alias for `--openclaw-url`) |
-| `OPENCLAW_GATEWAY_TOKEN` | OpenClaw gateway token used by `archal run` |
-| `OPENCLAW_GATEWAY_PASSWORD` | Password-mode OpenClaw gateway auth value |
-| `OPENCLAW_AGENT_ID` | Default OpenClaw agent/model id |
+| `OPENCLAW_URL` | Legacy OpenClaw gateway URL alias for API mode |
+| `OPENCLAW_GATEWAY_TOKEN` | Legacy OpenClaw gateway token |
+| `OPENCLAW_GATEWAY_PASSWORD` | Legacy password-mode OpenClaw gateway auth value |
+| `OPENCLAW_AGENT_ID` | Legacy OpenClaw agent/model id (prefer `ARCHAL_ENGINE_MODEL`) |
 
 ## Documentation
 

package/bin/archal.cjs

@@ -0,0 +1,15 @@
+#!/usr/bin/env node
+
+const { existsSync } = require("node:fs");
+const { join } = require("node:path");
+
+const distEntry = join(__dirname, "..", "dist", "index.cjs");
+
+if (!existsSync(distEntry)) {
+  console.error(
+    "The @archal/cli bundle has not been built yet. Run `pnpm --filter @archal/cli run build` first.",
+  );
+  process.exit(1);
+}
+
+require(distEntry);

package/dist/harnesses/_lib/agent-trace.mjs

@@ -0,0 +1,57 @@
+/**
+ * Structured agent trace writer for bundled harnesses.
+ *
+ * Records per-step model thinking, text output, and tool calls as a structured
+ * JSON trace. The orchestrator reads this file after the harness exits and flows
+ * it into RunResult → artifacts → dashboard.
+ *
+ * Transport: writes to ARCHAL_AGENT_TRACE_FILE (set by orchestrator).
+ * Safe no-op when the env var is not set.
+ *
+ * Trace format:
+ *   { version: 1, steps: [ { step, thinking, text, toolCalls, durationMs } ] }
+ */
+import { writeFileSync } from 'node:fs';
+
+/**
+ * @typedef {Object} TraceStep
+ * @property {number} step           - 1-indexed step number
+ * @property {string|null} thinking  - Model's internal reasoning (extended thinking / reasoning_content)
+ * @property {string|null} text      - Model's visible text output (reasoning "out loud")
+ * @property {Array<{name: string, arguments: object}>} toolCalls - Tools called this step
+ * @property {number} durationMs     - LLM call duration for this step
+ */
+
+/**
+ * Create a trace collector that accumulates steps and writes on flush.
+ * @returns {{ addStep: (step: TraceStep) => void, flush: () => void }}
+ */
+export function createAgentTrace() {
+  /** @type {TraceStep[]} */
+  const steps = [];
+
+  return {
+    /**
+     * Record a single agent step.
+     * @param {TraceStep} step
+     */
+    addStep(step) {
+      steps.push(step);
+    },
+
+    /**
+     * Write the accumulated trace to the file. Call once at the end.
+     */
+    flush() {
+      const tracePath = process.env['ARCHAL_AGENT_TRACE_FILE'];
+      if (!tracePath) return;
+
+      try {
+        const payload = { version: 1, steps };
+        writeFileSync(tracePath, JSON.stringify(payload));
+      } catch {
+        // Non-fatal — trace is best-effort
+      }
+    },
+  };
+}

package/dist/harnesses/_lib/env-utils.mjs

@@ -0,0 +1,23 @@
+/**
+ * Shared environment variable parsing utilities for bundled harnesses.
+ */
+
+/**
+ * Parse an integer from an environment variable with validation and clamping.
+ * Replaces the repeated IIFE pattern across agent files.
+ *
+ * @param {string} envVar - Environment variable name
+ * @param {number} defaultValue - Default if env var is not set or invalid
+ * @param {{ min?: number, max?: number }} [opts] - Optional min/max bounds
+ * @returns {number}
+ */
+export function parseEnvInt(envVar, defaultValue, { min, max } = {}) {
+  const raw = process.env[envVar]?.trim();
+  if (!raw) return defaultValue;
+  const parsed = parseInt(raw, 10);
+  if (Number.isNaN(parsed)) return defaultValue;
+  let value = parsed;
+  if (min !== undefined && value < min) value = min;
+  if (max !== undefined && value > max) value = max;
+  return value;
+}

package/dist/harnesses/_lib/harness-runner.mjs

@@ -0,0 +1,354 @@
+/**
+ * Shared harness scaffolding for bundled agent files.
+ *
+ * Extracts the common init sequence and run-loop structure that all 4
+ * bundled harnesses (naive, zero-shot, hardened, react) duplicate.
+ *
+ * Usage:
+ *   const ctx = await createHarnessContext('react');
+ *   await runAgentLoop(ctx, { ... });
+ */
+import { collectTwinUrls, discoverAllTools } from './rest-client.mjs';
+import {
+  detectProvider,
+  resolveApiKey,
+  formatToolsForProvider,
+  buildInitialMessages,
+  appendAssistantResponse,
+  appendToolResults,
+  appendUserInstruction,
+  callLlmWithMessages,
+  parseToolCalls,
+  getResponseText,
+  getThinkingContent,
+  getStopReason,
+  withRetry,
+} from './providers.mjs';
+import { createLogger } from './logging.mjs';
+import { writeMetrics } from './metrics.mjs';
+import { createAgentTrace } from './agent-trace.mjs';
+
+// ── Context creation ──────────────────────────────────────────────────
+
+/**
+ * @typedef {object} HarnessContext
+ * @property {string} harnessName
+ * @property {string} task
+ * @property {string} model
+ * @property {string} provider
+ * @property {string} apiKey
+ * @property {import('./logging.mjs').Logger} log
+ * @property {Record<string, string>} twinUrls
+ * @property {Array<{ name: string, description: string, inputSchema: object }>} allTools
+ * @property {Record<string, { twinName: string, baseUrl: string, originalName: string }>} toolToTwin
+ */
+
+/**
+ * Create the full harness context: validate env vars, detect provider,
+ * resolve API key, collect twin URLs, and discover tools.
+ *
+ * Exits with code 1 on missing env vars or unreachable twins.
+ *
+ * @param {string} harnessName
+ * @returns {Promise<HarnessContext>}
+ */
+export async function createHarnessContext(harnessName) {
+  const task = (process.env['ARCHAL_ENGINE_TASK'] || '').trim();
+  const model = process.env['ARCHAL_ENGINE_MODEL'];
+
+  if (!task) { console.error('ARCHAL_ENGINE_TASK not set or empty'); process.exit(1); }
+  if (!model) { console.error('ARCHAL_ENGINE_MODEL not set'); process.exit(1); }
+
+  const provider = detectProvider(model);
+  const apiKey = resolveApiKey(provider);
+  const log = createLogger({ harness: harnessName, model, provider });
+
+  const twinUrls = collectTwinUrls();
+  if (Object.keys(twinUrls).length === 0) {
+    console.error(`[${harnessName}] No twin URLs found. Check ARCHAL_TWIN_NAMES and ARCHAL_<TWIN>_URL env vars.`);
+    process.exit(1);
+  }
+
+  const { tools: allTools, toolToTwin } = await discoverAllTools(twinUrls);
+  if (allTools.length === 0) {
+    console.error(`[${harnessName}] No tools discovered from twins. Twin endpoints may be unreachable.`);
+    process.exit(1);
+  }
+
+  return { harnessName, task, model, provider, apiKey, log, twinUrls, allTools, toolToTwin };
+}
+
+// ── Run loop ──────────────────────────────────────────────────────────
+
+/**
+ * @typedef {object} RunLoopOptions
+ * @property {string} systemPrompt - System prompt text (empty string for none)
+ * @property {number} maxSteps - Maximum iteration count
+ * @property {boolean} [useRetry=false] - Wrap LLM calls in withRetry
+ * @property {number} [retryCount=4] - Max retries when useRetry is true
+ * @property {boolean} [useTrace=false] - Record agent trace
+ * @property {number} [maxConsecutiveErrors=0] - Bail threshold (0 = no limit)
+ * @property {number} [maxInitialNoToolRecoveries=0] - Reprompt attempts when model doesn't call tools initially
+ * @property {(ctx: HarnessContext, state: RunState) => Array} [selectTools] -
+ *   Per-step tool selection function. Receives context and current state,
+ *   returns the MCP tools array for this step. Default: use all tools.
+ * @property {(ctx: HarnessContext, state: RunState, stepResult: StepResult) => 'continue' | 'break' | void} [onBeforeToolExecution] -
+ *   Hook called after parsing tool calls but before executing them.
+ *   Return 'continue' to skip tool execution and loop, 'break' to stop.
+ * @property {(provider: string, messages: Array|object) => Array|object} [initMessages] -
+ *   Optional post-init hook to modify the initial messages array before the
+ *   run loop starts (e.g. to prepend a triage instruction).
+ * @property {(ctx: HarnessContext, state: RunState, stepResult: StepResult) => void} [onAfterToolExecution] -
+ *   Hook called after tool results are appended. Return value is ignored.
+ * @property {(ctx: HarnessContext, state: RunState, stepResult: StepResult) => 'continue' | void} [onNoToolCalls] -
+ *   Hook called when the model responds without tool calls. Return
+ *   'continue' to add instructions and continue the loop.
+ * @property {(tc: { name: string, arguments: object }) => void} [onToolSuccess] -
+ *   Called after each successful tool call.
+ */
+
+/**
+ * @typedef {object} RunState
+ * Mutable state tracked across loop iterations.
+ * @property {Array|object} messages
+ * @property {number} stepsCompleted
+ * @property {number} totalInputTokens
+ * @property {number} totalOutputTokens
+ * @property {number} totalToolCalls
+ * @property {number} totalToolErrors
+ * @property {number} consecutiveErrors
+ * @property {number} initialNoToolRecoveries
+ * @property {string} exitReason
+ * @property {import('./agent-trace.mjs').ReturnType<typeof createAgentTrace>|null} agentTrace
+ */
+
+/**
+ * @typedef {object} StepResult
+ * @property {number} step - 1-indexed step number
+ * @property {object} response - Raw LLM response wrapper
+ * @property {Array|null} toolCalls - Parsed tool calls or null
+ * @property {string|null} thinking - Model thinking content
+ * @property {string|null} text - Model text content
+ * @property {number} iterDurationMs
+ * @property {string|null} stopReason
+ */
+
+/**
+ * Run the agent loop with shared metrics, logging, and tool execution.
+ *
+ * @param {HarnessContext} ctx
+ * @param {RunLoopOptions} opts
+ */
+export async function runAgentLoop(ctx, opts) {
+  const {
+    systemPrompt,
+    maxSteps,
+    useRetry = false,
+    retryCount = 4,
+    useTrace = false,
+    maxConsecutiveErrors = 0,
+    maxInitialNoToolRecoveries = 0,
+    selectTools,
+    onBeforeToolExecution,
+    onAfterToolExecution,
+    onNoToolCalls,
+    onToolSuccess,
+  } = opts;
+
+  const { harnessName, task, model, provider, apiKey, log, allTools, toolToTwin } = ctx;
+
+  let messages = buildInitialMessages(provider, systemPrompt, task, model);
+
+  // Allow callers to modify initial messages (e.g. react's triage instruction)
+  if (opts.initMessages) {
+    messages = opts.initMessages(provider, messages);
+  }
+
+  const state = {
+    messages,
+    stepsCompleted: 0,
+    totalInputTokens: 0,
+    totalOutputTokens: 0,
+    totalToolCalls: 0,
+    totalToolErrors: 0,
+    consecutiveErrors: 0,
+    initialNoToolRecoveries: 0,
+    exitReason: 'max_steps',
+    agentTrace: useTrace ? createAgentTrace() : null,
+  };
+
+  const runStart = Date.now();
+
+  log.info('run_start', { task: task.slice(0, 200), maxSteps });
+
+  try {
+    for (let step = 0; step < maxSteps; step++) {
+      state.stepsCompleted = step + 1;
+      const iterStart = Date.now();
+
+      // Select tools for this step (default: all tools)
+      const stepTools = selectTools ? selectTools(ctx, state) : allTools;
+      const providerTools = formatToolsForProvider(provider, stepTools);
+
+      // Call the LLM (optionally with retry)
+      log.llmCall(step + 1);
+      let response;
+      try {
+        const llmCall = () => callLlmWithMessages(provider, model, apiKey, state.messages, providerTools);
+        response = useRetry ? await withRetry(llmCall, retryCount) : await llmCall();
+      } catch (err) {
+        const msg = err?.message ?? String(err);
+        log.error('llm_call_failed', { step: step + 1, error: msg });
+        process.stderr.write(`[${harnessName}] LLM API error: ${msg.slice(0, 500)}\n`);
+        state.exitReason = 'llm_error';
+        break;
+      }
+
+      const iterDurationMs = Date.now() - iterStart;
+      state.totalInputTokens += response.usage.inputTokens;
+      state.totalOutputTokens += response.usage.outputTokens;
+
+      const toolCalls = parseToolCalls(provider, response);
+      const hasToolCalls = !!toolCalls;
+      const stopReason = getStopReason(provider, response);
+      log.llmResponse(step + 1, iterDurationMs, hasToolCalls, stopReason);
+      log.tokenUsage(step + 1, response.usage, {
+        inputTokens: state.totalInputTokens,
+        outputTokens: state.totalOutputTokens,
+      });
+
+      const thinking = getThinkingContent(provider, response);
+      const text = getResponseText(provider, response);
+
+      state.messages = appendAssistantResponse(provider, state.messages, response);
+
+      /** @type {StepResult} */
+      const stepResult = { step: step + 1, response, toolCalls, thinking, text, iterDurationMs, stopReason };
+
+      if (!toolCalls) {
+        // Record trace for no-tool-call steps
+        if (state.agentTrace) {
+          state.agentTrace.addStep({ step: step + 1, thinking, text, toolCalls: [], durationMs: iterDurationMs });
+        }
+        if (text) {
+          process.stderr.write(`[${harnessName}] Step ${step + 1}: ${text.slice(0, 200)}\n`);
+        }
+
+        // Initial no-tool recovery (reprompt)
+        const shouldRecoverInitial = state.totalToolCalls === 0
+          && maxInitialNoToolRecoveries > 0
+          && state.initialNoToolRecoveries < maxInitialNoToolRecoveries;
+        if (shouldRecoverInitial) {
+          state.initialNoToolRecoveries++;
+          state.messages = appendUserInstruction(
+            provider,
+            state.messages,
+            'You must use tools to make progress. ' +
+              'On your next response, call at least one relevant tool before giving any summary or conclusion. ' +
+              'Start by gathering concrete evidence from the systems, then execute the required actions.',
+          );
+          log.info('no_tool_calls_reprompt', {
+            step: step + 1,
+            attempt: state.initialNoToolRecoveries,
+          });
+          continue;
+        }
+
+        // Harness-specific no-tool-call handling
+        if (onNoToolCalls) {
+          const directive = onNoToolCalls(ctx, state, stepResult);
+          if (directive === 'continue') continue;
+        }
+
+        state.exitReason = state.totalToolCalls === 0 ? 'no_tool_calls' : 'completed';
+        break;
+      }
+
+      state.initialNoToolRecoveries = 0;
+
+      // Pre-execution hook (e.g. react's repo content guard)
+      if (onBeforeToolExecution) {
+        const directive = onBeforeToolExecution(ctx, state, stepResult);
+        if (directive === 'continue') continue;
+        if (directive === 'break') break;
+      }
+
+      // Execute tool calls
+      const { executeToolCalls } = await import('./tool-executor.mjs');
+      const { results, bailout } = await executeToolCalls(toolCalls, {
+        toolToTwin,
+        harnessName,
+        step: step + 1,
+        log,
+        counters: state,
+        maxConsecutiveErrors,
+        onSuccess: onToolSuccess,
+      });
+
+      // Record trace
+      if (state.agentTrace) {
+        state.agentTrace.addStep({
+          step: step + 1,
+          thinking,
+          text,
+          toolCalls: toolCalls.map((tc) => ({ name: tc.name, arguments: tc.arguments })),
+          durationMs: iterDurationMs,
+        });
+      }
+
+      if (bailout) {
+        state.exitReason = 'consecutive_errors';
+        break;
+      }
+
+      // Append tool results to conversation
+      state.messages = appendToolResults(provider, state.messages, toolCalls, results);
+
+      // Post-execution hook
+      if (onAfterToolExecution) {
+        onAfterToolExecution(ctx, state, stepResult);
+      }
+    }
+  } finally {
+    const totalTimeMs = Date.now() - runStart;
+
+    log.summary({
+      iterations: state.stepsCompleted,
+      totalInputTokens: state.totalInputTokens,
+      totalOutputTokens: state.totalOutputTokens,
+      totalTimeMs,
+      toolCallCount: state.totalToolCalls,
+      toolErrorCount: state.totalToolErrors,
+      exitReason: state.exitReason,
+    });
+
+    writeMetrics({
+      inputTokens: state.totalInputTokens,
+      outputTokens: state.totalOutputTokens,
+      llmCallCount: state.stepsCompleted,
+      toolCallCount: state.totalToolCalls,
+      toolErrorCount: state.totalToolErrors,
+      totalTimeMs,
+      exitReason: state.exitReason,
+      provider,
+      model,
+    });
+
+    if (state.agentTrace) {
+      state.agentTrace.flush();
+    }
+
+    process.stderr.write(
+      `\n[${harnessName}] Summary: ${state.stepsCompleted} iterations, ${state.totalToolCalls} tool calls ` +
+      `(${state.totalToolErrors} errors), ${state.totalInputTokens} input tokens, ` +
+      `${state.totalOutputTokens} output tokens, ${(totalTimeMs / 1000).toFixed(1)}s total\n`
+    );
+
+    if (state.exitReason === 'llm_error') {
+      process.exit(1);
+    }
+  }
+}
+
+// Re-export for convenience — harnesses that need to build custom initial messages
+export { appendUserInstruction };