@archal/cli 0.7.11 → 0.8.0

package/README.md

@@ -20,19 +20,22 @@ archal init
 archal run scenario.md \
   --engine-endpoint "https://gateway.openclaw.ai/v1/responses" \
   --engine-token "$OPENCLAW_GATEWAY_TOKEN" \
-  --engine-model "openclaw:main"
+  --agent-model "openclaw:main"
 ```
 
 ## Commands
 
 | Command | Description |
 | --- | --- |
-| `archal init` | Scaffold a new archal project with example scenarios |
 | `archal run <scenario>` | Execute a scenario against hosted twins and evaluate results |
-| `archal login` | Authenticate via browser auth (hosted endpoint by default) |
+| `archal demo --model <model>` | Compare bundled harnesses on a scenario |
+| `archal batch run <suite>` | Run a configured suite as a hosted batch |
+| `archal batch compare <suite>` | Compare multiple named batch variants on one suite |
+| `archal batch list` | List suites and batch definitions from `.archal.json` |
+| `archal init` | Scaffold a new Archal project with example scenarios |
+| `archal login` | Authenticate via browser auth or token |
 | `archal whoami` | Show current auth/account status |
-| `archal twins` | Manage selected twins for your plan |
-| `archal trace` | Inspect run traces |
+| `archal logout` | Remove stored credentials |
 | `archal config` | View or update CLI configuration |
 
 ## Environment Variables
@@ -50,10 +53,10 @@ archal run scenario.md \
 | `ARCHAL_ENGINE_TIMEOUT` | Default API engine timeout in seconds |
 | `ARCHAL_ENGINE_TWIN_URLS` | Default path to remote twin URL overrides |
 | `ARCHAL_HARNESS_DIR` | Default harness directory for local mode |
-| `OPENCLAW_URL` | OpenClaw gateway URL (fallback alias for `--openclaw-url`) |
-| `OPENCLAW_GATEWAY_TOKEN` | OpenClaw gateway token used by `archal run` |
-| `OPENCLAW_GATEWAY_PASSWORD` | Password-mode OpenClaw gateway auth value |
-| `OPENCLAW_AGENT_ID` | Default OpenClaw agent/model id |
+| `OPENCLAW_URL` | Legacy OpenClaw gateway URL alias for API mode |
+| `OPENCLAW_GATEWAY_TOKEN` | Legacy OpenClaw gateway token |
+| `OPENCLAW_GATEWAY_PASSWORD` | Legacy password-mode OpenClaw gateway auth value |
+| `OPENCLAW_AGENT_ID` | Legacy OpenClaw agent/model id (prefer `ARCHAL_ENGINE_MODEL`) |
 
 ## Documentation
 

package/bin/archal.cjs

@@ -0,0 +1,15 @@
+#!/usr/bin/env node
+
+const { existsSync } = require("node:fs");
+const { join } = require("node:path");
+
+const distEntry = join(__dirname, "..", "dist", "index.cjs");
+
+if (!existsSync(distEntry)) {
+  console.error(
+    "The @archal/cli bundle has not been built yet. Run `pnpm --filter @archal/cli run build` first.",
+  );
+  process.exit(1);
+}
+
+require(distEntry);

package/dist/harnesses/_lib/agent-trace.mjs

@@ -0,0 +1,57 @@
+/**
+ * Structured agent trace writer for bundled harnesses.
+ *
+ * Records per-step model thinking, text output, and tool calls as a structured
+ * JSON trace. The orchestrator reads this file after the harness exits and flows
+ * it into RunResult → artifacts → dashboard.
+ *
+ * Transport: writes to ARCHAL_AGENT_TRACE_FILE (set by orchestrator).
+ * Safe no-op when the env var is not set.
+ *
+ * Trace format:
+ *   { version: 1, steps: [ { step, thinking, text, toolCalls, durationMs } ] }
+ */
+import { writeFileSync } from 'node:fs';
+
+/**
+ * @typedef {Object} TraceStep
+ * @property {number} step           - 1-indexed step number
+ * @property {string|null} thinking  - Model's internal reasoning (extended thinking / reasoning_content)
+ * @property {string|null} text      - Model's visible text output (reasoning "out loud")
+ * @property {Array<{name: string, arguments: object}>} toolCalls - Tools called this step
+ * @property {number} durationMs     - LLM call duration for this step
+ */
+
+/**
+ * Create a trace collector that accumulates steps and writes on flush.
+ * @returns {{ addStep: (step: TraceStep) => void, flush: () => void }}
+ */
+export function createAgentTrace() {
+  /** @type {TraceStep[]} */
+  const steps = [];
+
+  return {
+    /**
+     * Record a single agent step.
+     * @param {TraceStep} step
+     */
+    addStep(step) {
+      steps.push(step);
+    },
+
+    /**
+     * Write the accumulated trace to the file. Call once at the end.
+     */
+    flush() {
+      const tracePath = process.env['ARCHAL_AGENT_TRACE_FILE'];
+      if (!tracePath) return;
+
+      try {
+        const payload = { version: 1, steps };
+        writeFileSync(tracePath, JSON.stringify(payload));
+      } catch {
+        // Non-fatal — trace is best-effort
+      }
+    },
+  };
+}

package/dist/harnesses/_lib/logging.mjs

@@ -0,0 +1,176 @@
+/**
+ * Structured logging helper for bundled harnesses.
+ * Outputs JSON lines (one JSON object per line) to stderr.
+ *
+ * Each log line includes: timestamp, iteration, model, provider, event type,
+ * and event-specific fields.
+ *
+ * Log levels: debug, info, warn, error
+ * Controlled via ARCHAL_LOG_LEVEL env var (default: info).
+ */
+
+// ── Log levels ──────────────────────────────────────────────────────
+
+/** @enum {number} */
+const LOG_LEVELS = {
+  debug: 0,
+  info: 1,
+  warn: 2,
+  error: 3,
+};
+
+const currentLevel = LOG_LEVELS[process.env['ARCHAL_LOG_LEVEL']?.toLowerCase() ?? 'info'] ?? LOG_LEVELS.info;
+
+// ── Logger factory ──────────────────────────────────────────────────
+
+/**
+ * @typedef {Object} LogContext
+ * @property {string} harness  - Harness name (e.g. "react")
+ * @property {string} model    - Model identifier
+ * @property {string} provider - Provider name
+ */
+
+/**
+ * @typedef {Object} Logger
+ * @property {function} debug - Log at debug level
+ * @property {function} info  - Log at info level
+ * @property {function} warn  - Log at warn level
+ * @property {function} error - Log at error level
+ * @property {function} tokenUsage   - Log token usage event
+ * @property {function} toolCall     - Log tool call event
+ * @property {function} toolError    - Log tool error event
+ * @property {function} llmCall      - Log LLM call event
+ * @property {function} llmResponse  - Log LLM response event
+ * @property {function} summary      - Log run summary event
+ */
+
+/**
+ * Create a structured logger bound to a harness context.
+ * @param {LogContext} context
+ * @returns {Logger}
+ */
+export function createLogger(context) {
+  const { harness, model, provider } = context;
+
+  /**
+   * Write a structured log line to stderr.
+   * @param {'debug' | 'info' | 'warn' | 'error'} level
+   * @param {string} event
+   * @param {Record<string, unknown>} [fields]
+   * @param {number} [iteration]
+   */
+  function log(level, event, fields = {}, iteration = undefined) {
+    if (LOG_LEVELS[level] < currentLevel) return;
+
+    const line = {
+      ts: new Date().toISOString(),
+      level,
+      harness,
+      model,
+      provider,
+      event,
+      ...(iteration !== undefined ? { iteration } : {}),
+      ...fields,
+    };
+    process.stderr.write(JSON.stringify(line) + '\n');
+  }
+
+  return {
+    debug: (event, fields, iteration) => log('debug', event, fields, iteration),
+    info: (event, fields, iteration) => log('info', event, fields, iteration),
+    warn: (event, fields, iteration) => log('warn', event, fields, iteration),
+    error: (event, fields, iteration) => log('error', event, fields, iteration),
+
+    /**
+     * Log token usage for an LLM call.
+     * @param {number} iteration
+     * @param {object} usage - { inputTokens, outputTokens }
+     * @param {object} cumulative - { inputTokens, outputTokens }
+     */
+    tokenUsage(iteration, usage, cumulative) {
+      log('info', 'token_usage', {
+        inputTokens: usage.inputTokens,
+        outputTokens: usage.outputTokens,
+        cumulativeInputTokens: cumulative.inputTokens,
+        cumulativeOutputTokens: cumulative.outputTokens,
+      }, iteration);
+    },
+
+    /**
+     * Log a tool call.
+     * @param {number} iteration
+     * @param {string} toolName
+     * @param {object} args - Tool arguments (truncated)
+     * @param {number} durationMs
+     */
+    toolCall(iteration, toolName, args, durationMs) {
+      log('info', 'tool_call', {
+        tool: toolName,
+        args: truncate(JSON.stringify(args), 200),
+        durationMs,
+      }, iteration);
+    },
+
+    /**
+     * Log a tool error.
+     * @param {number} iteration
+     * @param {string} toolName
+     * @param {string} errorMessage
+     */
+    toolError(iteration, toolName, errorMessage) {
+      log('error', 'tool_error', {
+        tool: toolName,
+        error: truncate(errorMessage, 500),
+      }, iteration);
+    },
+
+    /**
+     * Log an LLM call start.
+     * @param {number} iteration
+     */
+    llmCall(iteration) {
+      log('debug', 'llm_call_start', {}, iteration);
+    },
+
+    /**
+     * Log an LLM response.
+     * @param {number} iteration
+     * @param {number} durationMs
+     * @param {boolean} hasToolCalls
+     * @param {string|null} stopReason
+     */
+    llmResponse(iteration, durationMs, hasToolCalls, stopReason) {
+      log('info', 'llm_response', {
+        durationMs,
+        hasToolCalls,
+        ...(stopReason ? { stopReason } : {}),
+      }, iteration);
+    },
+
+    /**
+     * Log a run summary at the end.
+     * @param {object} stats
+     * @param {number} stats.iterations
+     * @param {number} stats.totalInputTokens
+     * @param {number} stats.totalOutputTokens
+     * @param {number} stats.totalTimeMs
+     * @param {number} stats.toolCallCount
+     * @param {number} stats.toolErrorCount
+     * @param {string} stats.exitReason
+     */
+    summary(stats) {
+      log('info', 'run_summary', stats);
+    },
+  };
+}
+
+/**
+ * Truncate a string to a maximum length with ellipsis.
+ * @param {string} str
+ * @param {number} maxLen
+ * @returns {string}
+ */
+function truncate(str, maxLen) {
+  if (str.length <= maxLen) return str;
+  return str.slice(0, maxLen - 3) + '...';
+}

package/dist/harnesses/_lib/mcp-client.mjs

@@ -0,0 +1,80 @@
+/**
+ * Shared MCP client helper for bundled harnesses.
+ * Connects to cloud-hosted twins via HTTP MCP transport.
+ */
+import { readFileSync } from 'node:fs';
+import { Client } from '@modelcontextprotocol/sdk/client/index.js';
+import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js';
+import { SSEClientTransport } from '@modelcontextprotocol/sdk/client/sse.js';
+
+/**
+ * Connect to the first MCP server from the ARCHAL_MCP_CONFIG JSON file.
+ * Tries StreamableHTTP first, falls back to SSE transport.
+ * @returns {{ client: Client, serverName: string }}
+ */
+export async function connectMcp(configPath) {
+  if (!configPath) {
+    throw new Error('ARCHAL_MCP_CONFIG is not set — no MCP server config available');
+  }
+
+  const config = JSON.parse(readFileSync(configPath, 'utf-8'));
+  const serverName = Object.keys(config.mcpServers)[0];
+  if (!serverName) {
+    throw new Error('No MCP servers found in config');
+  }
+
+  const serverConfig = config.mcpServers[serverName];
+  const mcpUrl = serverConfig.url;
+  if (!mcpUrl) {
+    throw new Error(`MCP server "${serverName}" has no URL — cannot connect via HTTP`);
+  }
+
+  const client = new Client({ name: 'archal-harness-agent', version: '1.0.0' });
+
+  // Try StreamableHTTP first (modern MCP transport)
+  try {
+    const transport = new StreamableHTTPClientTransport(new URL(mcpUrl));
+    await client.connect(transport);
+    return { client, serverName };
+  } catch {
+    // StreamableHTTP may not be supported — fall back to SSE
+  }
+
+  // Fall back to SSE transport
+  try {
+    const transport = new SSEClientTransport(new URL(mcpUrl));
+    await client.connect(transport);
+    return { client, serverName };
+  } catch (err) {
+    throw new Error(
+      `Failed to connect to MCP server "${serverName}" at ${mcpUrl}: ${err.message}`
+    );
+  }
+}
+
+/**
+ * Discover available tools from the MCP server.
+ * @param {Client} client
+ * @returns {Array<{ name: string, description: string, inputSchema: object }>}
+ */
+export async function discoverTools(client) {
+  const { tools } = await client.listTools();
+  return tools.map((t) => ({
+    name: t.name,
+    description: t.description ?? '',
+    inputSchema: t.inputSchema ?? {},
+  }));
+}
+
+/**
+ * Call a tool on the MCP server and return the text content.
+ * @param {Client} client
+ * @param {string} name
+ * @param {object} args
+ * @returns {string}
+ */
+export async function callTool(client, name, args) {
+  const result = await client.callTool({ name, arguments: args ?? {} });
+  const text = result.content?.map((c) => c.text ?? '').join('\n') ?? 'No output';
+  return text;
+}

package/dist/harnesses/_lib/metrics.mjs

@@ -0,0 +1,34 @@
+/**
+ * Structured metrics writer for archal harnesses.
+ *
+ * Writes a JSON metrics file to the path specified by ARCHAL_METRICS_FILE.
+ * The orchestrator creates this path, reads it after the harness exits, and
+ * flows the data into RunResult.tokenUsage and telemetry.
+ *
+ * Safe no-op when ARCHAL_METRICS_FILE is not set (external harnesses that
+ * don't know about this protocol, or older orchestrator versions).
+ *
+ * @param {object} metrics
+ * @param {number} metrics.inputTokens
+ * @param {number} metrics.outputTokens
+ * @param {number} metrics.llmCallCount
+ * @param {number} metrics.toolCallCount
+ * @param {number} metrics.toolErrorCount
+ * @param {number} metrics.totalTimeMs
+ * @param {string} metrics.exitReason
+ * @param {string} [metrics.provider]
+ * @param {string} [metrics.model]
+ */
+import { writeFileSync } from 'node:fs';
+
+export function writeMetrics(metrics) {
+  const metricsPath = process.env['ARCHAL_METRICS_FILE'];
+  if (!metricsPath) return;
+
+  try {
+    const payload = { version: 1, ...metrics };
+    writeFileSync(metricsPath, JSON.stringify(payload));
+  } catch {
+    // Non-fatal — metrics are best-effort
+  }
+}