@archal/cli 0.8.0 → 0.9.1

package/README.md

@@ -13,16 +13,16 @@ npm install -g @archal/cli
 ## Quick Start
 
 ```bash
-# Initialize a project in the current directory
-archal init
-
-# Run a scenario (cloud-only transport)
-archal run scenario.md \
-  --engine-endpoint "https://gateway.openclaw.ai/v1/responses" \
-  --engine-token "$OPENCLAW_GATEWAY_TOKEN" \
-  --agent-model "openclaw:main"
+# Authenticate with Archal
+archal login
+# or: export ARCHAL_TOKEN=archal_...
+
+# Run a security-suite scenario with OpenClaw
+archal openclaw run scenarios/security-suite/exec-impersonation.md
 ```
 
+This is the recommended first run for safety testing. If you use a remote OpenClaw gateway, configure it first with `archal openclaw connect`.
+
 ## Commands
 
 | Command | Description |

package/dist/harnesses/_lib/env-utils.mjs

@@ -0,0 +1,23 @@
+/**
+ * Shared environment variable parsing utilities for bundled harnesses.
+ */
+
+/**
+ * Parse an integer from an environment variable with validation and clamping.
+ * Replaces the repeated IIFE pattern across agent files.
+ *
+ * @param {string} envVar - Environment variable name
+ * @param {number} defaultValue - Default if env var is not set or invalid
+ * @param {{ min?: number, max?: number }} [opts] - Optional min/max bounds
+ * @returns {number}
+ */
+export function parseEnvInt(envVar, defaultValue, { min, max } = {}) {
+  const raw = process.env[envVar]?.trim();
+  if (!raw) return defaultValue;
+  const parsed = parseInt(raw, 10);
+  if (Number.isNaN(parsed)) return defaultValue;
+  let value = parsed;
+  if (min !== undefined && value < min) value = min;
+  if (max !== undefined && value > max) value = max;
+  return value;
+}

package/dist/harnesses/_lib/harness-runner.mjs

@@ -0,0 +1,373 @@
+/**
+ * Shared harness scaffolding for bundled agent files.
+ *
+ * Extracts the common init sequence and run-loop structure that all 4
+ * bundled harnesses (naive, zero-shot, hardened, react) duplicate.
+ *
+ * Usage:
+ *   const ctx = await createHarnessContext('react');
+ *   await runAgentLoop(ctx, { ... });
+ */
+import { collectTwinUrls, discoverAllTools } from './rest-client.mjs';
+import {
+  detectProvider,
+  resolveApiKey,
+  formatToolsForProvider,
+  buildInitialMessages,
+  appendAssistantResponse,
+  appendToolResults,
+  appendUserInstruction,
+  callLlmWithMessages,
+  parseToolCalls,
+  getResponseText,
+  getThinkingContent,
+  getStopReason,
+  withRetry,
+} from './providers.mjs';
+import { createLogger } from './logging.mjs';
+import { writeMetrics } from './metrics.mjs';
+import { createAgentTrace } from './agent-trace.mjs';
+
+// ── Context creation ──────────────────────────────────────────────────
+
+/**
+ * @typedef {object} HarnessContext
+ * @property {string} harnessName
+ * @property {string} task
+ * @property {string} model
+ * @property {string} provider
+ * @property {string} apiKey
+ * @property {import('./logging.mjs').Logger} log
+ * @property {Record<string, string>} twinUrls
+ * @property {Array<{ name: string, description: string, inputSchema: object }>} allTools
+ * @property {Record<string, { twinName: string, baseUrl: string, originalName: string }>} toolToTwin
+ */
+
+/**
+ * Create the full harness context: validate env vars, detect provider,
+ * resolve API key, collect twin URLs, and discover tools.
+ *
+ * Exits with code 1 on missing env vars or unreachable twins.
+ *
+ * @param {string} harnessName
+ * @returns {Promise<HarnessContext>}
+ */
+export async function createHarnessContext(harnessName) {
+  let task = (process.env['ARCHAL_ENGINE_TASK'] || '').trim();
+  // If the task value is a file path inside the mounted config dir,
+  // read the actual task content from the file. This happens when the
+  // Docker harness writes multi-line task text to a file to avoid
+  // exposing it via docker -e flags (security: prevents secret leakage
+  // in docker ps / /proc/<pid>/cmdline).
+  if (task.startsWith('/archal-out/') || task.startsWith(process.env['ARCHAL_ENGINE_TASK_FILE'] ? '/' : '\0')) {
+    try {
+      const { readFileSync } = await import('node:fs');
+      task = readFileSync(task, 'utf-8').trim();
+    } catch { /* fall through to original value */ }
+  }
+  // Also check the _FILE convention: if ARCHAL_ENGINE_TASK is empty but
+  // ARCHAL_ENGINE_TASK_FILE points to a file, read from there.
+  if (!task && process.env['ARCHAL_ENGINE_TASK_FILE']) {
+    try {
+      const { readFileSync } = await import('node:fs');
+      task = readFileSync(process.env['ARCHAL_ENGINE_TASK_FILE'], 'utf-8').trim();
+    } catch { /* fall through */ }
+  }
+  const model = process.env['ARCHAL_ENGINE_MODEL'];
+
+  if (!task) { console.error('ARCHAL_ENGINE_TASK not set or empty'); process.exit(1); }
+  if (!model) { console.error('ARCHAL_ENGINE_MODEL not set'); process.exit(1); }
+
+  const provider = detectProvider(model);
+  const apiKey = resolveApiKey(provider);
+  const log = createLogger({ harness: harnessName, model, provider });
+
+  const twinUrls = collectTwinUrls();
+  if (Object.keys(twinUrls).length === 0) {
+    console.error(`[${harnessName}] No twin URLs found. Check ARCHAL_TWIN_NAMES and ARCHAL_<TWIN>_URL env vars.`);
+    process.exit(1);
+  }
+
+  const { tools: allTools, toolToTwin } = await discoverAllTools(twinUrls);
+  if (allTools.length === 0) {
+    console.error(`[${harnessName}] No tools discovered from twins. Twin endpoints may be unreachable.`);
+    process.exit(1);
+  }
+
+  return { harnessName, task, model, provider, apiKey, log, twinUrls, allTools, toolToTwin };
+}
+
+// ── Run loop ──────────────────────────────────────────────────────────
+
+/**
+ * @typedef {object} RunLoopOptions
+ * @property {string} systemPrompt - System prompt text (empty string for none)
+ * @property {number} maxSteps - Maximum iteration count
+ * @property {boolean} [useRetry=false] - Wrap LLM calls in withRetry
+ * @property {number} [retryCount=4] - Max retries when useRetry is true
+ * @property {boolean} [useTrace=false] - Record agent trace
+ * @property {number} [maxConsecutiveErrors=0] - Bail threshold (0 = no limit)
+ * @property {number} [maxInitialNoToolRecoveries=0] - Reprompt attempts when model doesn't call tools initially
+ * @property {(ctx: HarnessContext, state: RunState) => Array} [selectTools] -
+ *   Per-step tool selection function. Receives context and current state,
+ *   returns the MCP tools array for this step. Default: use all tools.
+ * @property {(ctx: HarnessContext, state: RunState, stepResult: StepResult) => 'continue' | 'break' | void} [onBeforeToolExecution] -
+ *   Hook called after parsing tool calls but before executing them.
+ *   Return 'continue' to skip tool execution and loop, 'break' to stop.
+ * @property {(provider: string, messages: Array|object) => Array|object} [initMessages] -
+ *   Optional post-init hook to modify the initial messages array before the
+ *   run loop starts (e.g. to prepend a triage instruction).
+ * @property {(ctx: HarnessContext, state: RunState, stepResult: StepResult) => void} [onAfterToolExecution] -
+ *   Hook called after tool results are appended. Return value is ignored.
+ * @property {(ctx: HarnessContext, state: RunState, stepResult: StepResult) => 'continue' | void} [onNoToolCalls] -
+ *   Hook called when the model responds without tool calls. Return
+ *   'continue' to add instructions and continue the loop.
+ * @property {(tc: { name: string, arguments: object }) => void} [onToolSuccess] -
+ *   Called after each successful tool call.
+ */
+
+/**
+ * @typedef {object} RunState
+ * Mutable state tracked across loop iterations.
+ * @property {Array|object} messages
+ * @property {number} stepsCompleted
+ * @property {number} totalInputTokens
+ * @property {number} totalOutputTokens
+ * @property {number} totalToolCalls
+ * @property {number} totalToolErrors
+ * @property {number} consecutiveErrors
+ * @property {number} initialNoToolRecoveries
+ * @property {string} exitReason
+ * @property {import('./agent-trace.mjs').ReturnType<typeof createAgentTrace>|null} agentTrace
+ */
+
+/**
+ * @typedef {object} StepResult
+ * @property {number} step - 1-indexed step number
+ * @property {object} response - Raw LLM response wrapper
+ * @property {Array|null} toolCalls - Parsed tool calls or null
+ * @property {string|null} thinking - Model thinking content
+ * @property {string|null} text - Model text content
+ * @property {number} iterDurationMs
+ * @property {string|null} stopReason
+ */
+
+/**
+ * Run the agent loop with shared metrics, logging, and tool execution.
+ *
+ * @param {HarnessContext} ctx
+ * @param {RunLoopOptions} opts
+ */
+export async function runAgentLoop(ctx, opts) {
+  const {
+    systemPrompt,
+    maxSteps,
+    useRetry = false,
+    retryCount = 4,
+    useTrace = false,
+    maxConsecutiveErrors = 0,
+    maxInitialNoToolRecoveries = 0,
+    selectTools,
+    onBeforeToolExecution,
+    onAfterToolExecution,
+    onNoToolCalls,
+    onToolSuccess,
+  } = opts;
+
+  const { harnessName, task, model, provider, apiKey, log, allTools, toolToTwin } = ctx;
+
+  let messages = buildInitialMessages(provider, systemPrompt, task, model);
+
+  // Allow callers to modify initial messages (e.g. react's triage instruction)
+  if (opts.initMessages) {
+    messages = opts.initMessages(provider, messages);
+  }
+
+  const state = {
+    messages,
+    stepsCompleted: 0,
+    totalInputTokens: 0,
+    totalOutputTokens: 0,
+    totalToolCalls: 0,
+    totalToolErrors: 0,
+    consecutiveErrors: 0,
+    initialNoToolRecoveries: 0,
+    exitReason: 'max_steps',
+    agentTrace: useTrace ? createAgentTrace() : null,
+  };
+
+  const runStart = Date.now();
+
+  log.info('run_start', { task: task.slice(0, 200), maxSteps });
+
+  try {
+    for (let step = 0; step < maxSteps; step++) {
+      state.stepsCompleted = step + 1;
+      const iterStart = Date.now();
+
+      // Select tools for this step (default: all tools)
+      const stepTools = selectTools ? selectTools(ctx, state) : allTools;
+      const providerTools = formatToolsForProvider(provider, stepTools);
+
+      // Call the LLM (optionally with retry)
+      log.llmCall(step + 1);
+      let response;
+      try {
+        const llmCall = () => callLlmWithMessages(provider, model, apiKey, state.messages, providerTools);
+        response = useRetry ? await withRetry(llmCall, retryCount) : await llmCall();
+      } catch (err) {
+        const msg = err?.message ?? String(err);
+        log.error('llm_call_failed', { step: step + 1, error: msg });
+        process.stderr.write(`[${harnessName}] LLM API error: ${msg.slice(0, 500)}\n`);
+        state.exitReason = 'llm_error';
+        break;
+      }
+
+      const iterDurationMs = Date.now() - iterStart;
+      state.totalInputTokens += response.usage.inputTokens;
+      state.totalOutputTokens += response.usage.outputTokens;
+
+      const toolCalls = parseToolCalls(provider, response);
+      const hasToolCalls = !!toolCalls;
+      const stopReason = getStopReason(provider, response);
+      log.llmResponse(step + 1, iterDurationMs, hasToolCalls, stopReason);
+      log.tokenUsage(step + 1, response.usage, {
+        inputTokens: state.totalInputTokens,
+        outputTokens: state.totalOutputTokens,
+      });
+
+      const thinking = getThinkingContent(provider, response);
+      const text = getResponseText(provider, response);
+
+      state.messages = appendAssistantResponse(provider, state.messages, response);
+
+      /** @type {StepResult} */
+      const stepResult = { step: step + 1, response, toolCalls, thinking, text, iterDurationMs, stopReason };
+
+      if (!toolCalls) {
+        // Record trace for no-tool-call steps
+        if (state.agentTrace) {
+          state.agentTrace.addStep({ step: step + 1, thinking, text, toolCalls: [], durationMs: iterDurationMs });
+        }
+        if (text) {
+          process.stderr.write(`[${harnessName}] Step ${step + 1}: ${text.slice(0, 200)}\n`);
+        }
+
+        // Initial no-tool recovery (reprompt)
+        const shouldRecoverInitial = state.totalToolCalls === 0
+          && maxInitialNoToolRecoveries > 0
+          && state.initialNoToolRecoveries < maxInitialNoToolRecoveries;
+        if (shouldRecoverInitial) {
+          state.initialNoToolRecoveries++;
+          state.messages = appendUserInstruction(
+            provider,
+            state.messages,
+            'You must use tools to make progress. ' +
+              'On your next response, call at least one relevant tool before giving any summary or conclusion. ' +
+              'Start by gathering concrete evidence from the systems, then execute the required actions.',
+          );
+          log.info('no_tool_calls_reprompt', {
+            step: step + 1,
+            attempt: state.initialNoToolRecoveries,
+          });
+          continue;
+        }
+
+        // Harness-specific no-tool-call handling
+        if (onNoToolCalls) {
+          const directive = onNoToolCalls(ctx, state, stepResult);
+          if (directive === 'continue') continue;
+        }
+
+        state.exitReason = state.totalToolCalls === 0 ? 'no_tool_calls' : 'completed';
+        break;
+      }
+
+      state.initialNoToolRecoveries = 0;
+
+      // Pre-execution hook (e.g. react's repo content guard)
+      if (onBeforeToolExecution) {
+        const directive = onBeforeToolExecution(ctx, state, stepResult);
+        if (directive === 'continue') continue;
+        if (directive === 'break') break;
+      }
+
+      // Execute tool calls
+      const { executeToolCalls } = await import('./tool-executor.mjs');
+      const { results, bailout } = await executeToolCalls(toolCalls, {
+        toolToTwin,
+        harnessName,
+        step: step + 1,
+        log,
+        counters: state,
+        maxConsecutiveErrors,
+        onSuccess: onToolSuccess,
+      });
+
+      // Record trace
+      if (state.agentTrace) {
+        state.agentTrace.addStep({
+          step: step + 1,
+          thinking,
+          text,
+          toolCalls: toolCalls.map((tc) => ({ name: tc.name, arguments: tc.arguments })),
+          durationMs: iterDurationMs,
+        });
+      }
+
+      if (bailout) {
+        state.exitReason = 'consecutive_errors';
+        break;
+      }
+
+      // Append tool results to conversation
+      state.messages = appendToolResults(provider, state.messages, toolCalls, results);
+
+      // Post-execution hook
+      if (onAfterToolExecution) {
+        onAfterToolExecution(ctx, state, stepResult);
+      }
+    }
+  } finally {
+    const totalTimeMs = Date.now() - runStart;
+
+    log.summary({
+      iterations: state.stepsCompleted,
+      totalInputTokens: state.totalInputTokens,
+      totalOutputTokens: state.totalOutputTokens,
+      totalTimeMs,
+      toolCallCount: state.totalToolCalls,
+      toolErrorCount: state.totalToolErrors,
+      exitReason: state.exitReason,
+    });
+
+    writeMetrics({
+      inputTokens: state.totalInputTokens,
+      outputTokens: state.totalOutputTokens,
+      llmCallCount: state.stepsCompleted,
+      toolCallCount: state.totalToolCalls,
+      toolErrorCount: state.totalToolErrors,
+      totalTimeMs,
+      exitReason: state.exitReason,
+      provider,
+      model,
+    });
+
+    if (state.agentTrace) {
+      state.agentTrace.flush();
+    }
+
+    process.stderr.write(
+      `\n[${harnessName}] Summary: ${state.stepsCompleted} iterations, ${state.totalToolCalls} tool calls ` +
+      `(${state.totalToolErrors} errors), ${state.totalInputTokens} input tokens, ` +
+      `${state.totalOutputTokens} output tokens, ${(totalTimeMs / 1000).toFixed(1)}s total\n`
+    );
+
+    if (state.exitReason === 'llm_error') {
+      process.exit(1);
+    }
+  }
+}
+
+// Re-export for convenience — harnesses that need to build custom initial messages
+export { appendUserInstruction };