@archal/cli 0.7.12 → 0.9.0

package/dist/harnesses/_lib/mcp-client.mjs

@@ -0,0 +1,80 @@
+/**
+ * Shared MCP client helper for bundled harnesses.
+ * Connects to cloud-hosted twins via HTTP MCP transport.
+ */
+import { readFileSync } from 'node:fs';
+import { Client } from '@modelcontextprotocol/sdk/client/index.js';
+import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js';
+import { SSEClientTransport } from '@modelcontextprotocol/sdk/client/sse.js';
+
+/**
+ * Connect to the first MCP server from the ARCHAL_MCP_CONFIG JSON file.
+ * Tries StreamableHTTP first, falls back to SSE transport.
+ * @returns {{ client: Client, serverName: string }}
+ */
+export async function connectMcp(configPath) {
+  if (!configPath) {
+    throw new Error('ARCHAL_MCP_CONFIG is not set — no MCP server config available');
+  }
+
+  const config = JSON.parse(readFileSync(configPath, 'utf-8'));
+  const serverName = Object.keys(config.mcpServers)[0];
+  if (!serverName) {
+    throw new Error('No MCP servers found in config');
+  }
+
+  const serverConfig = config.mcpServers[serverName];
+  const mcpUrl = serverConfig.url;
+  if (!mcpUrl) {
+    throw new Error(`MCP server "${serverName}" has no URL — cannot connect via HTTP`);
+  }
+
+  const client = new Client({ name: 'archal-harness-agent', version: '1.0.0' });
+
+  // Try StreamableHTTP first (modern MCP transport)
+  try {
+    const transport = new StreamableHTTPClientTransport(new URL(mcpUrl));
+    await client.connect(transport);
+    return { client, serverName };
+  } catch {
+    // StreamableHTTP may not be supported — fall back to SSE
+  }
+
+  // Fall back to SSE transport
+  try {
+    const transport = new SSEClientTransport(new URL(mcpUrl));
+    await client.connect(transport);
+    return { client, serverName };
+  } catch (err) {
+    throw new Error(
+      `Failed to connect to MCP server "${serverName}" at ${mcpUrl}: ${err.message}`
+    );
+  }
+}
+
+/**
+ * Discover available tools from the MCP server.
+ * @param {Client} client
+ * @returns {Array<{ name: string, description: string, inputSchema: object }>}
+ */
+export async function discoverTools(client) {
+  const { tools } = await client.listTools();
+  return tools.map((t) => ({
+    name: t.name,
+    description: t.description ?? '',
+    inputSchema: t.inputSchema ?? {},
+  }));
+}
+
+/**
+ * Call a tool on the MCP server and return the text content.
+ * @param {Client} client
+ * @param {string} name
+ * @param {object} args
+ * @returns {string}
+ */
+export async function callTool(client, name, args) {
+  const result = await client.callTool({ name, arguments: args ?? {} });
+  const text = result.content?.map((c) => c.text ?? '').join('\n') ?? 'No output';
+  return text;
+}

package/dist/harnesses/_lib/metrics.mjs

@@ -0,0 +1,34 @@
+/**
+ * Structured metrics writer for archal harnesses.
+ *
+ * Writes a JSON metrics file to the path specified by ARCHAL_METRICS_FILE.
+ * The orchestrator creates this path, reads it after the harness exits, and
+ * flows the data into RunResult.tokenUsage and telemetry.
+ *
+ * Safe no-op when ARCHAL_METRICS_FILE is not set (external harnesses that
+ * don't know about this protocol, or older orchestrator versions).
+ *
+ * @param {object} metrics
+ * @param {number} metrics.inputTokens
+ * @param {number} metrics.outputTokens
+ * @param {number} metrics.llmCallCount
+ * @param {number} metrics.toolCallCount
+ * @param {number} metrics.toolErrorCount
+ * @param {number} metrics.totalTimeMs
+ * @param {string} metrics.exitReason
+ * @param {string} [metrics.provider]
+ * @param {string} [metrics.model]
+ */
+import { writeFileSync } from 'node:fs';
+
+export function writeMetrics(metrics) {
+  const metricsPath = process.env['ARCHAL_METRICS_FILE'];
+  if (!metricsPath) return;
+
+  try {
+    const payload = { version: 1, ...metrics };
+    writeFileSync(metricsPath, JSON.stringify(payload));
+  } catch {
+    // Non-fatal — metrics are best-effort
+  }
+}

package/dist/harnesses/_lib/model-configs.mjs

@@ -0,0 +1,521 @@
+/**
+ * Model configuration system for bundled harnesses.
+ *
+ * Provides default configs per model family, known capabilities,
+ * and a merge function: hardcoded defaults -> model family defaults -> env overrides.
+ *
+ * Zero dependencies — pure data and functions.
+ */
+
+// ── Model capabilities ──────────────────────────────────────────────
+
+/**
+ * @typedef {Object} ModelCapabilities
+ * @property {boolean} supportsTools      - Can use function/tool calling
+ * @property {boolean} supportsSystemPrompt - Accepts a system prompt
+ * @property {boolean} supportsReasoning  - Has reasoning/thinking mode (o1, o3, etc.)
+ * @property {boolean} supportsThinking   - Has extended thinking / reasoning trace (Anthropic, Gemini 2.5)
+ * @property {number}  maxContextWindow   - Max context window in tokens
+ * @property {boolean} supportsStreaming   - Supports streaming responses
+ */
+
+/**
+ * @typedef {Object} ModelConfig
+ * @property {number}  [maxTokens]       - Max completion tokens
+ * @property {number}  [temperature]     - Sampling temperature
+ * @property {string}  [reasoningEffort] - For reasoning models: low/medium/high
+ * @property {number}  [topP]            - Top-p sampling
+ */
+
+/**
+ * @typedef {'working' | 'degraded' | 'broken' | 'untested'} BenchmarkStatus
+ */
+
+/**
+ * @typedef {Object} ModelInfo
+ * @property {string} family             - Model family key
+ * @property {string} provider           - Provider name
+ * @property {ModelCapabilities} capabilities
+ * @property {ModelConfig} defaults      - Default config for this model
+ * @property {BenchmarkStatus} benchmarkStatus - Status from benchmark testing
+ * @property {string} [benchmarkNotes]   - Notes about benchmark performance
+ */
+
+// ── Known model registry ────────────────────────────────────────────
+
+/** @type {Record<string, ModelInfo>} */
+const MODEL_REGISTRY = {
+  // ── Anthropic ──
+  'claude-opus-4-6': {
+    family: 'claude-opus',
+    provider: 'anthropic',
+    capabilities: {
+      supportsTools: true,
+      supportsSystemPrompt: true,
+      supportsReasoning: false,
+      supportsThinking: true,
+      maxContextWindow: 200000,
+      supportsStreaming: true,
+    },
+    defaults: { maxTokens: 32768, temperature: 0.2 },
+    benchmarkStatus: 'working',
+    benchmarkNotes: 'Top performer across all scenarios. Reliable tool use.',
+  },
+  'claude-sonnet-4-6': {
+    family: 'claude-sonnet',
+    provider: 'anthropic',
+    capabilities: {
+      supportsTools: true,
+      supportsSystemPrompt: true,
+      supportsReasoning: false,
+      supportsThinking: true,
+      maxContextWindow: 200000,
+      supportsStreaming: true,
+    },
+    defaults: { maxTokens: 32768, temperature: 0.2 },
+    benchmarkStatus: 'working',
+    benchmarkNotes: 'Strong performance, good cost/quality balance.',
+  },
+  'claude-sonnet-4-20250514': {
+    family: 'claude-sonnet',
+    provider: 'anthropic',
+    capabilities: {
+      supportsTools: true,
+      supportsSystemPrompt: true,
+      supportsReasoning: false,
+      supportsThinking: true,
+      maxContextWindow: 200000,
+      supportsStreaming: true,
+    },
+    defaults: { maxTokens: 32768, temperature: 0.2 },
+    benchmarkStatus: 'working',
+    benchmarkNotes: 'Solid tool use. Slightly behind claude-sonnet-4-6.',
+  },
+  'claude-haiku-4-5-20251001': {
+    family: 'claude-haiku',
+    provider: 'anthropic',
+    capabilities: {
+      supportsTools: true,
+      supportsSystemPrompt: true,
+      supportsReasoning: false,
+      supportsThinking: true,
+      maxContextWindow: 200000,
+      supportsStreaming: true,
+    },
+    defaults: { maxTokens: 16384, temperature: 0.2 },
+    benchmarkStatus: 'working',
+    benchmarkNotes: 'Fast and cheap. Struggles with multi-step reasoning.',
+  },
+
+  // ── OpenAI: GPT ──
+  'gpt-4o': {
+    family: 'gpt-4o',
+    provider: 'openai',
+    capabilities: {
+      supportsTools: true,
+      supportsSystemPrompt: true,
+      supportsReasoning: false,
+      supportsThinking: true,
+      maxContextWindow: 128000,
+      supportsStreaming: true,
+    },
+    defaults: { maxTokens: 32768, temperature: 0.2 },
+    benchmarkStatus: 'working',
+    benchmarkNotes: 'Reliable tool use. Good all-around performer.',
+  },
+  'gpt-4o-mini': {
+    family: 'gpt-4o-mini',
+    provider: 'openai',
+    capabilities: {
+      supportsTools: true,
+      supportsSystemPrompt: true,
+      supportsReasoning: false,
+      supportsThinking: true,
+      maxContextWindow: 128000,
+      supportsStreaming: true,
+    },
+    defaults: { maxTokens: 32768, temperature: 0.2 },
+    benchmarkStatus: 'working',
+    benchmarkNotes: 'Fast and cheap. Acceptable for simple scenarios.',
+  },
+  'gpt-4.1': {
+    family: 'gpt-4.1',
+    provider: 'openai',
+    capabilities: {
+      supportsTools: true,
+      supportsSystemPrompt: true,
+      supportsReasoning: false,
+      supportsThinking: true,
+      maxContextWindow: 1047576,
+      supportsStreaming: true,
+    },
+    defaults: { maxTokens: 65536, temperature: 0.2 },
+    benchmarkStatus: 'working',
+    benchmarkNotes: 'Large context window. Strong at complex scenarios.',
+  },
+
+  'gpt-5.1': {
+    family: 'gpt-5.1',
+    provider: 'openai',
+    capabilities: {
+      supportsTools: true,
+      supportsSystemPrompt: true,
+      supportsReasoning: false,
+      maxContextWindow: 1047576,
+      supportsStreaming: true,
+    },
+    defaults: { maxTokens: 32768 },
+    benchmarkStatus: 'untested',
+  },
+
+  // ── OpenAI: Reasoning ──
+  'o1': {
+    family: 'o1',
+    provider: 'openai',
+    capabilities: {
+      supportsTools: true,
+      supportsSystemPrompt: false,
+      supportsReasoning: true,
+      supportsThinking: true,
+      maxContextWindow: 200000,
+      supportsStreaming: false,
+    },
+    defaults: { maxTokens: 65536, reasoningEffort: 'medium' },
+    benchmarkStatus: 'degraded',
+    benchmarkNotes: 'No system prompt support. Tool calling works but slow.',
+  },
+  'o1-mini': {
+    family: 'o1-mini',
+    provider: 'openai',
+    capabilities: {
+      supportsTools: true,
+      supportsSystemPrompt: false,
+      supportsReasoning: true,
+      supportsThinking: true,
+      maxContextWindow: 128000,
+      supportsStreaming: false,
+    },
+    defaults: { maxTokens: 32768, reasoningEffort: 'medium' },
+    benchmarkStatus: 'degraded',
+    benchmarkNotes: 'No system prompt support. Cheaper but less reliable.',
+  },
+  'o1-preview': {
+    family: 'o1',
+    provider: 'openai',
+    capabilities: {
+      supportsTools: false,
+      supportsSystemPrompt: false,
+      supportsReasoning: true,
+      supportsThinking: true,
+      maxContextWindow: 128000,
+      supportsStreaming: false,
+    },
+    defaults: { maxTokens: 65536, reasoningEffort: 'medium' },
+    benchmarkStatus: 'broken',
+    benchmarkNotes: 'No tool calling support. Cannot complete agentic tasks.',
+  },
+  'o3-mini': {
+    family: 'o3-mini',
+    provider: 'openai',
+    capabilities: {
+      supportsTools: true,
+      supportsSystemPrompt: false,
+      supportsReasoning: true,
+      supportsThinking: true,
+      maxContextWindow: 200000,
+      supportsStreaming: false,
+    },
+    defaults: { maxTokens: 32768, reasoningEffort: 'medium' },
+    benchmarkStatus: 'working',
+    benchmarkNotes: 'Good reasoning, fast. No system prompt — task in user message.',
+  },
+  'o4-mini': {
+    family: 'o4-mini',
+    provider: 'openai',
+    capabilities: {
+      supportsTools: true,
+      supportsSystemPrompt: false,
+      supportsReasoning: true,
+      supportsThinking: true,
+      maxContextWindow: 200000,
+      supportsStreaming: false,
+    },
+    defaults: { maxTokens: 32768, reasoningEffort: 'medium' },
+    benchmarkStatus: 'untested',
+  },
+
+  // ── Gemini ──
+  'gemini-2.0-flash': {
+    family: 'gemini-flash',
+    provider: 'gemini',
+    capabilities: {
+      supportsTools: true,
+      supportsSystemPrompt: true,
+      supportsReasoning: false,
+      supportsThinking: true,
+      maxContextWindow: 1048576,
+      supportsStreaming: true,
+    },
+    defaults: { maxTokens: 16384, temperature: 0.2 },
+    benchmarkStatus: 'untested',
+  },
+  'gemini-2.5-pro': {
+    family: 'gemini-pro',
+    provider: 'gemini',
+    capabilities: {
+      supportsTools: true,
+      supportsSystemPrompt: true,
+      supportsReasoning: true,
+      supportsThinking: true,
+      maxContextWindow: 1048576,
+      supportsStreaming: true,
+    },
+    defaults: { maxTokens: 32768, temperature: 0.2 },
+    benchmarkStatus: 'untested',
+  },
+  'gemini-2.5-flash': {
+    family: 'gemini-flash',
+    provider: 'gemini',
+    capabilities: {
+      supportsTools: true,
+      supportsSystemPrompt: true,
+      supportsReasoning: true,
+      supportsThinking: true,
+      maxContextWindow: 1048576,
+      supportsStreaming: true,
+    },
+    defaults: { maxTokens: 16384, temperature: 0.2 },
+    benchmarkStatus: 'untested',
+  },
+
+  // ── Gemini 3.x ──
+  'gemini-3.0-pro': {
+    family: 'gemini-pro',
+    provider: 'gemini',
+    capabilities: {
+      supportsTools: true,
+      supportsSystemPrompt: true,
+      supportsReasoning: true,
+      supportsThinking: true,
+      maxContextWindow: 2097152,
+      supportsStreaming: true,
+    },
+    defaults: { maxTokens: 65536, temperature: 0.2 },
+    benchmarkStatus: 'untested',
+  },
+  'gemini-3.0-flash': {
+    family: 'gemini-flash',
+    provider: 'gemini',
+    capabilities: {
+      supportsTools: true,
+      supportsSystemPrompt: true,
+      supportsReasoning: true,
+      supportsThinking: true,
+      maxContextWindow: 2097152,
+      supportsStreaming: true,
+    },
+    defaults: { maxTokens: 32768, temperature: 0.2 },
+    benchmarkStatus: 'untested',
+  },
+};
+
+// ── Family defaults ─────────────────────────────────────────────────
+
+/** @type {Record<string, ModelConfig>} */
+const FAMILY_DEFAULTS = {
+  'claude-opus':   { maxTokens: 32768, temperature: 0.2 },
+  'claude-sonnet': { maxTokens: 32768, temperature: 0.2 },
+  'claude-haiku':  { maxTokens: 16384, temperature: 0.2 },
+  'gpt-4o':        { maxTokens: 32768, temperature: 0.2 },
+  'gpt-4o-mini':   { maxTokens: 32768, temperature: 0.2 },
+  'gpt-4.1':       { maxTokens: 65536, temperature: 0.2 },
+  'gpt-5.1':       { maxTokens: 32768 },
+  'o1':            { maxTokens: 65536, reasoningEffort: 'medium' },
+  'o1-mini':       { maxTokens: 32768, reasoningEffort: 'medium' },
+  'o3-mini':       { maxTokens: 32768, reasoningEffort: 'medium' },
+  'o4-mini':       { maxTokens: 32768, reasoningEffort: 'medium' },
+  'gemini-flash':  { maxTokens: 16384, temperature: 0.2 },
+  'gemini-pro':    { maxTokens: 32768, temperature: 0.2 },
+};
+
+/** @type {ModelConfig} */
+const GLOBAL_DEFAULTS = {
+  maxTokens: 32768,
+  temperature: 0.2,
+};
+
+// ── Lookup functions ────────────────────────────────────────────────
+
+/**
+ * Get the model info from the registry.
+ * Returns null for unknown models.
+ * @param {string} model
+ * @returns {ModelInfo | null}
+ */
+export function getModelInfo(model) {
+  return MODEL_REGISTRY[model] ?? null;
+}
+
+/**
+ * Get the capabilities of a model.
+ * Returns sensible defaults for unknown models.
+ * @param {string} model
+ * @returns {ModelCapabilities}
+ */
+export function getModelCapabilities(model) {
+  const info = MODEL_REGISTRY[model];
+  if (info) return info.capabilities;
+
+  // Sensible defaults for unknown models — assume thinking is supported
+  return {
+    supportsTools: true,
+    supportsSystemPrompt: true,
+    supportsReasoning: false,
+    supportsThinking: true,
+    maxContextWindow: 128000,
+    supportsStreaming: true,
+  };
+}
+
+/**
+ * Detect the model family from the model name.
+ * Tries exact registry lookup first, then prefix matching.
+ * @param {string} model
+ * @returns {string | null}
+ */
+export function detectModelFamily(model) {
+  const normalized = String(model ?? '').toLowerCase();
+  const info = MODEL_REGISTRY[normalized];
+  if (info) return info.family;
+
+  // Prefix-based heuristic for unregistered models
+  if (normalized.startsWith('claude-opus') || normalized.startsWith('opus-')) return 'claude-opus';
+  if (normalized.startsWith('claude-sonnet') || normalized.startsWith('sonnet-')) return 'claude-sonnet';
+  if (normalized.startsWith('claude-haiku') || normalized.startsWith('haiku-')) return 'claude-haiku';
+  if (normalized.startsWith('gpt-4o-mini')) return 'gpt-4o-mini';
+  if (normalized.startsWith('gpt-4o')) return 'gpt-4o';
+  if (normalized.startsWith('gpt-4.1')) return 'gpt-4.1';
+  if (normalized.startsWith('gpt-5')) return 'gpt-5.1';
+  if (normalized.startsWith('gpt-4')) return 'gpt-4o'; // assume 4o-class
+  if (normalized.startsWith('o1-mini')) return 'o1-mini';
+  if (normalized.startsWith('o1')) return 'o1';
+  if (normalized.startsWith('o3-mini')) return 'o3-mini';
+  if (normalized.startsWith('o4-mini')) return 'o4-mini';
+  if (normalized.startsWith('gemini') && normalized.includes('pro')) return 'gemini-pro';
+  if (normalized.startsWith('gemini') && normalized.includes('flash')) return 'gemini-flash';
+
+  return null;
+}
+
+// ── Config merge ────────────────────────────────────────────────────
+
+/**
+ * Parse env var overrides for model config.
+ * Only returns fields that are explicitly set.
+ * @returns {Partial<ModelConfig>}
+ */
+function getEnvOverrides() {
+  /** @type {Partial<ModelConfig>} */
+  const overrides = {};
+
+  const maxTokens = process.env['ARCHAL_MAX_TOKENS'];
+  if (maxTokens !== undefined && maxTokens !== '') {
+    const parsed = parseInt(maxTokens, 10);
+    if (!Number.isNaN(parsed) && parsed > 0) {
+      overrides.maxTokens = parsed;
+    }
+  }
+
+  const temperature = process.env['ARCHAL_TEMPERATURE'];
+  if (temperature !== undefined && temperature !== '') {
+    const parsed = parseFloat(temperature);
+    if (!Number.isNaN(parsed) && parsed >= 0 && parsed <= 2) {
+      overrides.temperature = parsed;
+    }
+  }
+
+  const reasoning = process.env['ARCHAL_REASONING_EFFORT'];
+  if (reasoning !== undefined && reasoning !== '') {
+    if (['low', 'medium', 'high'].includes(reasoning.toLowerCase())) {
+      overrides.reasoningEffort = reasoning.toLowerCase();
+    }
+  }
+
+  return overrides;
+}
+
+/**
+ * Get the merged configuration for a model.
+ * Priority: env var overrides > model-specific defaults > family defaults > global defaults.
+ *
+ * @param {string} model - Model identifier
+ * @returns {ModelConfig}
+ */
+export function getModelConfig(model) {
+  const family = detectModelFamily(model);
+  const familyDefaults = family ? (FAMILY_DEFAULTS[family] ?? {}) : {};
+  const modelDefaults = MODEL_REGISTRY[model]?.defaults ?? {};
+  const envOverrides = getEnvOverrides();
+
+  return {
+    ...GLOBAL_DEFAULTS,
+    ...familyDefaults,
+    ...modelDefaults,
+    ...envOverrides,
+  };
+}
+
+/**
+ * Check if a model is a reasoning model (o1, o3, o4 series).
+ * Reasoning models don't support temperature and use reasoning_effort instead.
+ * @param {string} model
+ * @returns {boolean}
+ */
+export function isReasoningModel(model) {
+  const info = MODEL_REGISTRY[model];
+  if (info) return info.capabilities.supportsReasoning;
+  // Fallback heuristic
+  return /^o[134]-/.test(model);
+}
+
+/**
+ * Check if a model supports extended thinking (Anthropic thinking blocks, Gemini thinking parts).
+ * @param {string} model
+ * @returns {boolean}
+ */
+export function isThinkingModel(model) {
+  const normalized = String(model ?? '').toLowerCase();
+  const info = MODEL_REGISTRY[normalized];
+  if (info) return info.capabilities.supportsThinking;
+  // Heuristic for unregistered models — most modern models support thinking
+  if (
+    normalized.startsWith('claude-')
+    || normalized.startsWith('sonnet-')
+    || normalized.startsWith('haiku-')
+    || normalized.startsWith('opus-')
+  ) return true;
+  if (normalized.startsWith('gemini-2.5') || normalized.startsWith('gemini-3')) return true;
+  if (normalized.startsWith('gpt-') || /^o[134]/.test(normalized)) return true;
+  return true; // default to true for unknown models
+}
+
+/**
+ * Get all known model names.
+ * @returns {string[]}
+ */
+export function listKnownModels() {
+  return Object.keys(MODEL_REGISTRY);
+}
+
+/**
+ * Get all known models grouped by benchmark status.
+ * @returns {Record<BenchmarkStatus, string[]>}
+ */
+export function listModelsByStatus() {
+  /** @type {Record<string, string[]>} */
+  const grouped = { working: [], degraded: [], broken: [], untested: [] };
+  for (const [name, info] of Object.entries(MODEL_REGISTRY)) {
+    grouped[info.benchmarkStatus].push(name);
+  }
+  return grouped;
+}

package/dist/harnesses/_lib/providers.mjs

@@ -0,0 +1,39 @@
+/**
+ * Shared provider detection and LLM calling for bundled harnesses.
+ *
+ * This is a thin re-export facade. Implementation lives in:
+ *   - llm-config.mjs   — provider detection, API key/base URL, timeout, thinking budget
+ *   - llm-call.mjs     — callLlm, callLlmWithMessages, LlmApiError, withRetry
+ *   - llm-response.mjs — response parsing, message building, tool formatting
+ */
+
+// ── Config ──────────────────────────────────────────────────────────
+export {
+  detectProvider,
+  resolveApiKey,
+  resolveBaseUrl,
+  isThinkingEnabled,
+} from './llm-config.mjs';
+
+// ── Calling ─────────────────────────────────────────────────────────
+export {
+  callLlm,
+  callLlmWithMessages,
+  LlmApiError,
+  withRetry,
+} from './llm-call.mjs';
+
+// ── Response parsing & message building ─────────────────────────────
+export {
+  extractTokenUsage,
+  formatToolsForProvider,
+  parseToolCalls,
+  getResponseText,
+  getThinkingContent,
+  getStopReason,
+  buildInitialMessages,
+  appendAssistantResponse,
+  appendToolResults,
+  appendUserInstruction,
+  extractCallArgs,
+} from './llm-response.mjs';