@archal/cli 0.7.12 → 0.9.0

package/dist/harnesses/_lib/llm-response.mjs

@@ -0,0 +1,483 @@
+/**
+ * Response parsing, message building, and tool formatting for each provider.
+ *
+ * Extracted from providers.mjs — pure functions, no HTTP calls.
+ */
+
+import { getModelCapabilities } from './model-configs.mjs';
+
+// ── Token usage tracking ────────────────────────────────────────────
+
+/**
+ * @typedef {Object} TokenUsage
+ * @property {number} inputTokens  - Input/prompt tokens used
+ * @property {number} outputTokens - Output/completion tokens used
+ */
+
+/**
+ * @typedef {Object} LlmResponse
+ * @property {object} body  - The raw API response body
+ * @property {TokenUsage} usage - Token usage for this call
+ */
+
+/**
+ * Extract token usage from a provider's response body.
+ * @param {'gemini' | 'anthropic' | 'openai'} provider
+ * @param {object} body
+ * @returns {TokenUsage}
+ */
+export function extractTokenUsage(provider, body) {
+  switch (provider) {
+    case 'gemini': {
+      const meta = body.usageMetadata ?? {};
+      return {
+        inputTokens: meta.promptTokenCount ?? 0,
+        outputTokens: meta.candidatesTokenCount ?? 0,
+      };
+    }
+    case 'anthropic': {
+      const usage = body.usage ?? {};
+      return {
+        inputTokens: usage.input_tokens ?? 0,
+        outputTokens: usage.output_tokens ?? 0,
+      };
+    }
+    case 'openai': {
+      const usage = body.usage ?? {};
+      return {
+        // Responses API uses input_tokens/output_tokens; Chat Completions uses prompt/completion tokens.
+        inputTokens: usage.input_tokens ?? usage.prompt_tokens ?? 0,
+        outputTokens: usage.output_tokens ?? usage.completion_tokens ?? 0,
+      };
+    }
+    default:
+      return { inputTokens: 0, outputTokens: 0 };
+  }
+}
+
+// ── Tool formatting ─────────────────────────────────────────────────
+
+/**
+ * Recursively strip JSON Schema keywords that the Gemini API rejects.
+ * Gemini does not support: additionalProperties, $schema, anyOf, oneOf, allOf.
+ */
+function sanitizeSchemaForGemini(schema) {
+  if (!schema || typeof schema !== 'object') return schema;
+  if (Array.isArray(schema)) return schema.map(sanitizeSchemaForGemini);
+
+  const cleaned = {};
+  for (const [key, value] of Object.entries(schema)) {
+    if (key === 'additionalProperties' || key === '$schema') continue;
+    // Gemini doesn't support anyOf/oneOf/allOf — flatten single-element unions,
+    // otherwise drop the keyword entirely (Gemini treats it as unknown).
+    if (key === 'anyOf' || key === 'oneOf' || key === 'allOf') {
+      if (Array.isArray(value) && value.length === 1) {
+        Object.assign(cleaned, sanitizeSchemaForGemini(value[0]));
+      }
+      // Multi-element unions are unsupported; skip the keyword
+      continue;
+    }
+    cleaned[key] = sanitizeSchemaForGemini(value);
+  }
+  return cleaned;
+}
+
+/**
+ * Convert MCP tool schemas to the format expected by each provider.
+ */
+export function formatToolsForProvider(provider, mcpTools) {
+  switch (provider) {
+    case 'gemini':
+      return [{
+        functionDeclarations: mcpTools.map((t) => ({
+          name: t.name,
+          description: t.description,
+          parameters: sanitizeSchemaForGemini(t.inputSchema),
+        })),
+      }];
+    case 'openai':
+      return mcpTools.map((t) => ({
+        type: 'function',
+        name: t.name,
+        description: t.description,
+        parameters: t.inputSchema,
+      }));
+    case 'anthropic':
+      return mcpTools.map((t) => ({
+        name: t.name,
+        description: t.description,
+        input_schema: t.inputSchema,
+      }));
+    default:
+      return mcpTools;
+  }
+}
+
+// ── Response parsing ────────────────────────────────────────────────
+
+/**
+ * Parse tool calls from the provider's response.
+ * Returns an array of { id, name, arguments } or null if no tool calls.
+ *
+ * Accepts either a raw response body or an LlmResponse wrapper.
+ */
+export function parseToolCalls(provider, responseOrWrapper) {
+  const response = responseOrWrapper?.body ?? responseOrWrapper;
+  switch (provider) {
+    case 'gemini':
+      return parseGeminiToolCalls(response);
+    case 'anthropic':
+      return parseAnthropicToolCalls(response);
+    case 'openai':
+      return parseOpenAiToolCalls(response);
+    default:
+      return parseOpenAiToolCalls(response);
+  }
+}
+
+function parseGeminiToolCalls(response) {
+  const parts = response.candidates?.[0]?.content?.parts ?? [];
+  const calls = parts
+    .filter((p) => p.functionCall)
+    .map((p) => ({
+      id: p.functionCall.name + '-' + Date.now(),
+      name: p.functionCall.name,
+      arguments: p.functionCall.args ?? {},
+    }));
+  return calls.length > 0 ? calls : null;
+}
+
+function parseAnthropicToolCalls(response) {
+  const content = response.content ?? [];
+  const calls = content
+    .filter((c) => c.type === 'tool_use')
+    .map((c) => ({
+      id: c.id,
+      name: c.name,
+      arguments: c.input ?? {},
+    }));
+  return calls.length > 0 ? calls : null;
+}
+
+function parseOpenAiToolCalls(response) {
+  const output = Array.isArray(response.output) ? response.output : [];
+  const calls = [];
+  for (const item of output) {
+    if (item?.type !== 'function_call') continue;
+
+    let parsedArguments = {};
+    if (typeof item.arguments === 'string' && item.arguments.trim()) {
+      try {
+        parsedArguments = JSON.parse(item.arguments);
+      } catch {
+        parsedArguments = { _raw: item.arguments };
+      }
+    } else if (item.arguments && typeof item.arguments === 'object') {
+      parsedArguments = item.arguments;
+    }
+
+    calls.push({
+      id: item.call_id ?? item.id ?? `${item.name ?? 'tool'}-${Date.now()}`,
+      name: item.name,
+      arguments: parsedArguments,
+    });
+  }
+
+  return calls.length > 0 ? calls : null;
+}
+
+/**
+ * Get the text content from the provider's response (if any).
+ *
+ * Accepts either a raw response body or an LlmResponse wrapper.
+ */
+export function getResponseText(provider, responseOrWrapper) {
+  const response = responseOrWrapper?.body ?? responseOrWrapper;
+  switch (provider) {
+    case 'gemini': {
+      const parts = response.candidates?.[0]?.content?.parts ?? [];
+      // Exclude thinking parts (thought === true) — those go to getThinkingContent()
+      const textParts = parts.filter((p) => p.text && !p.thought).map((p) => p.text);
+      return textParts.join('') || null;
+    }
+    case 'anthropic': {
+      const content = response.content ?? [];
+      const textBlocks = content.filter((c) => c.type === 'text').map((c) => c.text);
+      return textBlocks.join('') || null;
+    }
+    case 'openai': {
+      if (typeof response.output_text === 'string' && response.output_text.trim()) {
+        return response.output_text;
+      }
+      const output = Array.isArray(response.output) ? response.output : [];
+      const chunks = [];
+      for (const item of output) {
+        if (item?.type === 'output_text' && typeof item.text === 'string') {
+          chunks.push(item.text);
+          continue;
+        }
+        if (item?.type !== 'message' || !Array.isArray(item.content)) continue;
+        for (const part of item.content) {
+          if ((part?.type === 'output_text' || part?.type === 'text') && typeof part.text === 'string') {
+            chunks.push(part.text);
+          }
+        }
+      }
+      return chunks.join('') || null;
+    }
+    default:
+      return null;
+  }
+}
+
+/**
+ * Extract thinking/reasoning content from the provider's response.
+ * Returns the model's internal reasoning (Anthropic thinking blocks,
+ * Gemini thinking parts) or null if none.
+ *
+ * @param {'gemini' | 'anthropic' | 'openai'} provider
+ * @param {object} responseOrWrapper
+ * @returns {string | null}
+ */
+export function getThinkingContent(provider, responseOrWrapper) {
+  const response = responseOrWrapper?.body ?? responseOrWrapper;
+  switch (provider) {
+    case 'anthropic': {
+      const content = response.content ?? [];
+      const blocks = content
+        .filter((c) => c.type === 'thinking')
+        .map((c) => c.thinking);
+      return blocks.length > 0 ? blocks.join('\n') : null;
+    }
+    case 'openai': {
+      const output = Array.isArray(response.output) ? response.output : [];
+      const summaries = [];
+      for (const item of output) {
+        if (item?.type !== 'reasoning') continue;
+        if (Array.isArray(item.summary)) {
+          for (const summary of item.summary) {
+            if (typeof summary?.text === 'string' && summary.text.trim()) {
+              summaries.push(summary.text);
+            }
+          }
+        }
+      }
+      return summaries.length > 0 ? summaries.join('\n') : null;
+    }
+    case 'gemini': {
+      const parts = response.candidates?.[0]?.content?.parts ?? [];
+      const thinkingParts = parts
+        .filter((p) => p.thought === true)
+        .map((p) => p.text);
+      return thinkingParts.length > 0 ? thinkingParts.join('\n') : null;
+    }
+    default:
+      return null;
+  }
+}
+
+/**
+ * Get the stop reason from the provider's response.
+ * @param {'gemini' | 'anthropic' | 'openai'} provider
+ * @param {object} responseOrWrapper
+ * @returns {string | null}
+ */
+export function getStopReason(provider, responseOrWrapper) {
+  const response = responseOrWrapper?.body ?? responseOrWrapper;
+  switch (provider) {
+    case 'gemini':
+      return response.candidates?.[0]?.finishReason ?? null;
+    case 'anthropic':
+      return response.stop_reason ?? null;
+    case 'openai':
+      return parseOpenAiToolCalls(response) ? 'tool_calls' : (response.status ?? response.incomplete_details?.reason ?? null);
+    default:
+      return null;
+  }
+}
+
+// ── Message formatting ──────────────────────────────────────────────
+
+/**
+ * Build the initial messages array with system prompt and task for the provider.
+ * For reasoning models that don't support system prompts, the system prompt
+ * is prepended to the user message automatically.
+ *
+ * @param {'gemini' | 'anthropic' | 'openai'} provider
+ * @param {string} systemPrompt
+ * @param {string} task
+ * @param {string} [model] - Optional model name for capability checking
+ */
+export function buildInitialMessages(provider, systemPrompt, task, model) {
+  const capabilities = model ? getModelCapabilities(model) : null;
+  const supportsSystem = capabilities ? capabilities.supportsSystemPrompt : true;
+
+  switch (provider) {
+    case 'gemini':
+      return [
+        { role: 'user', parts: [{ text: (systemPrompt ? systemPrompt + '\n\n' : '') + task }] },
+      ];
+    case 'anthropic':
+      return {
+        system: systemPrompt || undefined,
+        messages: [{ role: 'user', content: task }],
+      };
+    case 'openai': {
+      let input;
+      if (!supportsSystem || !systemPrompt) {
+        // Reasoning models (o1, o3, o4) don't support system prompts.
+        // Merge system prompt into user message.
+        const combined = systemPrompt ? systemPrompt + '\n\n' + task : task;
+        input = [{ role: 'user', content: combined }];
+      } else {
+        input = [
+          { role: 'system', content: systemPrompt },
+          { role: 'user', content: task },
+        ];
+      }
+      return {
+        input,
+        previousResponseId: undefined,
+      };
+    }
+    default:
+      return [
+        { role: 'system', content: systemPrompt },
+        { role: 'user', content: task },
+      ];
+  }
+}
+
+/**
+ * Append the assistant response to the conversation for the next turn.
+ *
+ * Accepts either a raw response body or an LlmResponse wrapper.
+ */
+export function appendAssistantResponse(provider, messages, responseOrWrapper) {
+  const response = responseOrWrapper?.body ?? responseOrWrapper;
+  switch (provider) {
+    case 'gemini': {
+      const content = response.candidates?.[0]?.content;
+      if (content) messages.push(content);
+      return messages;
+    }
+    case 'anthropic': {
+      messages.messages.push({ role: 'assistant', content: response.content });
+      return messages;
+    }
+    case 'openai': {
+      if (Array.isArray(messages)) {
+        const text = getResponseText('openai', response);
+        messages.push({ role: 'assistant', content: text ?? '' });
+        return messages;
+      }
+      messages.previousResponseId = response.id ?? messages.previousResponseId;
+      messages.input = [];
+      return messages;
+    }
+    default:
+      return messages;
+  }
+}
+
+/**
+ * Append tool results to the conversation for the next turn.
+ */
+export function appendToolResults(provider, messages, toolCalls, results) {
+  switch (provider) {
+    case 'gemini': {
+      const parts = toolCalls.map((tc, i) => ({
+        functionResponse: {
+          name: tc.name,
+          response: { content: results[i] },
+        },
+      }));
+      messages.push({ role: 'user', parts });
+      return messages;
+    }
+    case 'anthropic': {
+      const content = toolCalls.map((tc, i) => ({
+        type: 'tool_result',
+        tool_use_id: tc.id,
+        content: results[i],
+      }));
+      messages.messages.push({ role: 'user', content });
+      return messages;
+    }
+    case 'openai': {
+      const toolOutputs = [];
+      for (let i = 0; i < toolCalls.length; i++) {
+        const output = typeof results[i] === 'string'
+          ? results[i]
+          : JSON.stringify(results[i]);
+        toolOutputs.push({
+          type: 'function_call_output',
+          call_id: toolCalls[i].id,
+          output,
+        });
+      }
+
+      if (Array.isArray(messages)) {
+        for (let i = 0; i < toolCalls.length; i++) {
+          messages.push({
+            role: 'tool',
+            tool_call_id: toolCalls[i].id,
+            content: toolOutputs[i].output,
+          });
+        }
+        return messages;
+      }
+
+      messages.input = toolOutputs;
+      return messages;
+    }
+    default:
+      return messages;
+  }
+}
+
+/**
+ * Append a plain-text user instruction for the next turn.
+ * Used for harness-level recovery nudges (for example, when the model
+ * responds without any tool calls before taking required actions).
+ *
+ * @param {'gemini' | 'anthropic' | 'openai'} provider
+ * @param {Array | object} messages
+ * @param {string} text
+ * @returns {Array | object}
+ */
+export function appendUserInstruction(provider, messages, text) {
+  switch (provider) {
+    case 'gemini': {
+      messages.push({ role: 'user', parts: [{ text }] });
+      return messages;
+    }
+    case 'anthropic': {
+      messages.messages.push({ role: 'user', content: text });
+      return messages;
+    }
+    case 'openai': {
+      if (Array.isArray(messages)) {
+        messages.push({ role: 'user', content: text });
+        return messages;
+      }
+      const nextInput = Array.isArray(messages.input) ? [...messages.input] : [];
+      nextInput.push({ role: 'user', content: text });
+      messages.input = nextInput;
+      return messages;
+    }
+    default:
+      return messages;
+  }
+}
+
+/**
+ * Extract the messages array and system prompt for the callLlm function.
+ * For Anthropic, the system prompt is separate from messages.
+ */
+export function extractCallArgs(provider, messages) {
+  if (provider === 'anthropic') {
+    return { system: messages.system, messages: messages.messages };
+  }
+  return { messages };
+}

package/dist/harnesses/_lib/logging.mjs

@@ -0,0 +1,176 @@
+/**
+ * Structured logging helper for bundled harnesses.
+ * Outputs JSON lines (one JSON object per line) to stderr.
+ *
+ * Each log line includes: timestamp, iteration, model, provider, event type,
+ * and event-specific fields.
+ *
+ * Log levels: debug, info, warn, error
+ * Controlled via ARCHAL_LOG_LEVEL env var (default: info).
+ */
+
+// ── Log levels ──────────────────────────────────────────────────────
+
+/** @enum {number} */
+const LOG_LEVELS = {
+  debug: 0,
+  info: 1,
+  warn: 2,
+  error: 3,
+};
+
+const currentLevel = LOG_LEVELS[process.env['ARCHAL_LOG_LEVEL']?.toLowerCase() ?? 'info'] ?? LOG_LEVELS.info;
+
+// ── Logger factory ──────────────────────────────────────────────────
+
+/**
+ * @typedef {Object} LogContext
+ * @property {string} harness  - Harness name (e.g. "react")
+ * @property {string} model    - Model identifier
+ * @property {string} provider - Provider name
+ */
+
+/**
+ * @typedef {Object} Logger
+ * @property {function} debug - Log at debug level
+ * @property {function} info  - Log at info level
+ * @property {function} warn  - Log at warn level
+ * @property {function} error - Log at error level
+ * @property {function} tokenUsage   - Log token usage event
+ * @property {function} toolCall     - Log tool call event
+ * @property {function} toolError    - Log tool error event
+ * @property {function} llmCall      - Log LLM call event
+ * @property {function} llmResponse  - Log LLM response event
+ * @property {function} summary      - Log run summary event
+ */
+
+/**
+ * Create a structured logger bound to a harness context.
+ * @param {LogContext} context
+ * @returns {Logger}
+ */
+export function createLogger(context) {
+  const { harness, model, provider } = context;
+
+  /**
+   * Write a structured log line to stderr.
+   * @param {'debug' | 'info' | 'warn' | 'error'} level
+   * @param {string} event
+   * @param {Record<string, unknown>} [fields]
+   * @param {number} [iteration]
+   */
+  function log(level, event, fields = {}, iteration = undefined) {
+    if (LOG_LEVELS[level] < currentLevel) return;
+
+    const line = {
+      ts: new Date().toISOString(),
+      level,
+      harness,
+      model,
+      provider,
+      event,
+      ...(iteration !== undefined ? { iteration } : {}),
+      ...fields,
+    };
+    process.stderr.write(JSON.stringify(line) + '\n');
+  }
+
+  return {
+    debug: (event, fields, iteration) => log('debug', event, fields, iteration),
+    info: (event, fields, iteration) => log('info', event, fields, iteration),
+    warn: (event, fields, iteration) => log('warn', event, fields, iteration),
+    error: (event, fields, iteration) => log('error', event, fields, iteration),
+
+    /**
+     * Log token usage for an LLM call.
+     * @param {number} iteration
+     * @param {object} usage - { inputTokens, outputTokens }
+     * @param {object} cumulative - { inputTokens, outputTokens }
+     */
+    tokenUsage(iteration, usage, cumulative) {
+      log('info', 'token_usage', {
+        inputTokens: usage.inputTokens,
+        outputTokens: usage.outputTokens,
+        cumulativeInputTokens: cumulative.inputTokens,
+        cumulativeOutputTokens: cumulative.outputTokens,
+      }, iteration);
+    },
+
+    /**
+     * Log a tool call.
+     * @param {number} iteration
+     * @param {string} toolName
+     * @param {object} args - Tool arguments (truncated)
+     * @param {number} durationMs
+     */
+    toolCall(iteration, toolName, args, durationMs) {
+      log('info', 'tool_call', {
+        tool: toolName,
+        args: truncate(JSON.stringify(args), 200),
+        durationMs,
+      }, iteration);
+    },
+
+    /**
+     * Log a tool error.
+     * @param {number} iteration
+     * @param {string} toolName
+     * @param {string} errorMessage
+     */
+    toolError(iteration, toolName, errorMessage) {
+      log('error', 'tool_error', {
+        tool: toolName,
+        error: truncate(errorMessage, 500),
+      }, iteration);
+    },
+
+    /**
+     * Log an LLM call start.
+     * @param {number} iteration
+     */
+    llmCall(iteration) {
+      log('debug', 'llm_call_start', {}, iteration);
+    },
+
+    /**
+     * Log an LLM response.
+     * @param {number} iteration
+     * @param {number} durationMs
+     * @param {boolean} hasToolCalls
+     * @param {string|null} stopReason
+     */
+    llmResponse(iteration, durationMs, hasToolCalls, stopReason) {
+      log('info', 'llm_response', {
+        durationMs,
+        hasToolCalls,
+        ...(stopReason ? { stopReason } : {}),
+      }, iteration);
+    },
+
+    /**
+     * Log a run summary at the end.
+     * @param {object} stats
+     * @param {number} stats.iterations
+     * @param {number} stats.totalInputTokens
+     * @param {number} stats.totalOutputTokens
+     * @param {number} stats.totalTimeMs
+     * @param {number} stats.toolCallCount
+     * @param {number} stats.toolErrorCount
+     * @param {string} stats.exitReason
+     */
+    summary(stats) {
+      log('info', 'run_summary', stats);
+    },
+  };
+}
+
+/**
+ * Truncate a string to a maximum length with ellipsis.
+ * @param {string} str
+ * @param {number} maxLen
+ * @returns {string}
+ */
+function truncate(str, maxLen) {
+  if (str.length <= maxLen) return str;
+  return str.slice(0, maxLen - 3) + '...';
+}