@archal/cli 0.8.0 → 0.9.1

package/harnesses/_lib/llm-call.mjs

@@ -0,0 +1,411 @@
+/**
+ * Unified LLM calling with provider dispatch, error handling, and retry logic.
+ *
+ * Extracted from providers.mjs — contains all HTTP-level concerns.
+ */
+
+import {
+  resolveBaseUrl,
+  getLlmTimeoutMs,
+  getAnthropicThinkingParam,
+  getGeminiThinkingConfig,
+  getModelConfig,
+  isReasoningModel,
+} from './llm-config.mjs';
+import { extractTokenUsage } from './llm-response.mjs';
+
+// ── Error handling ──────────────────────────────────────────────────
+
+/**
+ * Structured LLM API error with status code and retry-after support.
+ */
+export class LlmApiError extends Error {
+  /**
+   * @param {string} provider
+   * @param {number} status
+   * @param {string} responseText
+   * @param {Headers | null} [headers]
+   */
+  constructor(provider, status, responseText, headers) {
+    super(`${provider} API error ${status}: ${responseText.slice(0, 500)}`);
+    this.name = 'LlmApiError';
+    this.status = status;
+    this.provider = provider;
+    this.responseText = responseText;
+    this.retryAfterMs = parseRetryAfter(headers);
+  }
+}
+
+/**
+ * Parse the Retry-After header value into milliseconds.
+ * Supports both seconds (integer) and HTTP-date formats.
+ * Returns null if no valid Retry-After header is present.
+ * @param {Headers | null} [headers]
+ * @returns {number | null}
+ */
+function parseRetryAfter(headers) {
+  if (!headers) return null;
+  const value = headers.get?.('retry-after');
+  if (!value) return null;
+
+  // Try as integer (seconds)
+  const seconds = parseInt(value, 10);
+  if (!Number.isNaN(seconds) && seconds >= 0) {
+    return seconds * 1000;
+  }
+
+  // Try as HTTP-date
+  const date = new Date(value);
+  if (!Number.isNaN(date.getTime())) {
+    const delayMs = date.getTime() - Date.now();
+    return Math.max(0, delayMs);
+  }
+
+  return null;
+}
+
+// ── Timeout-aware fetch ─────────────────────────────────────────────
+
+/**
+ * Make an HTTP request with timeout via AbortController.
+ * @param {string} url
+ * @param {RequestInit} init
+ * @returns {Promise<Response>}
+ */
+async function fetchWithTimeout(url, init) {
+  const timeoutMs = getLlmTimeoutMs();
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+  try {
+    return await fetch(url, { ...init, signal: controller.signal });
+  } catch (err) {
+    if (err.name === 'AbortError') {
+      throw new LlmApiError('timeout', 0, `LLM call timed out after ${timeoutMs / 1000}s`, null);
+    }
+    throw err;
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+// ── Per-provider callers ────────────────────────────────────────────
+
+async function callGemini(model, apiKey, messages, tools) {
+  const baseUrl = resolveBaseUrl('gemini');
+  const url = `${baseUrl}/models/${model}:generateContent?key=${apiKey}`;
+  const config = getModelConfig(model);
+
+  const generationConfig = { maxOutputTokens: config.maxTokens };
+  if (config.temperature !== undefined && !isReasoningModel(model)) {
+    generationConfig.temperature = config.temperature;
+  }
+  const thinkingConfig = getGeminiThinkingConfig(model);
+  if (thinkingConfig) {
+    generationConfig.thinkingConfig = thinkingConfig;
+  }
+
+  const body = {
+    contents: messages,
+    generationConfig,
+  };
+  if (tools && tools.length > 0) {
+    body.tools = tools;
+  }
+  const res = await fetchWithTimeout(url, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify(body),
+  });
+  if (!res.ok) {
+    const text = await res.text();
+    throw new LlmApiError('Gemini', res.status, text, res.headers);
+  }
+  const responseBody = await res.json();
+  return {
+    body: responseBody,
+    usage: extractTokenUsage('gemini', responseBody),
+  };
+}
+
+async function callAnthropic(model, apiKey, messages, tools) {
+  const baseUrl = resolveBaseUrl('anthropic');
+  const url = `${baseUrl}/v1/messages`;
+  const config = getModelConfig(model);
+  const thinkingParam = getAnthropicThinkingParam(model);
+
+  const reqBody = {
+    model,
+    messages,
+    max_tokens: config.maxTokens,
+  };
+  if (thinkingParam) {
+    reqBody.thinking = thinkingParam;
+    // With thinking enabled, temperature must not be set
+  } else if (config.temperature !== undefined && !isReasoningModel(model)) {
+    reqBody.temperature = config.temperature;
+  }
+  if (tools && tools.length > 0) {
+    reqBody.tools = tools;
+    // With thinking enabled, tool_choice must be "auto" (not a specific tool)
+    if (thinkingParam) {
+      reqBody.tool_choice = { type: 'auto' };
+    }
+  }
+  const res = await fetchWithTimeout(url, {
+    method: 'POST',
+    headers: {
+      'x-api-key': apiKey,
+      'anthropic-version': '2023-06-01',
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify(reqBody),
+  });
+  if (!res.ok) {
+    const text = await res.text();
+    throw new LlmApiError('Anthropic', res.status, text, res.headers);
+  }
+  const responseBody = await res.json();
+  return {
+    body: responseBody,
+    usage: extractTokenUsage('anthropic', responseBody),
+  };
+}
+
+function isGpt5SeriesModel(model) {
+  return model.startsWith('gpt-5');
+}
+
+function shouldSendOpenAiTemperature(model) {
+  return !isReasoningModel(model) && !isGpt5SeriesModel(model);
+}
+
+function normalizeOpenAiConversation(messages) {
+  if (Array.isArray(messages)) {
+    return {
+      input: messages,
+      previousResponseId: undefined,
+    };
+  }
+  if (!messages || typeof messages !== 'object') {
+    return {
+      input: [],
+      previousResponseId: undefined,
+    };
+  }
+  return {
+    input: Array.isArray(messages.input) ? messages.input : [],
+    previousResponseId: typeof messages.previousResponseId === 'string'
+      ? messages.previousResponseId
+      : undefined,
+  };
+}
+
+async function callOpenAi(model, apiKey, messages, tools) {
+  const baseUrl = resolveBaseUrl('openai');
+  const url = `${baseUrl}/responses`;
+  const config = getModelConfig(model);
+  const conversation = normalizeOpenAiConversation(messages);
+
+  const reqBody = {
+    model,
+    input: conversation.input,
+    max_output_tokens: config.maxTokens,
+  };
+
+  if (conversation.previousResponseId) {
+    reqBody.previous_response_id = conversation.previousResponseId;
+  }
+
+  if (config.reasoningEffort && (isReasoningModel(model) || isGpt5SeriesModel(model))) {
+    reqBody.reasoning = { effort: config.reasoningEffort };
+  }
+
+  // GPT-5 series rejects temperature in many variants; never send it for gpt-5*.
+  if (shouldSendOpenAiTemperature(model) && config.temperature !== undefined) {
+    reqBody.temperature = config.temperature;
+  }
+
+  if (tools && tools.length > 0) {
+    reqBody.tools = tools;
+    reqBody.tool_choice = 'auto';
+  }
+
+  const res = await fetchWithTimeout(url, {
+    method: 'POST',
+    headers: {
+      'Authorization': `Bearer ${apiKey}`,
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify(reqBody),
+  });
+  if (!res.ok) {
+    const text = await res.text();
+    throw new LlmApiError('OpenAI', res.status, text, res.headers);
+  }
+  const responseBody = await res.json();
+  return {
+    body: responseBody,
+    usage: extractTokenUsage('openai', responseBody),
+  };
+}
+
+// ── Provider dispatch ───────────────────────────────────────────────
+
+const PROVIDER_CALLERS = {
+  gemini: callGemini,
+  anthropic: callAnthropic,
+  openai: callOpenAi,
+};
+
+/**
+ * Call the LLM with the given messages and tools.
+ * Returns an LlmResponse with the raw body and token usage.
+ * @param {'gemini' | 'anthropic' | 'openai'} provider
+ * @param {string} model
+ * @param {string} apiKey
+ * @param {Array | object} messages
+ * @param {Array} tools
+ * @returns {Promise<LlmResponse>}
+ */
+export async function callLlm(provider, model, apiKey, messages, tools) {
+  const caller = PROVIDER_CALLERS[provider] ?? callOpenAi;
+  return caller(model, apiKey, messages, tools);
+}
+
+/**
+ * Call the LLM with provider-appropriate message format.
+ * Returns an LlmResponse with body and token usage.
+ *
+ * For Anthropic, accepts { system, messages } wrapper and injects system prompt.
+ * For other providers, delegates to callLlm.
+ *
+ * @returns {Promise<LlmResponse>}
+ */
+export async function callLlmWithMessages(provider, model, apiKey, messagesOrWrapper, tools) {
+  if (provider === 'anthropic') {
+    const baseUrl = resolveBaseUrl('anthropic');
+    const url = `${baseUrl}/v1/messages`;
+    const config = getModelConfig(model);
+    const thinkingParam = getAnthropicThinkingParam(model);
+
+    const reqBody = {
+      model,
+      max_tokens: config.maxTokens,
+      messages: messagesOrWrapper.messages,
+    };
+    if (messagesOrWrapper.system) {
+      reqBody.system = messagesOrWrapper.system;
+    }
+    if (thinkingParam) {
+      reqBody.thinking = thinkingParam;
+      // With thinking enabled, temperature must not be set
+    } else if (config.temperature !== undefined && !isReasoningModel(model)) {
+      reqBody.temperature = config.temperature;
+    }
+    if (tools && tools.length > 0) {
+      reqBody.tools = tools;
+      if (thinkingParam) {
+        reqBody.tool_choice = { type: 'auto' };
+      }
+    }
+
+    const res = await fetchWithTimeout(url, {
+      method: 'POST',
+      headers: {
+        'x-api-key': apiKey,
+        'anthropic-version': '2023-06-01',
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify(reqBody),
+    });
+    if (!res.ok) {
+      const text = await res.text();
+      throw new LlmApiError('Anthropic', res.status, text, res.headers);
+    }
+    const responseBody = await res.json();
+    return {
+      body: responseBody,
+      usage: extractTokenUsage('anthropic', responseBody),
+    };
+  }
+
+  // Gemini uses flat message arrays; OpenAI accepts either arrays or wrapper state.
+  return callLlm(provider, model, apiKey, messagesOrWrapper, tools);
+}
+
+// ── Retry helper ────────────────────────────────────────────────────
+
+const RETRYABLE_STATUS_CODES = new Set([429, 500, 502, 503, 529]);
+
+/**
+ * Retry a function on transient errors with exponential backoff.
+ * Respects Retry-After headers from LlmApiError when available.
+ *
+ * @param {() => Promise<T>} fn
+ * @param {number} [maxRetries=3]
+ * @returns {Promise<T>}
+ * @template T
+ */
+export async function withRetry(fn, maxRetries = 3) {
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    try {
+      return await fn();
+    } catch (err) {
+      let isRetryable = false;
+
+      if (err instanceof LlmApiError) {
+        isRetryable = RETRYABLE_STATUS_CODES.has(err.status);
+        // Also retry on timeouts
+        if (err.status === 0 && err.message.includes('timed out')) {
+          isRetryable = true;
+        }
+      } else if (err.message) {
+        // Fallback: parse status from error message for backward compat
+        const statusMatch = err.message.match(/error (\d+)/);
+        if (statusMatch) {
+          isRetryable = RETRYABLE_STATUS_CODES.has(parseInt(statusMatch[1], 10));
+        }
+        if (err.message.includes('timed out')) {
+          isRetryable = true;
+        }
+      }
+
+      if (!isRetryable || attempt === maxRetries) throw err;
+
+      // Use retry-after header if available, then message body, then exponential backoff
+      let delay;
+      if (err instanceof LlmApiError && err.retryAfterMs !== null) {
+        delay = err.retryAfterMs;
+        // Cap retry-after at 90 seconds to avoid unreasonable waits
+        delay = Math.min(delay, 90_000);
+      } else if (err instanceof LlmApiError && err.status === 429) {
+        // OpenAI embeds wait time in the message body for TPM limits when
+        // no Retry-After header is present (e.g. batch/embedding endpoints):
+        // "Please try again in 14.902s."
+        const bodyMatch = err.responseText.match(/try again in (\d+(?:\.\d+)?)\s*s/i);
+        if (bodyMatch) {
+          delay = Math.ceil(parseFloat(bodyMatch[1]) * 1000) + 500; // +500ms buffer
+          delay = Math.min(delay, 90_000);
+        } else {
+          // Exponential backoff: 5s, 10s, 20s, 40s (capped at 60s) for 429
+          delay = Math.min(5000 * Math.pow(2, attempt), 60_000);
+        }
+      } else {
+        // Exponential backoff: 1s, 2s, 4s, 8s, 16s (capped at 30s)
+        delay = Math.min(1000 * Math.pow(2, attempt), 30_000);
+      }
+
+      // Add jitter: +/- 20%
+      const jitter = delay * 0.2 * (Math.random() * 2 - 1);
+      delay = Math.max(0, Math.round(delay + jitter));
+
+      process.stderr.write(
+        `[retry] Attempt ${attempt + 1}/${maxRetries} failed` +
+        `${err.status ? ` (${err.status})` : ''}, ` +
+        `retrying in ${(delay / 1000).toFixed(1)}s...\n`
+      );
+
+      await new Promise((r) => setTimeout(r, delay));
+    }
+  }
+}

package/harnesses/_lib/llm-config.mjs

@@ -0,0 +1,209 @@
+/**
+ * Provider detection, API key / base URL resolution, timeout parsing,
+ * and thinking budget configuration.
+ *
+ * Extracted from providers.mjs — pure config, no HTTP calls.
+ */
+
+import { getModelConfig, isReasoningModel, isThinkingModel, getModelCapabilities } from './model-configs.mjs';
+
+// ── Provider detection ──────────────────────────────────────────────
+
+/**
+ * Detect the LLM provider from the model name.
+ * @param {string} model
+ * @returns {'gemini' | 'anthropic' | 'openai'}
+ */
+export function detectProvider(model) {
+  const normalized = String(model ?? '').toLowerCase();
+  if (normalized.startsWith('gemini-')) return 'gemini';
+  if (
+    normalized.startsWith('claude-')
+    || normalized.startsWith('sonnet-')
+    || normalized.startsWith('haiku-')
+    || normalized.startsWith('opus-')
+  ) return 'anthropic';
+  if (
+    normalized.startsWith('gpt-') ||
+    /^o[134]/.test(normalized)
+  ) return 'openai';
+  // Default to OpenAI-compatible for unknown models
+  return 'openai';
+}
+
+const PROVIDER_ENV_VARS = {
+  gemini: 'GEMINI_API_KEY',
+  anthropic: 'ANTHROPIC_API_KEY',
+  openai: 'OPENAI_API_KEY',
+};
+
+function inferKeyProvider(key) {
+  if (!key) return null;
+  if (key.startsWith('AIza')) return 'gemini';
+  if (key.startsWith('sk-ant-')) return 'anthropic';
+  if (key.startsWith('sk-')) return 'openai';
+  return null;
+}
+
+/**
+ * Resolve the API key for the detected provider.
+ * Priority: ARCHAL_ENGINE_API_KEY > provider-specific env var.
+ * If ARCHAL_ENGINE_API_KEY clearly belongs to a different provider, fall back
+ * to provider-specific key when available, otherwise fail with a clear error.
+ * @param {string} provider
+ * @returns {string}
+ */
+export function resolveApiKey(provider) {
+  const envVar = PROVIDER_ENV_VARS[provider] ?? 'OPENAI_API_KEY';
+  const providerKey = process.env[envVar]?.trim();
+  const engineKey = process.env['ARCHAL_ENGINE_API_KEY']?.trim();
+  if (engineKey) {
+    const inferred = inferKeyProvider(engineKey);
+    if (!inferred || inferred === provider) return engineKey;
+    if (providerKey) {
+      process.stderr.write(
+        `[harness] Warning: ARCHAL_ENGINE_API_KEY appears to be for ${inferred}; using ${envVar} for ${provider} model.\n`,
+      );
+      return providerKey;
+    }
+    throw new Error(
+      `ARCHAL_ENGINE_API_KEY appears to be for ${inferred}, but provider "${provider}" requires ${envVar}. ` +
+      `Set ${envVar} or use a ${inferred} model.`
+    );
+  }
+  if (providerKey) return providerKey;
+
+  throw new Error(
+    `No API key found for provider "${provider}". ` +
+    `Set ${envVar} or ARCHAL_ENGINE_API_KEY environment variable, ` +
+    `or run: archal config set engine.apiKey <your-key>`
+  );
+}
+
+// ── Base URL resolution ─────────────────────────────────────────────
+
+const DEFAULT_BASE_URLS = {
+  openai: 'https://api.openai.com/v1',
+  anthropic: 'https://api.anthropic.com',
+  gemini: 'https://generativelanguage.googleapis.com/v1beta',
+};
+
+/**
+ * Resolve the base URL for a provider.
+ * Checks provider-specific env var override, then falls back to default.
+ * @param {'openai' | 'anthropic' | 'gemini'} provider
+ * @returns {string}
+ */
+export function resolveBaseUrl(provider) {
+  const envVars = {
+    openai: 'ARCHAL_OPENAI_BASE_URL',
+    anthropic: 'ARCHAL_ANTHROPIC_BASE_URL',
+    gemini: 'ARCHAL_GEMINI_BASE_URL',
+  };
+  const override = process.env[envVars[provider]]?.trim();
+  if (override) {
+    // Strip trailing slash for consistency
+    return override.replace(/\/+$/, '');
+  }
+  return DEFAULT_BASE_URLS[provider];
+}
+
+// ── Timeout ─────────────────────────────────────────────────────────
+
+/**
+ * Get the LLM call timeout in milliseconds.
+ * @returns {number}
+ */
+export function getLlmTimeoutMs() {
+  const envVal = process.env['ARCHAL_LLM_TIMEOUT'];
+  if (envVal !== undefined && envVal !== '') {
+    const parsed = parseInt(envVal, 10);
+    if (!Number.isNaN(parsed) && parsed > 0) {
+      return parsed * 1000;
+    }
+  }
+  return 180_000; // 180 seconds default
+}
+
+// ── Thinking configuration ──────────────────────────────────────────
+
+/**
+ * Parse the ARCHAL_THINKING_BUDGET env var.
+ * Defaults to "adaptive" (thinking on). Set to "off" to disable.
+ * @returns {null | 'adaptive' | number}
+ */
+export function parseThinkingBudget() {
+  const val = process.env['ARCHAL_THINKING_BUDGET']?.trim();
+  if (!val) return 'adaptive'; // thinking on by default
+  if (val.toLowerCase() === 'off' || val === '0') return null;
+  if (val.toLowerCase() === 'adaptive') return 'adaptive';
+  const parsed = parseInt(val, 10);
+  if (!Number.isNaN(parsed) && parsed > 0) return parsed;
+  return 'adaptive';
+}
+
+/**
+ * Build the Anthropic `thinking` request parameter for a model.
+ * Returns null if thinking should not be enabled.
+ *
+ * Opus 4.6: must use { type: "adaptive" } (type: "enabled" is deprecated).
+ * Other Claude models: use { type: "enabled", budget_tokens: N } or { type: "adaptive" }.
+ *
+ * @param {string} model
+ * @returns {object | null}
+ */
+export function getAnthropicThinkingParam(model) {
+  if (!isThinkingModel(model)) return null;
+  const budget = parseThinkingBudget();
+  if (budget === null) return null;
+
+  // Only 4.6 series models support adaptive thinking.
+  // Older models (claude-sonnet-4-20250514, claude-haiku-4-5-20251001) need
+  // { type: "enabled", budget_tokens: N } — "adaptive" returns a 400 error.
+  const normalized = String(model ?? '').toLowerCase();
+  const supportsAdaptive = normalized.includes('-4-6') || normalized.includes('4-6-');
+  const isOpus = normalized.startsWith('claude-opus') || normalized.startsWith('opus-');
+
+  if (isOpus || (supportsAdaptive && budget === 'adaptive')) {
+    return { type: 'adaptive' };
+  }
+
+  if (budget === 'adaptive') {
+    // For non-4.6 models with default "adaptive" budget, use a sensible fixed budget
+    return { type: 'enabled', budget_tokens: 10000 };
+  }
+
+  // Explicit numeric budget
+  return { type: 'enabled', budget_tokens: budget };
+}
+
+/**
+ * Build the Gemini thinkingConfig for generationConfig.
+ * Returns null if thinking should not be configured.
+ *
+ * @param {string} model
+ * @returns {object | null}
+ */
+export function getGeminiThinkingConfig(model) {
+  if (!isThinkingModel(model)) return null;
+  const budget = parseThinkingBudget();
+  if (budget === null) return null;
+
+  // Gemini 2.5 models think by default. An explicit budget overrides the default.
+  if (typeof budget === 'number') {
+    return { thinkingBudget: budget };
+  }
+  // "adaptive" — let Gemini use its default thinking behavior (no explicit config needed)
+  return null;
+}
+
+/**
+ * Check if extended thinking is enabled for the current run.
+ * @returns {boolean}
+ */
+export function isThinkingEnabled() {
+  return parseThinkingBudget() !== null;
+}
+
+// Re-export model-configs functions used by call/response modules
+export { getModelConfig, isReasoningModel, getModelCapabilities };