@archal/cli 0.5.1 → 0.6.1

package/harnesses/_lib/model-configs.mjs

@@ -0,0 +1,514 @@
+/**
+ * Model configuration system for bundled harnesses.
+ *
+ * Provides default configs per model family, known capabilities,
+ * and a merge function: hardcoded defaults -> model family defaults -> env overrides.
+ *
+ * Zero dependencies — pure data and functions.
+ */
+
+// ── Model capabilities ──────────────────────────────────────────────
+
+/**
+ * @typedef {Object} ModelCapabilities
+ * @property {boolean} supportsTools      - Can use function/tool calling
+ * @property {boolean} supportsSystemPrompt - Accepts a system prompt
+ * @property {boolean} supportsReasoning  - Has reasoning/thinking mode (o1, o3, etc.)
+ * @property {boolean} supportsThinking   - Has extended thinking / reasoning trace (Anthropic, Gemini 2.5)
+ * @property {number}  maxContextWindow   - Max context window in tokens
+ * @property {boolean} supportsStreaming   - Supports streaming responses
+ */
+
+/**
+ * @typedef {Object} ModelConfig
+ * @property {number}  [maxTokens]       - Max completion tokens
+ * @property {number}  [temperature]     - Sampling temperature
+ * @property {string}  [reasoningEffort] - For reasoning models: low/medium/high
+ * @property {number}  [topP]            - Top-p sampling
+ */
+
+/**
+ * @typedef {'working' | 'degraded' | 'broken' | 'untested'} BenchmarkStatus
+ */
+
+/**
+ * @typedef {Object} ModelInfo
+ * @property {string} family             - Model family key
+ * @property {string} provider           - Provider name
+ * @property {ModelCapabilities} capabilities
+ * @property {ModelConfig} defaults      - Default config for this model
+ * @property {BenchmarkStatus} benchmarkStatus - Status from benchmark testing
+ * @property {string} [benchmarkNotes]   - Notes about benchmark performance
+ */
+
+// ── Known model registry ────────────────────────────────────────────
+
+/** @type {Record<string, ModelInfo>} */
+const MODEL_REGISTRY = {
+  // ── Anthropic ──
+  'claude-opus-4-6': {
+    family: 'claude-opus',
+    provider: 'anthropic',
+    capabilities: {
+      supportsTools: true,
+      supportsSystemPrompt: true,
+      supportsReasoning: false,
+      supportsThinking: true,
+      maxContextWindow: 200000,
+      supportsStreaming: true,
+    },
+    defaults: { maxTokens: 32768, temperature: 0.2 },
+    benchmarkStatus: 'working',
+    benchmarkNotes: 'Top performer across all scenarios. Reliable tool use.',
+  },
+  'claude-sonnet-4-6': {
+    family: 'claude-sonnet',
+    provider: 'anthropic',
+    capabilities: {
+      supportsTools: true,
+      supportsSystemPrompt: true,
+      supportsReasoning: false,
+      supportsThinking: true,
+      maxContextWindow: 200000,
+      supportsStreaming: true,
+    },
+    defaults: { maxTokens: 32768, temperature: 0.2 },
+    benchmarkStatus: 'working',
+    benchmarkNotes: 'Strong performance, good cost/quality balance.',
+  },
+  'claude-sonnet-4-20250514': {
+    family: 'claude-sonnet',
+    provider: 'anthropic',
+    capabilities: {
+      supportsTools: true,
+      supportsSystemPrompt: true,
+      supportsReasoning: false,
+      supportsThinking: true,
+      maxContextWindow: 200000,
+      supportsStreaming: true,
+    },
+    defaults: { maxTokens: 32768, temperature: 0.2 },
+    benchmarkStatus: 'working',
+    benchmarkNotes: 'Solid tool use. Slightly behind claude-sonnet-4-6.',
+  },
+  'claude-haiku-4-5-20251001': {
+    family: 'claude-haiku',
+    provider: 'anthropic',
+    capabilities: {
+      supportsTools: true,
+      supportsSystemPrompt: true,
+      supportsReasoning: false,
+      supportsThinking: true,
+      maxContextWindow: 200000,
+      supportsStreaming: true,
+    },
+    defaults: { maxTokens: 16384, temperature: 0.2 },
+    benchmarkStatus: 'working',
+    benchmarkNotes: 'Fast and cheap. Struggles with multi-step reasoning.',
+  },
+
+  // ── OpenAI: GPT ──
+  'gpt-4o': {
+    family: 'gpt-4o',
+    provider: 'openai',
+    capabilities: {
+      supportsTools: true,
+      supportsSystemPrompt: true,
+      supportsReasoning: false,
+      supportsThinking: true,
+      maxContextWindow: 128000,
+      supportsStreaming: true,
+    },
+    defaults: { maxTokens: 32768, temperature: 0.2 },
+    benchmarkStatus: 'working',
+    benchmarkNotes: 'Reliable tool use. Good all-around performer.',
+  },
+  'gpt-4o-mini': {
+    family: 'gpt-4o-mini',
+    provider: 'openai',
+    capabilities: {
+      supportsTools: true,
+      supportsSystemPrompt: true,
+      supportsReasoning: false,
+      supportsThinking: true,
+      maxContextWindow: 128000,
+      supportsStreaming: true,
+    },
+    defaults: { maxTokens: 32768, temperature: 0.2 },
+    benchmarkStatus: 'working',
+    benchmarkNotes: 'Fast and cheap. Acceptable for simple scenarios.',
+  },
+  'gpt-4.1': {
+    family: 'gpt-4.1',
+    provider: 'openai',
+    capabilities: {
+      supportsTools: true,
+      supportsSystemPrompt: true,
+      supportsReasoning: false,
+      supportsThinking: true,
+      maxContextWindow: 1047576,
+      supportsStreaming: true,
+    },
+    defaults: { maxTokens: 65536, temperature: 0.2 },
+    benchmarkStatus: 'working',
+    benchmarkNotes: 'Large context window. Strong at complex scenarios.',
+  },
+
+  'gpt-5.1': {
+    family: 'gpt-5.1',
+    provider: 'openai',
+    capabilities: {
+      supportsTools: true,
+      supportsSystemPrompt: true,
+      supportsReasoning: false,
+      maxContextWindow: 1047576,
+      supportsStreaming: true,
+    },
+    defaults: { maxTokens: 32768, temperature: 0.0 },
+    benchmarkStatus: 'untested',
+  },
+
+  // ── OpenAI: Reasoning ──
+  'o1': {
+    family: 'o1',
+    provider: 'openai',
+    capabilities: {
+      supportsTools: true,
+      supportsSystemPrompt: false,
+      supportsReasoning: true,
+      supportsThinking: true,
+      maxContextWindow: 200000,
+      supportsStreaming: false,
+    },
+    defaults: { maxTokens: 65536, reasoningEffort: 'medium' },
+    benchmarkStatus: 'degraded',
+    benchmarkNotes: 'No system prompt support. Tool calling works but slow.',
+  },
+  'o1-mini': {
+    family: 'o1-mini',
+    provider: 'openai',
+    capabilities: {
+      supportsTools: true,
+      supportsSystemPrompt: false,
+      supportsReasoning: true,
+      supportsThinking: true,
+      maxContextWindow: 128000,
+      supportsStreaming: false,
+    },
+    defaults: { maxTokens: 32768, reasoningEffort: 'medium' },
+    benchmarkStatus: 'degraded',
+    benchmarkNotes: 'No system prompt support. Cheaper but less reliable.',
+  },
+  'o1-preview': {
+    family: 'o1',
+    provider: 'openai',
+    capabilities: {
+      supportsTools: false,
+      supportsSystemPrompt: false,
+      supportsReasoning: true,
+      supportsThinking: true,
+      maxContextWindow: 128000,
+      supportsStreaming: false,
+    },
+    defaults: { maxTokens: 65536, reasoningEffort: 'medium' },
+    benchmarkStatus: 'broken',
+    benchmarkNotes: 'No tool calling support. Cannot complete agentic tasks.',
+  },
+  'o3-mini': {
+    family: 'o3-mini',
+    provider: 'openai',
+    capabilities: {
+      supportsTools: true,
+      supportsSystemPrompt: false,
+      supportsReasoning: true,
+      supportsThinking: true,
+      maxContextWindow: 200000,
+      supportsStreaming: false,
+    },
+    defaults: { maxTokens: 32768, reasoningEffort: 'medium' },
+    benchmarkStatus: 'working',
+    benchmarkNotes: 'Good reasoning, fast. No system prompt — task in user message.',
+  },
+  'o4-mini': {
+    family: 'o4-mini',
+    provider: 'openai',
+    capabilities: {
+      supportsTools: true,
+      supportsSystemPrompt: false,
+      supportsReasoning: true,
+      supportsThinking: true,
+      maxContextWindow: 200000,
+      supportsStreaming: false,
+    },
+    defaults: { maxTokens: 32768, reasoningEffort: 'medium' },
+    benchmarkStatus: 'untested',
+  },
+
+  // ── Gemini ──
+  'gemini-2.0-flash': {
+    family: 'gemini-flash',
+    provider: 'gemini',
+    capabilities: {
+      supportsTools: true,
+      supportsSystemPrompt: true,
+      supportsReasoning: false,
+      supportsThinking: true,
+      maxContextWindow: 1048576,
+      supportsStreaming: true,
+    },
+    defaults: { maxTokens: 16384, temperature: 0.2 },
+    benchmarkStatus: 'untested',
+  },
+  'gemini-2.5-pro': {
+    family: 'gemini-pro',
+    provider: 'gemini',
+    capabilities: {
+      supportsTools: true,
+      supportsSystemPrompt: true,
+      supportsReasoning: true,
+      supportsThinking: true,
+      maxContextWindow: 1048576,
+      supportsStreaming: true,
+    },
+    defaults: { maxTokens: 32768, temperature: 0.2 },
+    benchmarkStatus: 'untested',
+  },
+  'gemini-2.5-flash': {
+    family: 'gemini-flash',
+    provider: 'gemini',
+    capabilities: {
+      supportsTools: true,
+      supportsSystemPrompt: true,
+      supportsReasoning: true,
+      supportsThinking: true,
+      maxContextWindow: 1048576,
+      supportsStreaming: true,
+    },
+    defaults: { maxTokens: 16384, temperature: 0.2 },
+    benchmarkStatus: 'untested',
+  },
+
+  // ── Gemini 3.x ──
+  'gemini-3.0-pro': {
+    family: 'gemini-pro',
+    provider: 'gemini',
+    capabilities: {
+      supportsTools: true,
+      supportsSystemPrompt: true,
+      supportsReasoning: true,
+      supportsThinking: true,
+      maxContextWindow: 2097152,
+      supportsStreaming: true,
+    },
+    defaults: { maxTokens: 65536, temperature: 0.2 },
+    benchmarkStatus: 'untested',
+  },
+  'gemini-3.0-flash': {
+    family: 'gemini-flash',
+    provider: 'gemini',
+    capabilities: {
+      supportsTools: true,
+      supportsSystemPrompt: true,
+      supportsReasoning: true,
+      supportsThinking: true,
+      maxContextWindow: 2097152,
+      supportsStreaming: true,
+    },
+    defaults: { maxTokens: 32768, temperature: 0.2 },
+    benchmarkStatus: 'untested',
+  },
+};
+
+// ── Family defaults ─────────────────────────────────────────────────
+
+/** @type {Record<string, ModelConfig>} */
+const FAMILY_DEFAULTS = {
+  'claude-opus':   { maxTokens: 32768, temperature: 0.2 },
+  'claude-sonnet': { maxTokens: 32768, temperature: 0.2 },
+  'claude-haiku':  { maxTokens: 16384, temperature: 0.2 },
+  'gpt-4o':        { maxTokens: 32768, temperature: 0.2 },
+  'gpt-4o-mini':   { maxTokens: 32768, temperature: 0.2 },
+  'gpt-4.1':       { maxTokens: 65536, temperature: 0.2 },
+  'gpt-5.1':       { maxTokens: 32768, temperature: 0.2 },
+  'o1':            { maxTokens: 65536, reasoningEffort: 'medium' },
+  'o1-mini':       { maxTokens: 32768, reasoningEffort: 'medium' },
+  'o3-mini':       { maxTokens: 32768, reasoningEffort: 'medium' },
+  'o4-mini':       { maxTokens: 32768, reasoningEffort: 'medium' },
+  'gemini-flash':  { maxTokens: 16384, temperature: 0.2 },
+  'gemini-pro':    { maxTokens: 32768, temperature: 0.2 },
+};
+
+/** @type {ModelConfig} */
+const GLOBAL_DEFAULTS = {
+  maxTokens: 32768,
+  temperature: 0.2,
+};
+
+// ── Lookup functions ────────────────────────────────────────────────
+
+/**
+ * Get the model info from the registry.
+ * Returns null for unknown models.
+ * @param {string} model
+ * @returns {ModelInfo | null}
+ */
+export function getModelInfo(model) {
+  return MODEL_REGISTRY[model] ?? null;
+}
+
+/**
+ * Get the capabilities of a model.
+ * Returns sensible defaults for unknown models.
+ * @param {string} model
+ * @returns {ModelCapabilities}
+ */
+export function getModelCapabilities(model) {
+  const info = MODEL_REGISTRY[model];
+  if (info) return info.capabilities;
+
+  // Sensible defaults for unknown models — assume thinking is supported
+  return {
+    supportsTools: true,
+    supportsSystemPrompt: true,
+    supportsReasoning: false,
+    supportsThinking: true,
+    maxContextWindow: 128000,
+    supportsStreaming: true,
+  };
+}
+
+/**
+ * Detect the model family from the model name.
+ * Tries exact registry lookup first, then prefix matching.
+ * @param {string} model
+ * @returns {string | null}
+ */
+export function detectModelFamily(model) {
+  const info = MODEL_REGISTRY[model];
+  if (info) return info.family;
+
+  // Prefix-based heuristic for unregistered models
+  if (model.startsWith('claude-opus')) return 'claude-opus';
+  if (model.startsWith('claude-sonnet')) return 'claude-sonnet';
+  if (model.startsWith('claude-haiku')) return 'claude-haiku';
+  if (model.startsWith('gpt-4o-mini')) return 'gpt-4o-mini';
+  if (model.startsWith('gpt-4o')) return 'gpt-4o';
+  if (model.startsWith('gpt-4.1')) return 'gpt-4.1';
+  if (model.startsWith('gpt-5')) return 'gpt-5.1';
+  if (model.startsWith('gpt-4')) return 'gpt-4o'; // assume 4o-class
+  if (model.startsWith('o1-mini')) return 'o1-mini';
+  if (model.startsWith('o1')) return 'o1';
+  if (model.startsWith('o3-mini')) return 'o3-mini';
+  if (model.startsWith('o4-mini')) return 'o4-mini';
+  if (model.startsWith('gemini') && model.includes('pro')) return 'gemini-pro';
+  if (model.startsWith('gemini') && model.includes('flash')) return 'gemini-flash';
+
+  return null;
+}
+
+// ── Config merge ────────────────────────────────────────────────────
+
+/**
+ * Parse env var overrides for model config.
+ * Only returns fields that are explicitly set.
+ * @returns {Partial<ModelConfig>}
+ */
+function getEnvOverrides() {
+  /** @type {Partial<ModelConfig>} */
+  const overrides = {};
+
+  const maxTokens = process.env['ARCHAL_MAX_TOKENS'];
+  if (maxTokens !== undefined && maxTokens !== '') {
+    const parsed = parseInt(maxTokens, 10);
+    if (!Number.isNaN(parsed) && parsed > 0) {
+      overrides.maxTokens = parsed;
+    }
+  }
+
+  const temperature = process.env['ARCHAL_TEMPERATURE'];
+  if (temperature !== undefined && temperature !== '') {
+    const parsed = parseFloat(temperature);
+    if (!Number.isNaN(parsed) && parsed >= 0 && parsed <= 2) {
+      overrides.temperature = parsed;
+    }
+  }
+
+  const reasoning = process.env['ARCHAL_REASONING_EFFORT'];
+  if (reasoning !== undefined && reasoning !== '') {
+    if (['low', 'medium', 'high'].includes(reasoning.toLowerCase())) {
+      overrides.reasoningEffort = reasoning.toLowerCase();
+    }
+  }
+
+  return overrides;
+}
+
+/**
+ * Get the merged configuration for a model.
+ * Priority: env var overrides > model-specific defaults > family defaults > global defaults.
+ *
+ * @param {string} model - Model identifier
+ * @returns {ModelConfig}
+ */
+export function getModelConfig(model) {
+  const family = detectModelFamily(model);
+  const familyDefaults = family ? (FAMILY_DEFAULTS[family] ?? {}) : {};
+  const modelDefaults = MODEL_REGISTRY[model]?.defaults ?? {};
+  const envOverrides = getEnvOverrides();
+
+  return {
+    ...GLOBAL_DEFAULTS,
+    ...familyDefaults,
+    ...modelDefaults,
+    ...envOverrides,
+  };
+}
+
+/**
+ * Check if a model is a reasoning model (o1, o3, o4 series).
+ * Reasoning models don't support temperature and use reasoning_effort instead.
+ * @param {string} model
+ * @returns {boolean}
+ */
+export function isReasoningModel(model) {
+  const info = MODEL_REGISTRY[model];
+  if (info) return info.capabilities.supportsReasoning;
+  // Fallback heuristic
+  return /^o[134]-/.test(model);
+}
+
+/**
+ * Check if a model supports extended thinking (Anthropic thinking blocks, Gemini thinking parts).
+ * @param {string} model
+ * @returns {boolean}
+ */
+export function isThinkingModel(model) {
+  const info = MODEL_REGISTRY[model];
+  if (info) return info.capabilities.supportsThinking;
+  // Heuristic for unregistered models — most modern models support thinking
+  if (model.startsWith('claude-')) return true;
+  if (model.startsWith('gemini-2.5') || model.startsWith('gemini-3')) return true;
+  if (model.startsWith('gpt-') || /^o[134]/.test(model)) return true;
+  return true; // default to true for unknown models
+}
+
+/**
+ * Get all known model names.
+ * @returns {string[]}
+ */
+export function listKnownModels() {
+  return Object.keys(MODEL_REGISTRY);
+}
+
+/**
+ * Get all known models grouped by benchmark status.
+ * @returns {Record<BenchmarkStatus, string[]>}
+ */
+export function listModelsByStatus() {
+  /** @type {Record<string, string[]>} */
+  const grouped = { working: [], degraded: [], broken: [], untested: [] };
+  for (const [name, info] of Object.entries(MODEL_REGISTRY)) {
+    grouped[info.benchmarkStatus].push(name);
+  }
+  return grouped;
+}