claude-autorouter 0.3.7 → 0.5.0

package/src/config.mjs

@@ -1,15 +1,12 @@
+// @ts-check
 import { DEFAULT_OLLAMA_MODEL, defaultOllamaTimeoutMs, validateOllamaEndpoint, validateOllamaModel } from './ollama-models.mjs';
 import { resolve } from 'node:path';
+import { supportsAutoMode } from './model-catalog.mjs';
 
+/** @type {import('./contracts.mjs').Tier[]} */
 export const TIERS = ['haiku', 'sonnet', 'opus'];
+/** @type {import('./contracts.mjs').ClientProfile[]} */
 export const CLIENT_PROFILES = ['compatible', 'native', 'auto'];
-// Exact Anthropic models whose Auto permission-mode support is documented.
-// Custom aliases are not proof of the capabilities of their upstream model.
-const AUTO_MODE_MODELS = new Set([
-  'claude-sonnet-4-6', 'claude-sonnet-5', 'claude-sonnet-5-5',
-  'claude-opus-4-6', 'claude-opus-4-7', 'claude-opus-4-8', 'claude-opus-5', 'claude-opus-5-5',
-]);
-
 export function parseStopHookBlockCap(value, name = 'CLAUDE_CODE_STOP_HOOK_BLOCK_CAP') {
   if (!['string', 'number'].includes(typeof value)
     || (typeof value === 'string' && !/^[0-9]+$/.test(value.trim()))
@@ -28,15 +25,19 @@ export function parseSessionLogDir(value, name = 'AUTOROUTER_SESSION_LOG_DIR') {
 }
 
 function number(env, key, fallback, min, max, integer = true) {
-  const value = Number(env[key] ?? fallback);
-  if (!Number.isFinite(value) || value < min || value > max || (integer && !Number.isInteger(value))) {
+  const raw = env[key] === undefined ? fallback : env[key];
+  const pattern = integer ? /^[0-9]+$/ : /^(?:[0-9]+(?:\.[0-9]+)?|\.[0-9]+)$/;
+  const value = Number(raw);
+  if (!['string', 'number'].includes(typeof raw) || (typeof raw === 'string' && !pattern.test(raw.trim()))
+    || !Number.isFinite(value) || value < min || value > max || (integer && !Number.isSafeInteger(value))) {
     throw new Error(`${key} must be ${integer ? 'an integer' : 'a number'} between ${min} and ${max}`);
   }
   return value;
 }
 
 function endpoint(value, name) {
-  const url = new URL(value);
+  let url;
+  try { url = new URL(value); } catch { throw new Error(`${name} must be a valid URL`); }
   const local = ['127.0.0.1', '[::1]', 'localhost'].includes(url.hostname);
   if ((url.protocol !== 'https:' && !(local && url.protocol === 'http:')) || url.username || url.password || url.search || url.hash) {
     throw new Error(`${name} must use HTTPS (HTTP is allowed on loopback) with no credentials, query, or fragment`);
@@ -44,37 +45,59 @@ function endpoint(value, name) {
   return url.href.replace(/\/$/, '');
 }
 
-export function readConfig(env = process.env) {
-  const evaluator = env.AUTOROUTER_EVALUATOR ?? 'jev';
-  if (!['jev', 'ollama'].includes(evaluator)) throw new Error('AUTOROUTER_EVALUATOR must be jev or ollama');
-  const ollamaModel = validateOllamaModel(env.AUTOROUTER_OLLAMA_MODEL ?? DEFAULT_OLLAMA_MODEL);
-  const rawOllamaTimeout = env.AUTOROUTER_OLLAMA_TIMEOUT_MS;
+function modelName(value, name) {
+  if (typeof value !== 'string' || !value.trim() || /[\u0000-\u001f\u007f-\u009f]/.test(value)) {
+    throw new Error(`${name} must be a nonempty model name without control characters`);
+  }
+  return value;
+}
+
+/**
+ * @param {Record<string,string|undefined>} [env]
+ * @param {{validateAll?:boolean}} [options]
+ * @returns {import('./contracts.mjs').RouterConfig}
+ */
+export function readConfig(env = process.env, { validateAll = false } = {}) {
+  const sessionLogMode = env.AUTOROUTER_SESSION_LOG_MODE === undefined ? 'prompts' : env.AUTOROUTER_SESSION_LOG_MODE;
+  if (sessionLogMode !== 'metadata' && sessionLogMode !== 'prompts') throw new Error('AUTOROUTER_SESSION_LOG_MODE must be metadata or prompts');
+  for (const key of ['AUTOROUTER_STATUSLINE', 'AUTOROUTER_DEBUG']) {
+    if (env[key] !== undefined && !['0', '1'].includes(env[key])) throw new Error(`${key} must be 0 or 1`);
+  }
+  const evaluator = env.AUTOROUTER_EVALUATOR ?? 'ollama';
+  if (evaluator !== 'jev' && evaluator !== 'ollama') throw new Error('AUTOROUTER_EVALUATOR must be jev or ollama');
+  // A stale inactive backend must not stop the selected evaluator. Config
+  // inspection can request validation of both providers explicitly.
+  const ollamaEnv = evaluator === 'ollama' || validateAll ? env : {};
+  const jevEnv = evaluator === 'jev' || validateAll ? env : {};
+  const ollamaModel = validateOllamaModel(ollamaEnv.AUTOROUTER_OLLAMA_MODEL ?? DEFAULT_OLLAMA_MODEL);
+  const rawOllamaTimeout = ollamaEnv.AUTOROUTER_OLLAMA_TIMEOUT_MS;
   // Zero is an explicit opt-out. Reject blanks, coercible non-numbers, and
   // non-integer strings (including exponents that could underflow to zero).
   if (rawOllamaTimeout !== undefined && (!['string', 'number'].includes(typeof rawOllamaTimeout)
     || (typeof rawOllamaTimeout === 'string' && !/^[0-9]+$/.test(rawOllamaTimeout.trim())))) {
     throw new Error('AUTOROUTER_OLLAMA_TIMEOUT_MS must be an integer between 0 and 30000 (0 disables the runtime deadline)');
   }
-  const ollamaKeepAlive = env.AUTOROUTER_OLLAMA_KEEP_ALIVE ?? '5m';
+  const ollamaKeepAlive = ollamaEnv.AUTOROUTER_OLLAMA_KEEP_ALIVE ?? '5m';
   if (!/^(?:0|[1-9]\d{0,3}(?:s|m|h))$/.test(ollamaKeepAlive)) {
     throw new Error('AUTOROUTER_OLLAMA_KEEP_ALIVE must be 0 or a positive duration such as 5m');
   }
   const authMode = env.AUTOROUTER_AUTH_MODE ?? 'api-key';
-  if (!['api-key', 'subscription'].includes(authMode)) {
+  if (authMode !== 'api-key' && authMode !== 'subscription') {
     throw new Error('AUTOROUTER_AUTH_MODE must be api-key or subscription');
   }
   const clientProfile = env.AUTOROUTER_CLIENT_PROFILE ?? 'compatible';
-  if (!CLIENT_PROFILES.includes(clientProfile)) {
+  if (clientProfile !== 'compatible' && clientProfile !== 'native' && clientProfile !== 'auto') {
     throw new Error('AUTOROUTER_CLIENT_PROFILE must be compatible, native or auto');
   }
   const models = {
-    haiku: env.AUTOROUTER_HAIKU_MODEL ?? 'claude-haiku-4-5-20251001',
-    sonnet: env.AUTOROUTER_SONNET_MODEL ?? (clientProfile === 'auto' ? 'claude-sonnet-5-5' : 'claude-sonnet-5'),
-    opus: env.AUTOROUTER_OPUS_MODEL ?? 'claude-opus-5-5',
+    haiku: env.AUTOROUTER_HAIKU_MODEL === undefined ? 'claude-haiku-4-5-20251001' : env.AUTOROUTER_HAIKU_MODEL,
+    sonnet: env.AUTOROUTER_SONNET_MODEL === undefined ? (clientProfile === 'auto' ? 'claude-sonnet-5-5' : 'claude-sonnet-5') : env.AUTOROUTER_SONNET_MODEL,
+    opus: env.AUTOROUTER_OPUS_MODEL === undefined ? 'claude-opus-5-5' : env.AUTOROUTER_OPUS_MODEL,
   };
+  for (const tier of TIERS) modelName(models[tier], `AUTOROUTER_${tier.toUpperCase()}_MODEL`);
   if (clientProfile === 'auto') {
     for (const tier of ['sonnet', 'opus']) {
-      if (!AUTO_MODE_MODELS.has(models[tier])) {
+      if (!supportsAutoMode(models[tier])) {
         throw new Error(`AUTOROUTER_${tier.toUpperCase()}_MODEL must be a known Auto-mode-capable Sonnet or Opus model for the auto profile`);
       }
     }
@@ -88,24 +111,25 @@ export function readConfig(env = process.env) {
     authMode,
     clientProfile,
     sessionLogDir: parseSessionLogDir(env.AUTOROUTER_SESSION_LOG_DIR),
+    sessionLogMode,
     stopHookBlockCap: env.CLAUDE_CODE_STOP_HOOK_BLOCK_CAP === undefined
       ? undefined : parseStopHookBlockCap(env.CLAUDE_CODE_STOP_HOOK_BLOCK_CAP),
     anthropicKey: authMode === 'api-key' ? env.ANTHROPIC_API_KEY : undefined,
     jevKey: env.TYPESAFE_API_KEY,
     localToken: env.AUTOROUTER_TOKEN,
     upstream,
-    jevEndpoint: endpoint(env.AUTOROUTER_JEV_URL ?? 'https://api.typesafe.ai/v1/systemone', 'AUTOROUTER_JEV_URL'),
-    jevModel: env.AUTOROUTER_JEV_MODEL ?? 'jev-latest',
-    ollamaEndpoint: validateOllamaEndpoint(env.AUTOROUTER_OLLAMA_URL ?? 'http://127.0.0.1:11434'),
+    jevEndpoint: endpoint(jevEnv.AUTOROUTER_JEV_URL ?? 'https://api.typesafe.ai/v1/systemone', 'AUTOROUTER_JEV_URL'),
+    jevModel: modelName(jevEnv.AUTOROUTER_JEV_MODEL === undefined ? 'jev-latest' : jevEnv.AUTOROUTER_JEV_MODEL, 'AUTOROUTER_JEV_MODEL'),
+    ollamaEndpoint: validateOllamaEndpoint(ollamaEnv.AUTOROUTER_OLLAMA_URL ?? 'http://127.0.0.1:11434'),
     ollamaModel,
-    ollamaTimeoutMs: number(env, 'AUTOROUTER_OLLAMA_TIMEOUT_MS', defaultOllamaTimeoutMs(ollamaModel), 0, 30000),
+    ollamaTimeoutMs: number(ollamaEnv, 'AUTOROUTER_OLLAMA_TIMEOUT_MS', defaultOllamaTimeoutMs(ollamaModel), 0, 30000),
     ollamaStateChars: 3000,
     ollamaKeepAlive,
     models,
     port: number(env, 'AUTOROUTER_PORT', 8787, 0, 65535),
-    jevTimeoutMs: number(env, 'AUTOROUTER_JEV_TIMEOUT_MS', 1500, 1, 10000),
+    jevTimeoutMs: number(jevEnv, 'AUTOROUTER_JEV_TIMEOUT_MS', 1500, 1, 10000),
     tokenCountTimeoutMs: number(env, 'AUTOROUTER_TOKEN_COUNT_TIMEOUT_MS', 1500, 1, 10000),
-    minConfidence: number(env, 'AUTOROUTER_MIN_CONFIDENCE', 0.75, 0, 1, false),
+    minConfidence: number(jevEnv, 'AUTOROUTER_MIN_CONFIDENCE', 0.75, 0, 1, false),
     stateChars: 12000,
     maxBodyBytes: 32 * 1024 * 1024,
     cacheEntries: 1000,
@@ -115,6 +139,7 @@ export function readConfig(env = process.env) {
   };
 }
 
+/** @param {import('./contracts.mjs').RouterConfig} config */
 export function requireKeys(config) {
   const keys = config.evaluator === 'ollama' ? [] : [['TYPESAFE_API_KEY', config.jevKey]];
   if (config.authMode === 'api-key') keys.push(['ANTHROPIC_API_KEY', config.anthropicKey]);

package/src/contracts.mjs

@@ -0,0 +1,123 @@
+// Development-time contracts only; JSDoc is erased by JavaScript engines.
+/** @typedef {'haiku'|'sonnet'|'opus'} Tier */
+/** @typedef {'jev'|'ollama'} Evaluator */
+/** @typedef {'compatible'|'native'|'auto'} ClientProfile */
+/** @typedef {'jev'|'ollama'|'cache'|'fallback'|'passthrough'} DecisionSource */
+/** @typedef {'timeout'|'http_error'|'invalid_response'|'network_error'|'capacity_exhausted'} ClassifierError */
+/** @typedef {'request_start'|'route'|'upstream_response'|'upstream_model'|'upstream_usage'|'upstream_error'|'request_complete'|'request_error'|'request_cancelled'} LifecycleName */
+/** @typedef {'selected'|'confirmed'|'unknown'|'capacity_exhausted'} ContinuityState */
+/**
+ * @typedef {object} RouterConfig
+ * @property {Evaluator} evaluator
+ * @property {'api-key'|'subscription'} authMode
+ * @property {ClientProfile} clientProfile
+ * @property {string|undefined} sessionLogDir
+ * @property {'metadata'|'prompts'} sessionLogMode
+ * @property {number|undefined} stopHookBlockCap
+ * @property {string|undefined} anthropicKey
+ * @property {string|undefined} jevKey
+ * @property {string|undefined} localToken
+ * @property {string} upstream
+ * @property {string} jevEndpoint
+ * @property {string} jevModel
+ * @property {string} ollamaEndpoint
+ * @property {string} ollamaModel
+ * @property {number} ollamaTimeoutMs
+ * @property {number} ollamaStateChars
+ * @property {string} ollamaKeepAlive
+ * @property {Record<Tier,string>} models
+ * @property {number} port
+ * @property {number} jevTimeoutMs
+ * @property {number} tokenCountTimeoutMs
+ * @property {number} [tokenCountCacheEntries]
+ * @property {number} [tokenCountCacheTtlMs]
+ * @property {number} minConfidence
+ * @property {number} stateChars
+ * @property {number} maxBodyBytes
+ * @property {number} cacheEntries
+ * @property {number} cacheTtlMs
+ * @property {number} turnTtlMs
+ * @property {number|undefined} [turnEntries]
+ * @property {number} upstreamTimeoutMs
+ */
+/**
+ * @typedef {object} ClassifierDecision
+ * @property {Tier} tier
+ * @property {Tier} [classified_tier]
+ * @property {number} [confidence]
+ * @property {Evaluator} evaluator
+ * @property {DecisionSource} source
+ * @property {string} reason
+ * @property {ClassifierError} [classifier_error]
+ * @property {number} [classifier_status]
+ */
+/**
+ * @typedef {object} RoutingDecision
+ * @property {string} model
+ * @property {DecisionSource} source
+ * @property {string} reason
+ * @property {number} latency_ms
+ * @property {number} evaluation_latency_ms
+ * @property {Tier} [tier]
+ * @property {Tier} [classified_tier]
+ * @property {number} [confidence]
+ * @property {Evaluator} [evaluator]
+ * @property {ClassifierError} [classifier_error]
+ * @property {number} [classifier_status]
+ * @property {string} [compatibility_reason]
+ * @property {ContinuityState} [continuity_state]
+ * @property {'within_budget'|'over_budget'|'count_unavailable'} [context_check]
+ * @property {number} [counted_input_tokens]
+ */
+/**
+ * @typedef {object} TelemetryFields
+ * @property {number} [schema_version]
+ * @property {string} [timestamp]
+ * @property {string} [request_id]
+ * @property {string} [session_id]
+ * @property {string} [agent_id]
+ * @property {string} [prompt_id]
+ * @property {string} [request_class]
+ * @property {string} [requested_model]
+ * @property {string} [selected_model]
+ * @property {string} [confirmed_model]
+ * @property {string} [model]
+ * @property {string} [baseline_model]
+ * @property {string} [pricing_version]
+ * @property {string} [reason]
+ * @property {string} [compatibility_reason]
+ * @property {ContinuityState} [continuity_state]
+ * @property {DecisionSource} [source]
+ * @property {Evaluator} [evaluator]
+ * @property {Tier} [tier]
+ * @property {Tier} [classified_tier]
+ * @property {number} [latency_ms]
+ * @property {number} [evaluation_latency_ms]
+ * @property {number} [routing_latency_ms]
+ * @property {number} [decision_latency_ms]
+ * @property {number} [first_response_ms]
+ * @property {number} [upstream_latency_ms]
+ * @property {number} [total_latency_ms]
+ * @property {ClassifierError} [classifier_error]
+ * @property {number} [classifier_status]
+ * @property {string} [error_type]
+ * @property {number} [http_status]
+ * @property {number|string} [status]
+ * @property {'within_budget'|'over_budget'|'count_unavailable'} [context_check]
+ * @property {number} [counted_input_tokens]
+ * @property {string[]} [model_transitions]
+ * @property {boolean} [model_transitions_truncated]
+ * @property {object} [usage]
+ * @property {object} [pricing_context]
+ * @property {boolean} [usage_complete]
+ * @property {boolean} [pricing_eligible]
+ * @property {boolean} [completion_confirmed]
+ * @property {string} [unpriced_reason]
+ * @property {object} [savings]
+ * @property {object} [savings_coverage]
+ * @property {string} [prompt_excerpt]
+ * @property {boolean} [prompt_truncated]
+ */
+/** @typedef {TelemetryFields & {event:LifecycleName,request_id:string}} LifecycleEvent */
+/** @typedef {TelemetryFields & {event:'decision'|'outcome',request_id:string}} SessionRecord */
+export {};

package/src/evaluation-report.mjs

@@ -0,0 +1,114 @@
+// Pure acceptance rules shared by the opt-in evaluation harnesses. Thresholds
+// are validated before any provider calls; passing transport is not evidence
+// of evaluator availability, routing correctness, or completed task quality.
+const TIERS = ['haiku', 'sonnet', 'opus'];
+const PROFILES = ['compatible', 'native', 'auto'];
+const gate = (passed, details = {}) => ({ passed, ...details });
+const allPassed = gates => Object.values(gates).every(value => value.passed !== false);
+
+export function modelTier(model) {
+  return typeof model === 'string' ? /^claude-(haiku|sonnet|opus)(?:-|$)/.exec(model)?.[1] : undefined;
+}
+
+export function profileTier(tier, profile) {
+  return profile === 'auto' && tier === 'haiku' ? 'sonnet' : tier;
+}
+
+export function parseQualityThreshold(value) {
+  if (typeof value !== 'string' || !/^(?:\d+(?:\.\d+)?|\.\d+)$/.test(value)) throw new Error('Quality thresholds must be decimal numbers between 0 and 1');
+  const number = Number(value);
+  if (!Number.isFinite(number) || number < 0 || number > 1) throw new Error('Quality thresholds must be between 0 and 1');
+  return number;
+}
+
+export function createEvaluationPolicy({ profile = 'compatible', requiredTiers,
+  minAgreement = 1, maxUnderRouteRate = 0 } = {}) {
+  if (!PROFILES.includes(profile)) throw new Error('Evaluation profile must be compatible, native, or auto');
+  for (const [name, value] of [['minAgreement', minAgreement], ['maxUnderRouteRate', maxUnderRouteRate]]) {
+    if (typeof value !== 'number' || !Number.isFinite(value) || value < 0 || value > 1) throw new Error(`${name} must be between 0 and 1`);
+  }
+  const tiers = requiredTiers ?? (profile === 'native' ? [] : profile === 'auto' ? ['sonnet', 'opus'] : TIERS);
+  if (!Array.isArray(tiers) || tiers.some(tier => !TIERS.includes(tier)) || new Set(tiers).size !== tiers.length
+    || (profile === 'auto' && tiers.includes('haiku'))) throw new Error('Invalid required tier coverage for evaluation profile');
+  return Object.freeze({ profile, minAgreement, maxUnderRouteRate, requiredTiers: Object.freeze([...tiers]) });
+}
+
+function evaluatorGate(rows, evaluator, expectOutage) {
+  if (!['jev', 'ollama'].includes(evaluator)) throw new Error('Unknown evaluation backend');
+  const active = rows.filter(row => row.source !== 'passthrough');
+  const fresh = active.filter(row => row.source === evaluator).length;
+  const fallbacks = active.filter(row => row.source === 'fallback').length;
+  const passed = active.length > 0 && (expectOutage ? fallbacks === active.length
+    : fresh > 0 && active.every(row => row.source === evaluator || (row.source === 'cache' && row.evaluator === evaluator)));
+  return gate(passed, { expected: expectOutage ? 'fallback' : evaluator, evaluated_requests: active.length,
+    successful_evaluations: fresh, fallbacks });
+}
+
+export function tierCoverage(tiers, policy) {
+  const observed = [...new Set(tiers.filter(tier => TIERS.includes(tier)))].sort();
+  const missing = policy.requiredTiers.filter(tier => !observed.includes(tier));
+  return gate(missing.length === 0, { required: [...policy.requiredTiers], observed, missing });
+}
+
+export function evaluateRoutingReport(rows, { evaluator, policy = createEvaluationPolicy(), classifierOnly = false } = {}) {
+  policy = createEvaluationPolicy(policy);
+  if (!Array.isArray(rows) || rows.some(row => !row || !TIERS.includes(row.expected))) throw new Error('Invalid evaluation rows');
+  const valid = rows.filter(row => TIERS.includes(row.classified_tier)
+    && (row.source === evaluator || (row.source === 'cache' && row.evaluator === evaluator)));
+  const agreement = rows.length ? valid.filter(row => row.classified_tier === row.expected).length / rows.length : 0;
+  const underRoutes = valid.filter(row => TIERS.indexOf(row.classified_tier) < TIERS.indexOf(row.expected)).length;
+  const underRouteRate = rows.length ? underRoutes / rows.length : 0;
+  const eligibleTiers = valid.map(row => classifierOnly ? row.classified_tier : row.selected_tier ?? modelTier(row.selected_model));
+  const gates = {
+    transport: gate(null, { reason: 'Claude transport not exercised' }),
+    evaluator: evaluatorGate(rows, evaluator, false),
+    rubric: gate(rows.length > 0 && valid.length === rows.length && agreement >= policy.minAgreement
+      && underRouteRate <= policy.maxUnderRouteRate, { agreement, under_routes: underRoutes, under_route_rate: underRouteRate,
+      min_agreement: policy.minAgreement, max_under_route_rate: policy.maxUnderRouteRate }),
+    policy: gate(classifierOnly ? null : rows.length > 0 && rows.every(row => {
+      const selected = row.selected_tier ?? modelTier(row.selected_model);
+      return TIERS.includes(selected) && (policy.profile !== 'auto' || selected !== 'haiku')
+        && (row.expected_selected_tier === undefined || selected === row.expected_selected_tier)
+        && (row.expected_reason === undefined || row.reason === row.expected_reason);
+    }), classifierOnly ? { reason: 'Classifier-only benchmark; routing policy not exercised' } : {}),
+    coverage: tierCoverage(eligibleTiers, policy),
+    task: gate(null, { reason: 'No Claude task completion measured' }),
+  };
+  return { schema_version: 1, policy, gates, passed: allPassed(gates), requests: rows.length };
+}
+
+const TRANSPORT_CHECKS = new Set(['claude_success', 'requests_reached_router', 'upstream_model_evidence', 'no_upstream_api_errors', 'no_proxy_errors']);
+const TASK_CHECKS = new Set(['expected_result', 'original_tests_preserved', 'independent_tests_pass', 'read_edit_bash_exercised']);
+function checkGate(checks, names) {
+  const selected = Object.fromEntries(Object.entries(checks).filter(([name]) => names.has(name)));
+  return gate(Object.keys(selected).length ? Object.values(selected).every(value => value === true) : null, { checks: selected });
+}
+
+export function evaluateLiveCase({ checks, routes, evaluator, expectOutage = false, expectedClassifiedTier } = {}) {
+  const policyChecks = new Set(Object.keys(checks).filter(name => !TRANSPORT_CHECKS.has(name) && !TASK_CHECKS.has(name)));
+  const main = routes.filter(row => ['main', 'unspecified', undefined, ''].includes(row.request_class));
+  const labels = main.map(row => expectedClassifiedTier ?? row.expected_classified_tier);
+  const rubric = expectOutage ? gate(null, { reason: 'Explicit evaluator outage' })
+    : gate(main.length > 0 && labels.every(tier => TIERS.includes(tier))
+      && main.every((row, index) => row.classified_tier === labels[index]),
+    { expected: expectedClassifiedTier ?? labels, ...(labels.some(tier => !TIERS.includes(tier)) ? { reason: 'Missing declared live classifier label' } : {}) });
+  const gates = {
+    transport: checkGate(checks, TRANSPORT_CHECKS),
+    evaluator: evaluatorGate(routes, evaluator, expectOutage),
+    rubric,
+    policy: checkGate(checks, policyChecks),
+    task: checkGate(checks, TASK_CHECKS),
+  };
+  // Empty or incomplete harness evidence can never be a successful live case.
+  if (!routes.length || gates.transport.passed === null || gates.task.passed === null) gates.transport.passed = false;
+  return { gates, passed: allPassed(gates) };
+}
+
+export function evaluateLiveReport(cases, { policy = createEvaluationPolicy(), expectOutage = false } = {}) {
+  policy = createEvaluationPolicy(policy);
+  const tiers = cases.flatMap(item => (item.routes ?? []).filter(row =>
+    ['main', 'unspecified', undefined, ''].includes(row.request_class)
+      && ['jev', 'ollama', 'cache'].includes(row.source)).map(row => modelTier(row.model)));
+  const coverage = expectOutage ? gate(null, { reason: 'Explicit evaluator outage' }) : tierCoverage(tiers, policy);
+  return { policy, gates: { coverage }, passed: cases.length > 0 && cases.every(item => item.passed === true) && coverage.passed !== false };
+}

package/src/keychain.mjs

@@ -0,0 +1,58 @@
+import { spawnSync } from 'node:child_process';
+
+// macOS Keychain access through the system `security` tool. Secret values are
+// passed on stdin to its interactive mode, never as process arguments, and are
+// read back from stdout. Tool output can describe items, so failures report
+// only a generic category.
+const SECURITY = '/usr/bin/security';
+const SERVICE = 'claude-autorouter';
+const NOT_FOUND = 44;
+const PRINTABLE = /^[\x20-\x7e]+$/;
+
+function keychainError(message) {
+  const error = new Error(message);
+  error.code = 'AUTOROUTER_CONFIG_ERROR';
+  return error;
+}
+
+// Quote for the `security -i` command parser, which accepts backslash escapes
+// inside double quotes. Values are validated as printable single-line ASCII.
+const quote = value => `"${value.replace(/[\\"]/g, '\\$&')}"`;
+
+export function createKeychain({ run = spawnSync, platform = process.platform } = {}) {
+  const available = platform === 'darwin';
+  const call = (args, input) => {
+    if (!available) throw keychainError('The macOS Keychain secret store is available only on macOS.');
+    const result = run(SECURITY, args, {
+      input, encoding: 'utf8', timeout: 15000, maxBuffer: 64 * 1024, stdio: ['pipe', 'pipe', 'pipe'],
+    });
+    if (result.error) throw keychainError('Could not run the macOS Keychain tool.');
+    return result;
+  };
+  const read = account => {
+    const result = call(['find-generic-password', '-s', SERVICE, '-a', account, '-w']);
+    if (result.status === NOT_FOUND) return undefined;
+    if (result.status !== 0) {
+      throw keychainError('Could not read an AutoRouter secret from the macOS Keychain. Unlock the login keychain, or set the key in the environment.');
+    }
+    return String(result.stdout).replace(/\n$/, '');
+  };
+  return {
+    available,
+    read,
+    write(account, value, label) {
+      if (typeof value !== 'string' || !PRINTABLE.test(value)) {
+        throw keychainError('Keychain secrets must be printable single-line ASCII.');
+      }
+      call(['-i'], `add-generic-password -U -s ${quote(SERVICE)} -a ${quote(account)} -l ${quote(label)} -w ${quote(value)}\n`);
+      // Interactive mode exits successfully even when a command fails.
+      if (read(account) !== value) throw keychainError('Could not save an AutoRouter secret to the macOS Keychain.');
+    },
+    remove(account) {
+      const result = call(['delete-generic-password', '-s', SERVICE, '-a', account]);
+      if (result.status !== 0 && result.status !== NOT_FOUND) {
+        throw keychainError('Could not remove an AutoRouter secret from the macOS Keychain.');
+      }
+    },
+  };
+}

package/src/local-diagnostic.mjs

@@ -0,0 +1,191 @@
+import { createHash } from 'node:crypto';
+import { Router } from './router.mjs';
+import { buildOllamaState, OLLAMA_QUESTIONS } from './ollama-evaluator.mjs';
+import { inspectOllama } from './ollama-setup.mjs';
+import { validateOllamaEndpoint, validateOllamaModel } from './ollama-models.mjs';
+import { createEvaluationPolicy, evaluateRoutingReport } from './evaluation-report.mjs';
+
+// Packaged synthetic fixtures: no repository, transcript, or account data is
+// read. Labels test this rubric on six examples, not general task quality.
+export const LOCAL_DIAGNOSTIC_VERSION = 1;
+export const LOCAL_DIAGNOSTIC_CASES = Object.freeze([
+  { id: 'literal', expected: 'haiku', prompt: 'Print the literal word READY exactly. Do not add any explanation.' },
+  { id: 'array-length', expected: 'haiku', prompt: 'What does the JavaScript expression `[].length` evaluate to? Reply with only the integer.' },
+  { id: 'bounded-feature', expected: 'sonnet', prompt: 'Add pagination to this REST endpoint using `page` and `pageSize`. Validate the parameters, preserve existing filtering, and add tests for empty results and out-of-range pages.' },
+  { id: 'distributed-fencing', expected: 'opus', prompt: "Review this distributed locking design: worker A's lease expires while paused. Worker B acquires a newer fencing token and writes successfully. A resumes and writes using its old token. The database checks only whether a token was ever issued. Explain the failure sequence and design the minimum atomic database check that prevents stale writes, including duplicate retries." },
+  { id: 'new-mechanical-task', expected: 'haiku', history: [
+    { role: 'user', content: 'Prove the safety of a distributed ledger during failover and concurrent retries.' },
+    { role: 'assistant', content: 'The proof and regression tests are complete.' },
+  ], prompt: "Separate task: replace the exact text 'recieve' with 'receive' in a label. Make no other changes." },
+  { id: 'new-difficult-task', expected: 'opus', history: [
+    { role: 'user', content: 'What is the value of [].length?' },
+    { role: 'assistant', content: '0' },
+  ], prompt: 'New task: diagnose nondeterministic deadlocks between several processes after a rolling deployment. Reconcile conflicting traces, identify the broken ordering invariant, and prove that the repair cannot introduce message loss.' },
+].map(item => Object.freeze({ ...item, ...(item.history ? { history: Object.freeze(item.history.map(Object.freeze)) } : {}) })));
+
+const STARTUP_TIMEOUT_MS = 60000;
+const METADATA_TIMEOUT_MS = 5000;
+const MAX_METADATA_BYTES = 1024 * 1024;
+const digest = value => createHash('sha256').update(JSON.stringify(value)).digest('hex');
+const rounded = value => Math.round(value * 100) / 100;
+const identity = model => {
+  const name = model.replace(/^registry\.ollama\.ai\//, '').replace(/^library\//, '');
+  return name.slice(name.lastIndexOf('/') + 1).includes(':') ? name : `${name}:latest`;
+};
+
+const MESSAGES = Object.freeze({
+  evaluator_required: 'Local evaluation requires AUTOROUTER_EVALUATOR=ollama; the diagnostic does not change your configuration.',
+  invalid_configuration: 'Check the local Ollama endpoint, model, and runtime deadline configuration.',
+  positive_keep_alive_required: 'The local diagnostic requires a positive keep-alive, such as AUTOROUTER_OLLAMA_KEEP_ALIVE=5m. Normal routing still supports 0.',
+  model_missing: 'The configured local model is not installed. Install it explicitly before rerunning this diagnostic.',
+  unrelated_models_resident: 'Other Ollama models are resident. Retry when only the selected model, or no model, is loaded; this diagnostic leaves them running.',
+  residency_unavailable: 'Could not safely inspect local Ollama model residency. No further evaluation was attempted.',
+  OLLAMA_VERSION: 'Local evaluation requires Ollama 0.35 or newer with /v1/systemone.',
+  OLLAMA_CLOUD: 'The selected Ollama model uses a remote service. Select an installed local model.',
+  OLLAMA_TIMEOUT: 'Local Ollama metadata inspection timed out.',
+  OLLAMA_HTTP: 'Local Ollama rejected metadata inspection. Check its version and selected model.',
+  OLLAMA_RESPONSE: 'Local Ollama returned invalid or oversized metadata.',
+  OLLAMA_UNAVAILABLE: 'Could not reach local Ollama. Start the existing service and retry.',
+  startup_failed: 'The initial synthetic evaluation failed; measured cases were not run.',
+});
+const failure = code => Object.assign(new Error(MESSAGES[code]), { code });
+
+function requestBody(item) {
+  return { model: 'claude-haiku-4-5-20251001', max_tokens: 32000, tools: [],
+    system: [{ type: 'text', text: 'Synthetic coding assistant guidance: inspect relevant code and verify changes. '.repeat(110) }],
+    messages: [...structuredClone(item.history ?? []), { role: 'user', content: [
+      { type: 'text', text: '<system-reminder>SYNTHETIC_REMINDER_ONLY: unrelated environment metadata.</system-reminder>' },
+      { type: 'text', text: item.prompt },
+    ] }],
+  };
+}
+
+async function residency(config, fetchImpl, signal) {
+  const timeout = AbortSignal.timeout(METADATA_TIMEOUT_MS);
+  const combined = signal ? AbortSignal.any([signal, timeout]) : timeout;
+  const response = await fetchImpl(`${config.ollamaEndpoint}/api/ps`, { redirect: 'error', signal: combined });
+  if (!response.ok || response.redirected || !response.body) {
+    await response.body?.cancel();
+    throw failure('residency_unavailable');
+  }
+  const reader = response.body.getReader();
+  const chunks = [];
+  let size = 0;
+  const abort = () => { void reader.cancel(combined.reason).catch(() => {}); };
+  combined.addEventListener('abort', abort, { once: true });
+  try {
+    for (;;) {
+      combined.throwIfAborted();
+      const { value, done } = await reader.read();
+      if (done) break;
+      size += value.byteLength;
+      if (size > MAX_METADATA_BYTES) throw failure('residency_unavailable');
+      chunks.push(value);
+    }
+    combined.throwIfAborted();
+    const payload = JSON.parse(Buffer.concat(chunks).toString('utf8'));
+    if (!Array.isArray(payload?.models) || payload.models.some(item => typeof (item?.name ?? item?.model) !== 'string')) {
+      throw failure('residency_unavailable');
+    }
+    const models = payload.models.map(item => identity(item.name ?? item.model));
+    if (models.some(model => model !== identity(config.ollamaModel))) throw failure('unrelated_models_resident');
+    return models.length ? 'resident' : 'not_resident';
+  } catch (error) {
+    signal?.throwIfAborted();
+    throw error?.code === 'unrelated_models_resident' ? error : failure('residency_unavailable');
+  } finally {
+    combined.removeEventListener('abort', abort);
+    await reader.cancel().catch(() => {});
+    reader.releaseLock();
+  }
+}
+
+function finish(report) {
+  const evaluation = evaluateRoutingReport(report.rows, { evaluator: 'ollama', classifierOnly: true,
+    policy: createEvaluationPolicy({ profile: 'compatible' }) });
+  report.gates = { ...evaluation.gates,
+    preflight: { passed: report.preflight_passed },
+    startup: { passed: report.startup?.source === 'ollama' },
+    cases: { passed: report.rows.length === LOCAL_DIAGNOSTIC_CASES.length, expected: LOCAL_DIAGNOSTIC_CASES.length, completed: report.rows.length },
+  };
+  report.passed = !report.error && Object.values(report.gates).every(gate => gate.passed !== false);
+  return report;
+}
+
+// No warmup helper with pull/unload controls is reachable from this diagnostic.
+// Caller cancellation is preserved, including when runtime timeout is zero.
+export async function runLocalDiagnostic(config, { fetchImpl = fetch, signal, onProgress = () => {} } = {}) {
+  const report = { schema_version: 1, type: 'local_evaluator_diagnostic', fixture_version: LOCAL_DIAGNOSTIC_VERSION,
+    fixture_sha256: digest(LOCAL_DIAGNOSTIC_CASES), questions_sha256: digest(OLLAMA_QUESTIONS),
+    evaluator: 'ollama', startup_timeout_ms: STARTUP_TIMEOUT_MS, preflight_passed: false,
+    residency_before: 'unknown', startup: null, rows: [], paid_provider_calls: 0, downloads: 0,
+    models_unloaded: 0, configuration_changed: false };
+  const progress = event => { try { Promise.resolve(onProgress(event)).catch(() => {}); } catch {} };
+  try {
+    signal?.throwIfAborted();
+    if (config?.evaluator !== 'ollama') throw failure('evaluator_required');
+    try {
+      validateOllamaEndpoint(config.ollamaEndpoint);
+      validateOllamaModel(config.ollamaModel);
+      if (!Number.isSafeInteger(config.ollamaTimeoutMs) || config.ollamaTimeoutMs < 0 || config.ollamaTimeoutMs > 30000) throw new Error();
+    } catch { throw failure('invalid_configuration'); }
+    if (typeof config.ollamaKeepAlive !== 'string' || !/^[1-9]\d{0,3}[smh]$/.test(config.ollamaKeepAlive)) throw failure('positive_keep_alive_required');
+    report.evaluator_model = config.ollamaModel;
+    report.runtime_timeout_ms = config.ollamaTimeoutMs;
+    report.keep_alive = config.ollamaKeepAlive;
+    progress({ event: 'preflight' });
+    const inspection = await inspectOllama(config, { fetchImpl, signal });
+    if (!inspection.installed) throw failure('model_missing');
+    report.residency_before = await residency(config, fetchImpl, signal);
+    report.preflight_passed = true;
+    progress({ event: 'startup', residency_before: report.residency_before, timeout_ms: STARTUP_TIMEOUT_MS });
+    const initialStart = performance.now();
+    const startup = await new Router({ ...config, ollamaTimeoutMs: STARTUP_TIMEOUT_MS }, { fetchImpl })
+      .classify(requestBody({ prompt: 'Return the literal word ready.' }), signal);
+    report.startup = { latency_ms: rounded(performance.now() - initialStart), source: startup.source,
+      classified_tier: startup.classified_tier, classifier_error: startup.classifier_error, classifier_status: startup.classifier_status,
+      residency_before: report.residency_before, timeout_ms: STARTUP_TIMEOUT_MS };
+    progress({ event: 'startup_complete', ...report.startup });
+    if (startup.source !== 'ollama') throw failure('startup_failed');
+    for (const item of LOCAL_DIAGNOSTIC_CASES) {
+      signal?.throwIfAborted();
+      const resident = await residency(config, fetchImpl, signal);
+      progress({ event: 'case_start', case: item.id, residency_before: resident });
+      const body = requestBody(item);
+      const state = buildOllamaState(body, config.ollamaStateChars);
+      // A new classifier prevents cache hits from appearing as local inference.
+      const start = performance.now();
+      const decision = await new Router(config, { fetchImpl }).classify(body, signal);
+      const row = { case: item.id, expected: item.expected, classified_tier: decision.classified_tier,
+        tier: decision.tier, source: decision.source, evaluator: decision.evaluator, reason: decision.reason,
+        classifier_error: decision.classifier_error, classifier_status: decision.classifier_status,
+        latency_ms: rounded(performance.now() - start), residency_before: resident,
+        state_bytes: Buffer.byteLength(JSON.stringify(state)), current_task_matches: state.current_task === item.prompt };
+      report.rows.push(row);
+      progress({ event: 'case_complete', ...row });
+    }
+  } catch (error) {
+    signal?.throwIfAborted();
+    const code = Object.hasOwn(MESSAGES, error?.code) ? error.code : 'OLLAMA_UNAVAILABLE';
+    report.error = { code, message: MESSAGES[code] };
+  }
+  const result = finish(report);
+  // Correct labels alone cannot conceal broken task extraction.
+  if (report.rows.some(row => !row.current_task_matches)) {
+    result.gates.cases.passed = false;
+    result.passed = false;
+  }
+  return result;
+}
+
+export function formatLocalDiagnostic(report) {
+  const cause = row => row.classifier_error ? ` (${row.classifier_error}${row.classifier_status ? ` ${row.classifier_status}` : ''})` : '';
+  const lines = [`Local evaluator diagnostic: ${report.evaluator_model ?? 'not configured'}`];
+  if (report.runtime_timeout_ms !== undefined) lines.push(`Runtime deadline: ${report.runtime_timeout_ms === 0 ? 'disabled' : `${report.runtime_timeout_ms} ms`}; initial preparation bound: ${report.startup_timeout_ms} ms.`);
+  if (report.startup) lines.push(`Initial synthetic call: ${report.startup.latency_ms} ms; model ${report.startup.residency_before === 'resident' ? 'resident' : 'not resident'} before call; ${report.startup.source}${cause(report.startup)}.`);
+  for (const row of report.rows) lines.push(`${row.case}: expected ${row.expected}, got ${row.classified_tier ?? 'no verdict'}; ${row.source}${cause(row)}; ${row.latency_ms} ms; ${row.residency_before} before call.`);
+  if (report.error) lines.push(report.error.message);
+  lines.push(`${report.passed ? 'PASS' : 'FAIL'}: ${report.rows.filter(row => row.source === 'ollama' && row.classified_tier === row.expected).length}/${LOCAL_DIAGNOSTIC_CASES.length} synthetic cases; missing tiers: ${report.gates.coverage.missing.join(', ') || 'none'}.`);
+  lines.push('Residency is observed, not a guarantee of cold or warm caches. This checks six classifier labels, not Claude task completion or general accuracy.');
+  return lines;
+}