auto-model-router 0.2.31 → 0.3.0

package/src/config/schema.ts

@@ -33,6 +33,28 @@ const openrouter = z.strictObject({
 	catalogRefreshMs: z.number().nonnegative().optional(),
 });
 
+const ollamaRate = z.strictObject({
+	input: z.number().nonnegative(),
+	cachedInput: z.number().nonnegative().optional(),
+	output: z.number().nonnegative(),
+});
+
+const ollama = z.strictObject({
+	enabled: z.boolean().optional(),
+	baseUrl: z.string().min(1).optional(),
+	apiKey: z.string().optional(),
+	timeoutMs: z.number().positive().optional(),
+	catalogTtlMs: z.number().positive().optional(),
+	includeLocal: z.boolean().optional(),
+	prices: z.record(z.string(), ollamaRate).optional(),
+	twins: z.record(z.string(), z.string()).optional(),
+	costBias: z.number().positive().optional(),
+	biasUntilUsage: z.number().min(0).max(1).optional(),
+	usagePollMs: z.number().nonnegative().optional(),
+	quotaCooldownMs: z.number().nonnegative().optional(),
+	rateLimitCooldownMs: z.number().nonnegative().optional(),
+});
+
 const benchmarks = z.strictObject({
 	enabled: z.boolean().optional(),
 	artificialAnalysisApiKey: z.string().optional(),
@@ -73,6 +95,8 @@ const filters = z.strictObject({
 	latencyReferenceMs: z.number().positive().optional(),
 	latencyReferenceTokensPerSec: z.number().positive().optional(),
 	latencyMinSamples: z.number().int().nonnegative().optional(),
+	maxExpectedWaitMs: z.number().positive().optional(),
+	escalationCostWeight: z.number().min(0).max(1).optional(),
 });
 
 const classifier = z.strictObject({
@@ -85,6 +109,7 @@ const classifier = z.strictObject({
 	toolAxis: qualityAxis.optional(),
 	chatAxis: qualityAxis.optional(),
 	agenticLoopDepth: z.number().int().nonnegative().optional(),
+	mechanicalRetryFactor: z.number().min(0).max(1).optional(),
 	reasoningWeights: z
 		.strictObject({
 			medium: z.number().nonnegative().optional(),
@@ -109,6 +134,7 @@ const hysteresis = z.strictObject({
 	holdTurns: z.number().int().nonnegative().optional(),
 	holdTurnsAfterEscalation: z.number().int().nonnegative().optional(),
 	switchMargin: z.number().positive().optional(),
+	switchHorizonTurns: z.number().int().positive().optional(),
 	cacheWarmTtlMs: z.number().nonnegative().optional(),
 	maxDowngradePerTurn: z.number().int().nonnegative().optional(),
 	breakHoldOnMechanical: z.boolean().optional(),
@@ -163,6 +189,7 @@ const compaction = z.strictObject({
 	enabled: z.boolean().optional(),
 	budgetTokens: z.number().int().positive().optional(),
 	floorRatio: z.number().positive().max(1).optional(),
+	replanGrowthRatio: z.number().min(1).optional(),
 	fitToWindow: z.boolean().optional(),
 	protectRecentTurns: z.number().int().positive().optional(),
 	maxToolResultBytes: z.number().int().positive().optional(),
@@ -225,6 +252,7 @@ export const configInputSchema = z.strictObject({
 			chat: taskConfig.optional(),
 		})
 		.optional(),
+	ollama: ollama.optional(),
 	filters: filters.optional(),
 	classifier: classifier.optional(),
 	escalation: escalation.optional(),

package/src/config/types.ts

@@ -64,6 +64,58 @@ export interface OpenRouterConfig {
 	catalogRefreshMs: number;
 }
 
+/**
+ * Ollama Cloud as a second upstream, ranked in the same catalog as OpenRouter.
+ *
+ * Off by default. Reached either through a local Ollama daemon (the default
+ * `baseUrl`; it proxies `:cloud` models under the signed-in account and lists
+ * their context length and capabilities) or directly at `https://ollama.com/v1`
+ * with an API key. Slugs are `ollama/<id>`.
+ *
+ * Ollama publishes no prices via API, so rates come from a shipped snapshot
+ * plus `prices`; a model with no rate is dropped. Quality scores come from the
+ * model's OpenRouter twin (matched by name, or pinned via `twins`), because
+ * Ollama publishes none — an unmatched model is unscored and serves only
+ * `trivial`. Ollama reports no cost per response, so the ledger's predicted
+ * figure is what gets recorded.
+ */
+export interface OllamaConfig {
+	enabled: boolean;
+	/** `http://127.0.0.1:11434/v1` (daemon) or `https://ollama.com/v1` (direct). */
+	baseUrl: string;
+	/** Resolved from config, then `OLLAMA_API_KEY`. Needed for ollama.com; the daemon uses its own sign-in. */
+	apiKey: string;
+	/** Per-request timeout, ms. */
+	timeoutMs: number;
+	/** Re-list models when the last listing is older than this, ms. */
+	catalogTtlMs: number;
+	/** Also expose the daemon's LOCAL models (unpriced unless `prices` names them). Off: cloud only. */
+	includeLocal: boolean;
+	/** USD per million tokens, keyed by bare cloud name; overrides/extends the shipped snapshot. */
+	prices: Record<string, { input: number; cachedInput?: number; output: number }>;
+	/** Bare cloud name → OpenRouter slug, when the name-based twin match is wrong or missing. */
+	twins: Record<string, string>;
+	/**
+	 * Multiplier on an Ollama model's effective cost in ranking, 1 = list price.
+	 * Below 1 prefers Ollama when a plan's included credits would otherwise go
+	 * unused; the ledger still records list price, so spend stays honest.
+	 */
+	costBias: number;
+	/**
+	 * Plan-usage fraction (0-1) at which `costBias` switches off and Ollama
+	 * ranks at list price. Read from ollama.com's `/api/usage`, which reports
+	 * consumption as a share of the plan's included monthly credits — so the
+	 * same setting is right on Pro, Max or Team. 1 keeps the bias regardless.
+	 */
+	biasUntilUsage: number;
+	/** How often to re-read plan usage, ms. 0 disables the read (bias stays static). */
+	usagePollMs: number;
+	/** How long to route around Ollama after a 402 (credits exhausted), ms. */
+	quotaCooldownMs: number;
+	/** How long to route around Ollama after a 429 (concurrency cap), ms. */
+	rateLimitCooldownMs: number;
+}
+
 /**
  * External benchmark feeds that BACKFILL quality scores OpenRouter does not
  * publish. OpenRouter embeds Artificial Analysis scores for the models it has,
@@ -206,6 +258,37 @@ export interface FilterConfig {
 	latencyReferenceTokensPerSec: number;
 	/** Streamed samples required before latency is scored against a model. */
 	latencyMinSamples: number;
+	/**
+	 * Absolute expected-wait ceiling (ms). A hard drop, mirroring the price
+	 * ceiling: any model whose expected total wait (TTFT + streaming the expected
+	 * completion at its measured throughput) exceeds this is rejected outright,
+	 * regardless of tier or price. This is the gate the latency *penalty* cannot
+	 * be — the penalty is multiplicative on cost and capped, so on an ultra-cheap
+	 * model even the capped multiple leaves it cheapest; a slow-but-cheap model is
+	 * never demoted by scoring alone. Only models with at least `latencyMinSamples`
+	 * observations are dropped, so a new model still gets its cold-start turns.
+	 * Relaxed alongside trust in tier rescue so a narrowed catalog never 500s.
+	 * Undefined ⇒ off (the default).
+	 */
+	maxExpectedWaitMs?: number;
+	/**
+	 * How much of a model's measured escalation risk to price into its effective
+	 * cost, 0-1. 0 (the default) disables the term.
+	 *
+	 * The trust divisor treats a failure as a proportional retry of the same
+	 * model, so a 4% escalation rate reads as a 4% surcharge. The real cost of
+	 * a probe escalation is a whole re-dispatch on the NEXT tier's model:
+	 * measured over a week, escalated attempts billed ~$0.08 each while the
+	 * cheap model that failed had billed ~$0.0006 — a 700x multiple, not 4%. A
+	 * cheap model with a 3.6% escalation rate therefore cost more than a
+	 * reliable one at 4x its price, and the divisor could never see it.
+	 *
+	 * At 1, `effectiveUsd += escalationRate × (this prompt × the ledger's
+	 * measured $/prompt-token of escalated attempts)`, so flakiness is priced
+	 * at what it actually costs. Inert until the ledger holds enough escalated
+	 * attempts to measure.
+	 */
+	escalationCostWeight: number;
 }
 
 export interface ClassifierConfig {
@@ -229,6 +312,13 @@ export interface ClassifierConfig {
 	chatAxis: QualityAxis;
 	/** Tool-loop depth above which the agentic axis takes over. */
 	agenticLoopDepth: number;
+	/**
+	 * Fraction of the failed-tool weight that survives when the turn is a
+	 * mechanical tool-result continuation. A retry after a failed tool call is
+	 * the most mechanical turn there is; the flat weight let automated retry
+	 * loops buy the hard tier. 1 preserves the shipped behaviour.
+	 */
+	mechanicalRetryFactor: number;
 	/**
 	 * Score added when the CLIENT asks for a reasoning effort, per level. The
 	 * premise is that asking for reasoning states expected difficulty directly.
@@ -283,6 +373,19 @@ export interface HysteresisConfig {
 	 * discount by this multiple. 1.0 ⇒ break even; higher ⇒ stickier.
 	 */
 	switchMargin: number;
+	/**
+	 * Turns over which a model switch is amortised in the stay/switch decision.
+	 * 1 (the default) is the one-turn comparison: stay at the warm model's
+	 * cache-read price vs switch at the new model's cold price. That is right
+	 * for one turn and wrong for the run that follows: it kept a $2.55/Mtok
+	 * model warm for 33 consecutive `moderate` dispatches ("stay $0.0589 ≤
+	 * switch $0.1131 × 1.3") where the ranked winner would have been $0.003
+	 * per turn once ITS cache was warm. With a horizon H the comparison is
+	 * `H × stayWarm` against `switchCold + (H − 1) × newWarm`, so a switch that
+	 * pays for itself within H turns is taken. Deep loops average ~25
+	 * dispatches per user-visible turn, so single digits are conservative.
+	 */
+	switchHorizonTurns: number;
 	/** Assume a warm cache expires after this long. OpenRouter sticky sessions: 5-10 min. */
 	cacheWarmTtlMs: number;
 	/** Downgrade at most this many tiers per turn, so quality never falls off a cliff. */
@@ -504,6 +607,24 @@ export interface CompactionConfig {
 	 * already-cached prompt bytes, and a cold prompt costs ~4.3x a warm one.
 	 */
 	floorRatio: number;
+	/**
+	 * Only extend an existing plan once the compacted prompt has grown by this
+	 * factor since the plan was last made. 1 (the default) re-plans on every
+	 * over-budget turn.
+	 *
+	 * `floorRatio` rations re-planning only when the budget is reachable. On
+	 * the traffic actually observed it is not — compacted prompts sit at
+	 * 100–160k tokens against a 40k budget — so every turn is over budget and a
+	 * new edit is added the moment a tool result ages out of the protected
+	 * window. Measured over a week of same-model turns: the plan changed on
+	 * 1,031 dispatches at a 79.5% cache hit and $0.0120 each, against 92.6% and
+	 * $0.0067 when it held. At 1.1 a plan holds until the prompt is 10% larger
+	 * than when it was made — several turns in a deep loop — at the cost of
+	 * that much more stale tool output riding along in between. Fit-to-window
+	 * compaction is never rationed; a prompt that would overflow always
+	 * re-plans.
+	 */
+	replanGrowthRatio: number;
 	/** Also compact when the prompt would overflow the profile's context window. */
 	fitToWindow: boolean;
 	/** Never touch the last N user/assistant turns or the volatile tail. */
@@ -523,6 +644,7 @@ export interface CompactionConfig {
 export interface RouterConfig {
 	server: ServerConfig;
 	openrouter: OpenRouterConfig;
+	ollama: OllamaConfig;
 	benchmarks: BenchmarksConfig;
 	tiers: Record<Tier, TierConfig>;
 	tasks: Record<TaskType, TaskConfig>;
@@ -538,10 +660,13 @@ export interface RouterConfig {
 	profiles: ProfileConfig[];
 	ledger: LedgerConfig;
 	/**
-	 * Derive each tier's quality floor from the models actually available at
-	 * every catalog refresh, relaxing (never tightening) the configured floors.
+	 * Relax a tier's quality floor to a catalog-derived band when the configured
+	 * floor is met by fewer than three available models (never tightening it).
 	 * Without this, a narrow OpenRouter guardrail leaves every tier above
 	 * `trivial` permanently empty and the router is stuck on the cheapest model.
+	 * A floor that at least three models meet stands exactly as configured, so
+	 * on a wide catalog this is a no-op — an earlier version relaxed
+	 * unconditionally and a wide catalog's weak tail dragged every floor down.
 	 */
 	adaptiveTierFloors: boolean;
 	/**

package/src/cost/ledger.ts

@@ -18,10 +18,26 @@ import type { RouterConfig } from "../config/types.ts";
 import { consumePendingEstimate } from "../tokens/estimate.ts";
 import { computeBlendedRate } from "./blended.ts";
 import { computeCost } from "./forecast.ts";
-import type { BlendedRate, Ledger, LedgerEntry, ModelLatency, ModelTrust, UsageCounts } from "./types.ts";
+import type {
+	BlendedRate,
+	EscalationCost,
+	Ledger,
+	LedgerEntry,
+	LedgerSignals,
+	ModelLatency,
+	ModelTrust,
+	UsageCounts,
+} from "./types.ts";
 
 /** Estimates below this many samples are noise; the default ratio is better. */
 const MIN_CALIBRATION_SAMPLES = 20;
+/** Calibration samples outside this bytes-per-token band are provider accounting quirks, not tokenizer facts. */
+const MIN_SANE_BYTES_PER_TOKEN = 1.5;
+const MAX_SANE_BYTES_PER_TOKEN = 8;
+/** Escalated attempts needed before their measured cost is trusted. */
+const MIN_ESCALATION_SAMPLES = 10;
+/** The escalation-cost aggregate scans a window of rows; memoised for this long. */
+const ESCALATION_COST_MEMO_MS = 60_000;
 const DAY_MS = 86_400_000;
 
 // Row shapes below are fixed by our own schema in util/sqlite.ts.
@@ -99,7 +115,9 @@ interface CalibrationRow {
  * unclassifiable legacy row and stays attributable, preserving the old,
  * stricter behaviour rather than silently forgiving it.
  */
-const UNATTRIBUTABLE_KINDS = "('aborted', 'auth', 'moderation', 'model_unavailable')";
+// `quota` joins the list for the same reason as `auth`: an exhausted plan
+// allowance is a fact about the account, identical for every model behind it.
+const UNATTRIBUTABLE_KINDS = "('aborted', 'auth', 'moderation', 'model_unavailable', 'quota')";
 
 const ATTRIBUTABLE_ERROR = `error IS NOT NULL AND (error_kind IS NULL OR error_kind NOT IN ${UNATTRIBUTABLE_KINDS})`;
 
@@ -258,6 +276,16 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
 	);
 	const ratioStmt = db.query("SELECT est_bytes, actual_tokens, samples FROM token_calibration WHERE tokenizer = ?");
 	const recentStmt = db.query("SELECT * FROM ledger ORDER BY created_at_ms DESC LIMIT ?");
+	// What an escalated retry actually bills, per prompt token, over a window.
+	// attempt > 0 rows are the re-dispatches that followed a rejected attempt;
+	// errored ones carry no usage and are excluded.
+	const escalationCostStmt = db.query(
+		`SELECT COUNT(*) AS samples,
+			COALESCE(SUM(COALESCE(reported_usd, predicted_usd)), 0) AS usd,
+			COALESCE(SUM(json_extract(usage, '$.promptTokens')), 0) AS prompt_tokens
+		 FROM ledger WHERE attempt > 0 AND error IS NULL AND created_at_ms >= ?`,
+	);
+	let escalationMemo: { atMs: number; windowDays: number; value: EscalationCost | null } | null = null;
 	const cacheMetaStmt = db.query("SELECT fetched_at_ms FROM catalog_cache WHERE id = 1");
 	const cachePayloadStmt = db.query("SELECT payload FROM catalog_cache WHERE id = 1");
 
@@ -286,7 +314,8 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
 	return {
 		record(entry: LedgerEntry): void {
 			const models = priceIndex();
-			const model = (entry.servedSlug !== null ? models?.get(entry.servedSlug) : undefined) ?? models?.get(entry.slug) ?? null;
+			const model =
+				entry.priceModel ?? (entry.servedSlug !== null ? models?.get(entry.servedSlug) : undefined) ?? models?.get(entry.slug) ?? null;
 			insertStmt.run(
 				entry.id,
 				entry.createdAtMs,
@@ -331,7 +360,15 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
 				// The SERVED model's tokenizer produced the billing; the estimate-time
 				// family is the fallback when the model is unknown to the catalog.
 				const tokenizer = (model?.tokenizer ?? pending.tokenizer).trim().toLowerCase();
-				calibrationStmt.run(tokenizer, pending.bytes, entry.usage.promptTokens);
+				// Reject samples no real tokenizer could produce. The rows are
+				// running sums, so one provider that reports inflated counts (seen:
+				// ~8x the bytes-implied tokens, i.e. 0.4 bytes/token) poisons a
+				// whole family for thousands of samples. Text tokenizers land
+				// between ~2 and ~5 bytes/token; the band is generous around that.
+				const bytesPerToken = pending.bytes / entry.usage.promptTokens;
+				if (bytesPerToken >= MIN_SANE_BYTES_PER_TOKEN && bytesPerToken <= MAX_SANE_BYTES_PER_TOKEN) {
+					calibrationStmt.run(tokenizer, pending.bytes, entry.usage.promptTokens);
+				}
 			}
 		},
 
@@ -379,6 +416,38 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
 			if (row === null) return null;
 			return toLatency(slug, row);
 		},
+		signals(slugs: readonly string[], harnessId?: string): Map<string, LedgerSignals> {
+			const cutoff = cfg.filters.trustWindowDays > 0 ? Date.now() - cfg.filters.trustWindowDays * DAY_MS : 0;
+			const hasHarness = harnessId !== undefined && harnessId !== "";
+			const out = new Map<string, LedgerSignals>();
+			for (const slug of slugs) {
+				const trustRow = hasHarness
+					? (trustHarnessStmt.get(slug, harnessId, cutoff) as TrustRow | null)
+					: (trustStmt.get(slug, cutoff) as TrustRow | null);
+				const latencyRow = hasHarness
+					? (latencyHarnessStmt.get(slug, harnessId) as LatencyRow | null)
+					: (latencyStmt.get(slug) as LatencyRow | null);
+				out.set(slug, {
+					trust: trustRow === null || trustRow.attempts === 0 ? null : toTrust(slug, trustRow),
+					latency: latencyRow === null ? null : toLatency(slug, latencyRow),
+				});
+			}
+			return out;
+		},
+
+		escalationCost(windowDays: number): EscalationCost | null {
+			const now = Date.now();
+			if (escalationMemo !== null && escalationMemo.windowDays === windowDays && now - escalationMemo.atMs < ESCALATION_COST_MEMO_MS) {
+				return escalationMemo.value;
+			}
+			const row = escalationCostStmt.get(now - windowDays * DAY_MS) as { samples: number; usd: number; prompt_tokens: number } | null;
+			const value: EscalationCost | null =
+				row === null || row.samples < MIN_ESCALATION_SAMPLES || row.prompt_tokens <= 0
+					? null
+					: { usdPerPromptToken: row.usd / row.prompt_tokens, samples: row.samples, windowDays };
+			escalationMemo = { atMs: now, windowDays, value };
+			return value;
+		},
 
 		tokenRatio(tokenizer: string): number | null {
 			const row = ratioStmt.get(tokenizer.trim().toLowerCase()) as CalibrationRow | null;