auto-model-router 0.2.31 → 0.3.0

package/src/server/turn.ts

@@ -11,9 +11,11 @@
 import type { CatalogSource } from "../catalog/types.ts";
 import type { ContextBridge } from "../context/types.ts";
 import type { RouterConfig } from "../config/types.ts";
+import { computeCost } from "../cost/forecast.ts";
 import { EMPTY_USAGE, type Ledger, type UsageCounts } from "../cost/types.ts";
 import { createProbe, type Probe } from "../router/escalate.ts";
 import { resolveHoldTurns } from "../router/explore.ts";
+import { adjustPendingEstimate } from "../tokens/estimate.ts";
 import {
 	TIER_ORDER,
 	type ConversationStore,
@@ -36,6 +38,23 @@ import type { NormRequest, ResponseSink, TurnSummary, UpstreamChunk } from "../w
  */
 const MAX_SAME_TIER_FAILOVERS = 2;
 
+/**
+ * Probe signals that indict the PROVIDER rather than the tier: an empty
+ * stream, a refusal, or an error finish says nothing about whether the work
+ * needed a stronger model. Measured on 7 days of live traffic, 49 of 71
+ * escalations re-dispatched the very slug that had just failed, one tier up,
+ * paying the tier premium for a provider hiccup. These signals get the same
+ * same-tier failover an HTTP 5xx gets, and only then step up a tier. Structural
+ * signals (malformed arguments, a repeated call) still escalate directly — a
+ * stronger model is the remedy there.
+ */
+const PROVIDER_SIGNALS: ReadonlySet<string> = new Set(["empty_completion", "refusal", "upstream_error"]);
+
+/** A client hang-up: the request signal fired, or the transport reported the abort. */
+function isClientAbort(err: unknown, signal: AbortSignal): boolean {
+	return signal.aborted || (err instanceof UpstreamError && err.kind === "aborted");
+}
+
 export interface TurnDeps {
 	config: RouterConfig;
 	router: Router;
@@ -170,6 +189,14 @@ export async function runTurn(
 			...(decision.compactionPlan.length > 0 ? { compactionPlan: decision.compactionPlan } : {}),
 		});
 
+		// Calibrate the token estimate against the bytes that actually go out —
+		// after compaction shrank the prompt and the context block was appended
+		// — not the raw request the estimate was taken from.
+		adjustPendingEstimate(
+			req.conversationKey,
+			req.promptBytes - decision.compactionSavedBytes + (contextBlock === undefined ? 0 : Buffer.byteLength(contextBlock)),
+		);
+
 		// Our own abort composes with the client's: escalation teardown and
 		// client disconnect both kill the upstream connection.
 		const attemptAbort = new AbortController();
@@ -196,6 +223,7 @@ export async function runTurn(
 			if (generationId === null && dispatch !== null) {
 				generationId = await dispatch.generationId().catch(() => null);
 			}
+			const priceModel = deps.catalog.find(servedSlug ?? decision.slug);
 			ledger.record({
 				id: crypto.randomUUID(),
 				createdAtMs: Date.now(),
@@ -229,6 +257,9 @@ export async function runTurn(
 				upstreamGenerationId: generationId,
 				error: fields.error,
 				promptTokensSaved: decision.promptTokensSaved,
+				// The ledger prices OpenRouter slugs from its own cached payload; any
+				// other provider's model exists only in the live catalog.
+				...(priceModel === undefined ? {} : { priceModel }),
 			});
 			// Book the money HERE, beside the ledger row, so the two can never
 			// disagree. Every dispatch that reaches this point was billed —
@@ -239,6 +270,28 @@ export async function runTurn(
 			conversations.accrue(req.conversationKey, { spentUsd: reportedUsd ?? decision.forecast.expectedUsd });
 		};
 
+		// Same-tier failover: re-route with every failed slug excluded and accept
+		// the result only if it stays in this tier on a different model. A
+		// 404/429/5xx, an empty stream, or a refusal indicts the slug, not the
+		// tier, so a sibling is tried before the tier is abandoned. Returns null
+		// when the bound is hit or the router widened tiers on its own; the
+		// caller then falls through to tier escalation.
+		const trySameTierFailover = async (why: string): Promise<Decision | null> => {
+			if (sameTierFailovers >= MAX_SAME_TIER_FAILOVERS) return null;
+			let failover: Decision | null = null;
+			try {
+				failover = await router.route(req, { attempt: attempt + 1, excludeSlugs: failedSlugs });
+			} catch {
+				// A routing failure here must not kill the turn; tier escalation
+				// may still find a model.
+				return null;
+			}
+			if (failover === null || failover.tier !== decision.tier || failedSlugs.includes(failover.slug)) return null;
+			sameTierFailovers++;
+			failover.reasons = [...failover.reasons, `failover: ${decision.slug} ${why}; retrying ${failover.slug} in ${failover.tier}`];
+			return failover;
+		};
+
 		// "retry" re-enters the attempt loop; "done" means the turn is settled
 		// (client told, or client gone) and runTurn must return.
 		const onUpstreamError = async (err: unknown): Promise<"retry" | "done"> => {
@@ -261,30 +314,15 @@ export async function runTurn(
 			}
 			if (uerr.retryable && attempt + 1 < maxAttempts) {
 				failedSlugs.push(decision.slug);
-				if (sameTierFailovers < MAX_SAME_TIER_FAILOVERS) {
-					// Before jumping a tier, try a DIFFERENT model in the same
-					// tier: a 404/429/5xx indicts the slug, not the tier.
-					let failover: Decision | null = null;
-					try {
-						failover = await router.route(req, { attempt: attempt + 1, excludeSlugs: failedSlugs });
-					} catch {
-						// A routing failure here must not kill the turn; tier
-						// escalation below may still find a model.
-						failover = null;
-					}
-					if (failover !== null && failover.tier === decision.tier && !failedSlugs.includes(failover.slug)) {
-						sameTierFailovers++;
-						failover.reasons = [
-							...failover.reasons,
-							`failover: ${decision.slug} returned ${uerr.kind}; retrying ${failover.slug} in ${failover.tier}`,
-						];
-						pendingDecision = failover;
-						await writeEntry({ wasted: true, escalationSignal: null, error: `${uerr.kind}: ${uerr.message}` });
-						return "retry";
-					}
-					// No different candidate at this tier — the router widened on
-					// its own or only the failed slug qualifies. Fall through to
-					// tier escalation.
+				// Before jumping a tier, try a DIFFERENT model in the same tier:
+				// a 404/429/5xx indicts the slug, not the tier. Null means no other
+				// candidate at this tier — the router widened on its own or only
+				// the failed slug qualifies — so fall through to tier escalation.
+				const failover = await trySameTierFailover(`returned ${uerr.kind}`);
+				if (failover !== null) {
+					pendingDecision = failover;
+					await writeEntry({ wasted: true, escalationSignal: null, error: `${uerr.kind}: ${uerr.message}` });
+					return "retry";
 				}
 				const topTier = TIER_ORDER[TIER_ORDER.length - 1];
 				if (decision.tier !== topTier) {
@@ -307,6 +345,11 @@ export async function runTurn(
 		let escalateVerdict: Extract<ProbeVerdict, { action: "escalate" }> | null = null;
 		let streamError: unknown = null;
 		let sinkDied = false;
+		// The upstream generation ran to its end — either the stream closed
+		// normally, or the client hung up after the finish event had already
+		// arrived. Both are settled generations; only the second used to be
+		// recorded as an error.
+		let streamEnded = false;
 
 		try {
 			dispatch = await upstream.dispatch({ body, sessionId: decision.sessionId, signal: attemptSignal });
@@ -374,21 +417,22 @@ export async function runTurn(
 					attemptAbort.abort();
 					break;
 				}
-				if (!committed && escalateVerdict === null) {
-					const verdict = probe.verdictOnEnd();
-					if (verdict.action === "commit") {
-						committed = true;
-						for (const heldChunk of probe.held()) await emit(heldChunk);
-					} else {
-						escalateVerdict = verdict;
-						attemptAbort.abort();
-					}
-				}
+				if (escalateVerdict === null) streamEnded = true;
 			} catch (err) {
 				if (escalateVerdict !== null || attemptAbort.signal.aborted) {
 					// Teardown noise from our own abort; the escalate path owns the outcome.
 				} else if (err instanceof SinkError) {
 					sinkDied = true;
+				} else if (finishReason !== null && isClientAbort(err, signal)) {
+					// The client closed the connection AFTER the finish event: the
+					// generation is complete and billed, the client already has its
+					// answer, only the trailing `[DONE]` went unread. Measured live:
+					// 1,842 of 2,068 "request aborted" rows carried a finish reason
+					// and full usage, and every one skipped the state save below —
+					// stale hysteresis, cache-warmth, and compaction state on 12% of
+					// turns. A settled generation is settled however the socket
+					// closed.
+					streamEnded = true;
 				} else {
 					streamError = err;
 				}
@@ -396,6 +440,29 @@ export async function runTurn(
 				// Never leak the upstream connection, whatever happened above.
 				attemptAbort.abort();
 			}
+			if (streamEnded && !committed && escalateVerdict === null) {
+				const verdict = probe.verdictOnEnd();
+				if (verdict.action === "commit") {
+					committed = true;
+					try {
+						for (const heldChunk of probe.held()) await emit(heldChunk);
+					} catch (err) {
+						if (err instanceof SinkError) sinkDied = true;
+						else streamError = err;
+					}
+				} else {
+					escalateVerdict = verdict;
+				}
+			}
+		}
+
+		// A provider that reports no cost (Ollama) still reports usage: price the
+		// actual tokens at the served model's catalog rate rather than leaving the
+		// pre-dispatch forecast (with its assumed 1,024 completion tokens) to stand
+		// in for a turn that produced 26.
+		if (reportedUsd === null && usage.promptTokens > 0) {
+			const served = deps.catalog.find(servedSlug ?? decision.slug);
+			if (served !== undefined) reportedUsd = computeCost(served, usage).total;
 		}
 
 		if (sinkDied) {
@@ -408,6 +475,26 @@ export async function runTurn(
 		}
 
 		if (escalateVerdict !== null) {
+			// The model that produced the rejected output must not serve the
+			// retry, at this tier or the next: without this, 49 of 71 measured
+			// escalations re-dispatched the same slug one tier up.
+			failedSlugs.push(decision.slug);
+			if (PROVIDER_SIGNALS.has(escalateVerdict.signal) && attempt + 1 < maxAttempts) {
+				const failover = await trySameTierFailover(`${escalateVerdict.signal} (${escalateVerdict.reason})`);
+				if (failover !== null) {
+					pendingDecision = failover;
+					await writeEntry({ wasted: true, escalationSignal: escalateVerdict.signal, error: null });
+					log.info("same-tier failover", {
+						signal: escalateVerdict.signal,
+						reason: escalateVerdict.reason,
+						from: decision.slug,
+						to: failover.slug,
+						tier: decision.tier,
+						attempt,
+					});
+					continue;
+				}
+			}
 			const hasRunway = attempt + 1 < maxAttempts && decision.probe.escalateTo !== null;
 			if (hasRunway) {
 				escalations++;
@@ -461,6 +548,7 @@ export async function runTurn(
 		// it verbatim (after byte-length validation), keeping shrunk tool results
 		// shrunk so the prompt cache survives and the savings compound.
 		state.compactionPlan = decision.compactionPlan.length > 0 ? decision.compactionPlan : null;
+		state.compactionPlanTokens = decision.compactionPlanTokens;
 		if (usage.cachedTokens > 0 || usage.cacheWriteTokens > 0) {
 			// Non-zero cache traffic is direct evidence the upstream cache exists.
 			state.cacheWarmSlug = servedSlug ?? decision.slug;

package/src/tokens/estimate.ts

@@ -66,6 +66,22 @@ export function estimatePromptTokens(req: NormRequest, tokenizer: string, ledger
 	return tokens;
 }
 
+/**
+ * Replaces the byte count the pending estimate will be calibrated against.
+ *
+ * The estimate is taken from the RAW request, but what the upstream bills is
+ * the prompt after compaction shrank it and the context block was appended.
+ * Pairing raw bytes with billed tokens taught every family a ratio that had
+ * compaction baked in (measured: the compacted estimate ran 33% under the
+ * billed count). The turn orchestrator calls this with the dispatched size
+ * once it knows it, so the ratio describes the tokenizer and nothing else.
+ */
+export function adjustPendingEstimate(conversationKey: string, dispatchedBytes: number): void {
+	const pending = pendingEstimates.get(conversationKey);
+	if (pending === undefined || dispatchedBytes <= 0) return;
+	pending.bytes = dispatchedBytes;
+}
+
 /** Internal: called by cost/ledger.ts when recording a turn. */
 export function consumePendingEstimate(conversationKey: string): { tokenizer: string; bytes: number } | null {
 	const pending = pendingEstimates.get(conversationKey) ?? null;

package/src/upstream/multi.ts

@@ -0,0 +1,26 @@
+/**
+ * One `UpstreamClient` over several providers, keyed by catalog slug prefix.
+ *
+ * `ollama/…` slugs go to the Ollama client; everything else is OpenRouter.
+ * Catalog fetches and the adjudicator's `complete` stay on OpenRouter, whose
+ * catalog is the router's baseline; Ollama's own listing is read by its
+ * catalog source, not through this seam.
+ */
+
+import { isOllamaSlug } from "../catalog/ollama-catalog.ts";
+import type { Dispatch, DispatchOptions, UpstreamClient } from "./types.ts";
+
+export function createMultiUpstream(openrouter: UpstreamClient, ollama: UpstreamClient): UpstreamClient {
+	return {
+		dispatch(opts: DispatchOptions): Promise<Dispatch> {
+			const model = opts.body.model;
+			return typeof model === "string" && isOllamaSlug(model) ? ollama.dispatch(opts) : openrouter.dispatch(opts);
+		},
+		complete(body, signal) {
+			const model = body.model;
+			return typeof model === "string" && isOllamaSlug(model) ? ollama.complete(body, signal) : openrouter.complete(body, signal);
+		},
+		fetchModels: (signal) => openrouter.fetchModels(signal),
+		fetchModelsForUser: (signal) => openrouter.fetchModelsForUser(signal),
+	};
+}

package/src/upstream/ollama-usage.ts

@@ -0,0 +1,157 @@
+/**
+ * Ollama Cloud plan usage, read from `GET https://ollama.com/api/usage`.
+ *
+ * The endpoint is what ollama.com's own dashboard reads. It is not in the
+ * public docs (two open feature requests ask for exactly this), so everything
+ * here is defensive: unknown shape ⇒ null, never a throw, and the raw value is
+ * surfaced on /health so an operator can check it against the dashboard.
+ *
+ * Observed shape (2026-09-06, a Pro account):
+ *
+ *   { activity: { cost: "0.00000", period: {type: "last_4_weeks", …}, models: [] },
+ *     limits:   { monthly: { usage: 0, models: [{ name, request_count }] } } }
+ *
+ * `limits.monthly.usage` is the share of the plan's included monthly credits
+ * consumed. It is reported relative to the plan, which is the point: the
+ * router never needs to know whether the account is Pro ($60) or Max ($300)
+ * to know how close it is to the edge. `request_count` moves immediately;
+ * `usage` and `activity.cost` are aggregated with a lag and round to whole
+ * units, so a few cents of test traffic reads as 0.
+ *
+ * The scale of `usage` is inferred, not documented: a value above 1 is a
+ * percentage; 0 is 0; a non-integer at or below 1 is a fraction. The one
+ * ambiguous reading, exactly 1, is taken as 1% (dashboards show whole
+ * percents) rather than 100%, which errs toward keeping the bias on.
+ */
+
+import type { Logger } from "../util/log.ts";
+
+export interface OllamaUsage {
+	/** Share of the plan's included monthly credits used, 0-1. Null when the payload lacks it. */
+	monthlyUsedFraction: number | null;
+	/** `limits.monthly.usage` exactly as reported, for the dashboard cross-check. */
+	monthlyUsageRaw: number | null;
+	/** `activity.cost` (rolling 4 weeks, USD) as reported, or null. */
+	activityCostUsd: number | null;
+	/** Requests this billing month, summed over models. */
+	requestsThisMonth: number;
+	fetchedAtMs: number;
+}
+
+function asRec(v: unknown): Record<string, unknown> | null {
+	return typeof v === "object" && v !== null && !Array.isArray(v) ? (v as Record<string, unknown>) : null;
+}
+
+/** See the module comment for the scale inference. */
+export function usageFraction(raw: number): number {
+	if (!Number.isFinite(raw) || raw <= 0) return 0;
+	if (raw > 1) return Math.min(1, raw / 100);
+	if (raw === 1) return 0.01;
+	return raw;
+}
+
+export function parseOllamaUsage(json: unknown, nowMs = Date.now()): OllamaUsage | null {
+	const root = asRec(json);
+	if (root === null) return null;
+	const limits = asRec(root.limits);
+	const monthly = limits === null ? null : asRec(limits.monthly);
+	const usageRaw = monthly !== null && typeof monthly.usage === "number" && Number.isFinite(monthly.usage) ? monthly.usage : null;
+	let requests = 0;
+	if (monthly !== null && Array.isArray(monthly.models)) {
+		for (const m of monthly.models) {
+			const rec = asRec(m);
+			if (rec !== null && typeof rec.request_count === "number") requests += rec.request_count;
+		}
+	}
+	const activity = asRec(root.activity);
+	const costRaw = activity?.cost;
+	const cost = typeof costRaw === "number" ? costRaw : typeof costRaw === "string" ? Number(costRaw) : NaN;
+	if (usageRaw === null && limits === null && activity === null) return null;
+	return {
+		monthlyUsedFraction: usageRaw === null ? null : usageFraction(usageRaw),
+		monthlyUsageRaw: usageRaw,
+		activityCostUsd: Number.isFinite(cost) ? cost : null,
+		requestsThisMonth: requests,
+		fetchedAtMs: nowMs,
+	};
+}
+
+export type FetchLike = (input: string, init?: RequestInit) => Promise<Response>;
+
+export interface OllamaUsageSource {
+	/** Latest usage, refreshed when older than the poll interval; last good value on failure. */
+	get(): Promise<OllamaUsage | null>;
+	/** Last fetched value without touching the network. */
+	peek(): OllamaUsage | null;
+}
+
+/** Inert source for setups with no key (the daemon path without `/login ollama-cloud`). */
+export const NO_USAGE: OllamaUsageSource = { get: async () => null, peek: () => null };
+
+export function createOllamaUsageSource(
+	opts: { apiKey: string; pollMs: number; timeoutMs: number; log: Logger; fetchImpl?: FetchLike; root?: string },
+): OllamaUsageSource {
+	if (opts.apiKey === "" || opts.pollMs <= 0) return NO_USAGE;
+	const fetchImpl = opts.fetchImpl ?? fetch;
+	const root = (opts.root ?? "https://ollama.com").replace(/\/+$/, "");
+	let current: OllamaUsage | null = null;
+	let checkedAtMs = 0;
+	let inflight: Promise<OllamaUsage | null> | null = null;
+	let warned = false;
+
+	async function refresh(): Promise<OllamaUsage | null> {
+		try {
+			const res = await fetchImpl(`${root}/api/usage`, {
+				headers: { authorization: `Bearer ${opts.apiKey}` },
+				signal: AbortSignal.timeout(opts.timeoutMs),
+			});
+			if (res.ok) {
+				const parsed = parseOllamaUsage(await res.json());
+				if (parsed !== null) {
+					current = parsed;
+					warned = false;
+				} else if (!warned) {
+					warned = true;
+					opts.log.warn("ollama usage payload had no recognisable fields; credit-aware bias stays on its last reading");
+				}
+			} else if (!warned) {
+				warned = true;
+				opts.log.warn("ollama usage endpoint unavailable; credit-aware bias stays on its last reading", { status: res.status });
+			}
+		} catch (err) {
+			if (!warned) {
+				warned = true;
+				opts.log.warn("ollama usage fetch failed; credit-aware bias stays on its last reading", {
+					error: err instanceof Error ? err.message : String(err),
+				});
+			}
+		}
+		checkedAtMs = Date.now();
+		return current;
+	}
+
+	return {
+		async get() {
+			if (Date.now() - checkedAtMs < opts.pollMs) return current;
+			inflight ??= refresh().finally(() => {
+				inflight = null;
+			});
+			return inflight;
+		},
+		peek: () => current,
+	};
+}
+
+/**
+ * The cost multiplier to apply to Ollama candidates right now: `costBias`
+ * while the plan's included credits are below `biasUntilUsage`, list price
+ * (1) once they are exhausted. Unknown usage keeps the bias — the plan is far
+ * more often under its allowance than over it, and a 402 still trips the
+ * breaker if that guess is wrong.
+ */
+export function effectiveOllamaBias(costBias: number, biasUntilUsage: number, usage: OllamaUsage | null): number {
+	if (costBias >= 1) return costBias;
+	const used = usage?.monthlyUsedFraction ?? null;
+	if (used === null) return costBias;
+	return used >= biasUntilUsage ? 1 : costBias;
+}