auto-model-router 0.31.0 → 0.33.0

package/src/server/http.ts

@@ -1,10 +1,11 @@
-import { mkdirSync } from "node:fs";
-import { dirname } from "node:path";
+import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
 import type { Server } from "bun";
 import { createProviders } from "./providers.ts";
 import { createBridgeFromConfig } from "../context/index.ts";
 import { createFeedbackStore, type Verdict } from "../cost/feedback.ts";
-import { createLedger } from "../cost/ledger.ts";
+import { createSqlLedger } from "../cost/ledger-sql.ts";
 import { createRetentionRunner } from "../cost/retention.ts";
 import { redactionRulesFor } from "../config/redaction.ts";
 import { createSessionOverrides } from "./overrides.ts";
@@ -27,11 +28,11 @@ import { baselinePrices, buildUsageReport, renderUsageReport } from "../cost/rep
 import { decisionEntries, exportCsv, exportRows, feedbackView, harnessScopeParam, spendUsdSince } from "../cost/views.ts";
 import { anthropicErrorResponse, countAnthropicTokens, createMessagesWire } from "../wire/anthropic/messages.ts";
 import { buildDailySummary, createKv, markSummaryShown, renderDailySummary, summaryDue, summaryHasNews, type SummaryOllama } from "../cost/summary.ts";
-import type { Ledger, ModelTrust } from "../cost/types.ts";
+import type { AsyncLedger, ModelTrust } from "../cost/types.ts";
 import { createRouter } from "../router/index.ts";
 import { createConversationStore } from "../router/state.ts";
 import { UpstreamError } from "../upstream/types.ts";
-import { apiKeySource, ollamaKeySource } from "../config/load.ts";
+import { apiKeySource, ollamaKeySource, routerHome } from "../config/load.ts";
 import { ollamaMeter } from "../upstream/ollama-usage.ts";
 import { routerConfigPath } from "../cli/config-cmd.ts";
 import { applyConfigPatch, touched } from "../config/apply.ts";
@@ -40,6 +41,8 @@ import { PINNED_CONFIG_PATHS, watchConfig } from "../config/hot-reload.ts";
 import type { RouterConfig } from "../config/types.ts";
 import { createLogger } from "../util/log.ts";
 import { openDb } from "../util/sqlite.ts";
+import { dialectOf, openSqlDb } from "../util/sql.ts";
+import { migrateStore } from "../util/schema.ts";
 import { WireErrorException, renderErrorEnvelope } from "../wire/openai/errors.ts";
 import { renderModelList } from "../wire/openai/models.ts";
 import { parseChatRequest } from "../wire/openai/request.ts";
@@ -110,14 +113,14 @@ export interface RouterStats {
  * computed over a bounded tail of recent entries; the headline spend numbers
  * use `spendSince`, which is exact.
  */
-export function computeStats(ledger: Ledger, opts?: { windowDays?: number; nowMs?: number }): RouterStats {
+export async function computeStats(ledger: AsyncLedger, opts?: { windowDays?: number; nowMs?: number }): Promise<RouterStats> {
 	const nowMs = opts?.nowMs ?? Date.now();
 	const windowDays = opts?.windowDays;
 	const cutoffMs = windowDays === undefined ? 0 : nowMs - windowDays * 86_400_000;
 
 	// 100k turns is operational eternity for a single-operator router; the cap
 	// only bounds memory on this read, never what the ledger retains.
-	const entries = ledger.recentEntries(100_000).filter((e) => e.createdAtMs >= cutoffMs);
+	const entries = (await ledger.recentEntries(100_000)).filter((e) => e.createdAtMs >= cutoffMs);
 
 	const dayStart = new Date(nowMs);
 	dayStart.setHours(0, 0, 0, 0);
@@ -157,16 +160,16 @@ export function computeStats(ledger: Ledger, opts?: { windowDays?: number; nowMs
 	return {
 		generatedAtMs: nowMs,
 		windowDays: windowDays ?? null,
-		spendTodayUsd: ledger.spendSince(dayStart.getTime()),
-		spend7dUsd: ledger.spendSince(nowMs - 7 * 86_400_000),
-		spendAllTimeUsd: ledger.spendSince(0),
+		spendTodayUsd: await ledger.spendSince(dayStart.getTime()),
+		spend7dUsd: await ledger.spendSince(nowMs - 7 * 86_400_000),
+		spendAllTimeUsd: await ledger.spendSince(0),
 		windowSpendUsd,
 		requests: entries.length,
 		escalations,
 		escalationRate: entries.length > 0 ? escalations / entries.length : 0,
 		meanPredictionError: errorSamples > 0 ? errorSum / errorSamples : null,
 		perModel: rows,
-		trust: ledger.allTrust(),
+		trust: await ledger.allTrust(),
 	};
 }
 
@@ -250,16 +253,47 @@ function clampDays(raw: string | null, dflt: number): number {
 export function startServer(cfg: RouterConfig): StartedServer {
 	const log = createLogger(cfg.logLevel);
 
-	if (cfg.ledger.path !== ":memory:") mkdirSync(dirname(cfg.ledger.path), { recursive: true });
-	const db = openDb(cfg.ledger.path);
-	const ledger = createLedger(db, cfg);
-	const providers = createProviders(cfg, db, log);
+	// `:memory:` names a store that cannot be shared: a second handle gets its
+	// OWN empty database, so the ledger views would query a table that does not
+	// exist there. Resolve it to a private file instead — still discarded, still
+	// isolated per server, but one store that both handles agree on.
+	const memoryStore = cfg.ledger.path === ":memory:" ? mkdtempSync(join(tmpdir(), "amr-mem-")) : null;
+	// The config is NOT rewritten: a caller that asked for `:memory:` still
+	// reads `:memory:` back, and `reconfigure` refuses a ledger-path change by
+	// comparing against what it was given.
+	const storePath = memoryStore === null ? cfg.ledger.path : join(memoryStore, "router.db");
+	// Two stores, chosen by what the data is for:
+	//   - the SHARED store (`ledger.path`, SQLite file or Postgres URL) holds
+	//     what a second replica must see the same copy of: the turn rows a cap
+	//     counts, conversation routing memory, context blocks.
+	//   - the LOCAL store is always a SQLite file and holds only caches — the
+	//     catalog payload, benchmark feeds, the once-a-day summary marker.
+	//     Sharing a cache buys contention and nothing else, and its one durable
+	//     output (`local_scores`) belongs to the machine that measured it.
+	// On SQLite they are the same file, which is what every existing install
+	// already has.
+	const postgres = dialectOf(storePath) === "postgres";
+	const cachePath = postgres ? join(routerHome(), "cache.db") : storePath;
+	mkdirSync(dirname(cachePath), { recursive: true });
+	// `openDb` is the migration path for a SQLite file: nineteen versions, in
+	// order, on whatever an older release left behind.
+	const db = openDb(cachePath);
+	const sqlDb = openSqlDb(storePath);
+	// A Postgres store has no bootstrap of its own to run synchronously, so the
+	// shape is created on the way up and every entry point waits for it once.
+	// Resolved already on SQLite, where `openDb` just did it.
+	const storeReady = postgres ? migrateStore(sqlDb) : Promise.resolve();
+	// The ledger reads and writes through the engine-agnostic handle. `findModel`
+	// closes over the catalog built just below: a shared store has no catalog
+	// cache of its own to price a row from.
+	const ledger = createSqlLedger(sqlDb, cfg, { findModel: (slug: string) => catalog.find(slug) });
+	const providers = createProviders(cfg, db, sqlDb, log);
 	const { upstream, catalog, ollama, ollamaServing, ollamaUsage, ollamaCostScale } = providers;
-	const conversations = createConversationStore(db);
+	const conversations = createConversationStore(sqlDb);
 	const router = createRouter({ config: cfg, catalog, ledger, conversations, upstream });
-	const context = createBridgeFromConfig(cfg, db);
+	const context = createBridgeFromConfig(cfg, sqlDb);
 	const overrides = createSessionOverrides();
-	const feedback = createFeedbackStore(db);
+	const feedback = createFeedbackStore(sqlDb);
 	const kv = createKv(db);
 	/**
 	 * Background local-benchmark runs. In memory and not persisted: a run is an explicit,
@@ -341,9 +375,10 @@ export function startServer(cfg: RouterConfig): StartedServer {
 	// run a whole-ledger delete more often than that. It reads the window live,
 	// so a hot reload that lowers it applies on the next tick.
 	const retention = createRetentionRunner({ ledger, retentionDays: () => cfg.ledger.retentionDays });
-	const retain = (): void => {
+	const retain = async (): Promise<void> => {
+		await storeReady;
 		try {
-			const result = retention.maybeRun();
+			const result = await retention.maybeRun();
 			if (result !== null && result.deleted > 0) {
 				log.info("pruned ledger rows past retention", { deleted: result.deleted, retentionDays: cfg.ledger.retentionDays });
 			}
@@ -355,8 +390,12 @@ export function startServer(cfg: RouterConfig): StartedServer {
 	// One housekeeping timer for all three tables. `unref`'d so it never holds
 	// the process open.
 	const pruneTimer = setInterval(() => {
+		// Fire and forget, with both halves guarded: a housekeeping failure must
+		// never become an unhandled rejection that takes the process down.
+		void (async () => {
+		await storeReady;
 		try {
-			const dropped = conversations.prune(cfg.ledger.conversationTtlMs);
+			const dropped = await conversations.prune(cfg.ledger.conversationTtlMs);
 			if (dropped > 0) log.debug("pruned stale conversations", { dropped });
 		} catch (err) {
 			log.warn("conversation prune failed", { error: err instanceof Error ? err.message : String(err) });
@@ -366,18 +405,21 @@ export function startServer(cfg: RouterConfig): StartedServer {
 			// block of that age has no future reader. Nothing else reclaims these:
 			// blocks are content-addressed and shared, so they accumulated for the
 			// life of the install (measured: 220 rows / 2.7 MB, 68 unreferenced).
-			const dropped = context.pruneBlocks(cfg.context.maxStalenessMs);
+			const dropped = await context.pruneBlocks(cfg.context.maxStalenessMs);
 			if (dropped > 0) log.debug("pruned unreferenced context blocks", { dropped });
 		} catch (err) {
 			log.warn("context block prune failed", { error: err instanceof Error ? err.message : String(err) });
 		}
-		retain();
+			await retain();
+		})();
 	}, 60_000);
 	pruneTimer.unref();
 
 	// Once shortly after boot, so a lowered window takes effect without waiting
 	// out an hour; the runner's own floor governs everything after that.
-	setTimeout(retain, 5_000).unref();
+	// `void`: a boot-time prune failure is logged inside `retain`, and an
+	// unhandled rejection here would take the process down.
+	setTimeout(() => void retain(), 5_000).unref();
 
 	// Periodically refetch the (key-scoped) catalog in the background so
 	// guardrail/preference changes are picked up without needing traffic and a
@@ -483,6 +525,10 @@ export function startServer(cfg: RouterConfig): StartedServer {
 		// SECONDS (max 255), so convert from the ms upstream timeout and cap.
 		idleTimeout: Math.min(Math.ceil(cfg.openrouter.timeoutMs / 1000), 255),
 		async fetch(req: Request): Promise<Response> {
+			// One await on the first request of a Postgres deployment, already
+			// resolved on SQLite: no route may read a table the bootstrap has
+			// not created yet.
+			await storeReady;
 			// Reject requests whose Host header does not name a loopback address
 			// when the server is bound to loopback. This blunts DNS rebinding: a
 			// malicious page resolving a host to 127.0.0.1 sends its own domain as
@@ -523,17 +569,19 @@ export function startServer(cfg: RouterConfig): StartedServer {
 				}
 				if (req.method === "POST" && url.pathname === "/v1/messages/count_tokens") {
 					try {
-						return json({ input_tokens: countAnthropicTokens(await req.json(), cfg.anthropic.models, ledger) });
+						return json({
+							input_tokens: countAnthropicTokens(await req.json(), cfg.anthropic.models, await ledger.tokenRatio("anthropic")),
+						});
 					} catch (err) {
 						if (err instanceof WireErrorException) return anthropicErrorResponse(err.wireError);
 						return anthropicErrorResponse({ status: 400, code: "invalid_json", message: err instanceof Error ? err.message : "request body is not valid JSON" });
 					}
 				}
 				if (req.method === "GET" && url.pathname === "/v1/models") {
-					return json(renderModelList(cfg, ledger.blendedRate(cfg.ledger.blendWindowDays)));
+					return json(renderModelList(cfg, await ledger.blendedRate(cfg.ledger.blendWindowDays)));
 				}
 				if (req.method === "GET" && url.pathname === "/v1/router/stats") {
-					return json(computeStats(ledger));
+					return json(await computeStats(ledger));
 				}
 				if (req.method === "GET" && url.pathname === "/v1/router/catalog") {
 					// The catalog as data, judged under `?policy=` (the X-Omp-Policy
@@ -576,15 +624,15 @@ export function startServer(cfg: RouterConfig): StartedServer {
 					if (!Number.isFinite(since)) return wireErrorResponse({ status: 400, code: "invalid_request_error", message: "sinceMs required" });
 					// `scope` narrows to one agentdox context scope: a project's own spend.
 					const contextScope = url.searchParams.get("scope") ?? "";
-					return json({ sinceMs: since, usd: spendUsdSince(db, since, harnessScopeParam(url.searchParams.get("harness")), contextScope), ...(contextScope === "" ? {} : { scope: contextScope }) });
+					return json({ sinceMs: since, usd: await spendUsdSince(sqlDb, since, harnessScopeParam(url.searchParams.get("harness")), contextScope), ...(contextScope === "" ? {} : { scope: contextScope }) });
 				}
 				if (req.method === "GET" && url.pathname === "/v1/router/feedback") {
 					const days = clampDays(url.searchParams.get("days"), 30);
-					return json({ days, ...feedbackView(db, Date.now() - days * 86_400_000, harnessScopeParam(url.searchParams.get("harness"))) });
+					return json({ days, ...await feedbackView(sqlDb, Date.now() - days * 86_400_000, harnessScopeParam(url.searchParams.get("harness"))) });
 				}
 				if (req.method === "GET" && url.pathname === "/v1/router/export") {
 					const days = clampDays(url.searchParams.get("days"), 30);
-					const rows = exportRows(db, Date.now() - days * 86_400_000, harnessScopeParam(url.searchParams.get("harness")));
+					const rows = await exportRows(sqlDb, Date.now() - days * 86_400_000, harnessScopeParam(url.searchParams.get("harness")));
 					if (url.searchParams.get("format") === "json") return json({ days, rows });
 					return new Response(exportCsv(rows), { headers: { "content-type": "text/csv; charset=utf-8", "content-disposition": `attachment; filename="auto-model-router-export-${new Date().toISOString().slice(0, 10)}.csv"` } });
 				}
@@ -593,7 +641,7 @@ export function startServer(cfg: RouterConfig): StartedServer {
 					// optional harness scope (the X-Omp-Harness header value).
 					const windowDays = clampDays(url.searchParams.get("days"), 7);
 					const harnessId = url.searchParams.get("harness") ?? "";
-					const report = buildUsageReport(db, { windowDays, harnessId, baselines: baselinePrices(cfg.report.baselines, (s) => catalog.find(s)) });
+					const report = await buildUsageReport(sqlDb, { windowDays, harnessId, baselines: baselinePrices(cfg.report.baselines, (s) => catalog.find(s)) });
 					// ?format=text: the rendered report for harnesses without a renderer of their own (the Hermes plugin).
 					if (url.searchParams.get("format") === "text") return new Response(renderUsageReport(report), { headers: { "content-type": "text/plain; charset=utf-8" } });
 					return json(report);
@@ -627,13 +675,13 @@ export function startServer(cfg: RouterConfig): StartedServer {
 					if (auto && !cfg.report.dailySummary) return json({ due: false, reason: "report.dailySummary is off", summary: null });
 					if (auto && !summaryDue(kv, harnessId)) return json({ due: false, reason: "posted in the last 20h", summary: null });
 					const meter = ollamaMeter(ollamaUsage.peek(), cfg.ollama.planCreditsUsd);
-					const runway = ollamaRunway(meter, ledger.providerSpendSince?.("ollama/", Date.now() - 7 * 86_400_000) ?? 0, ollamaUsage.calibration()?.factor ?? 1);
+					const runway = ollamaRunway(meter, await ledger.providerSpendSince("ollama/", Date.now() - 7 * 86_400_000), ollamaUsage.calibration()?.factor ?? 1);
 					const ollamaSummary: SummaryOllama | null =
 						!cfg.ollama.enabled || meter === null ? null : { plan: meter.plan ?? null, usedUsd: meter.usedUsd, creditsUsd: meter.creditsUsd, runwayDays: runway?.days ?? null };
-					const summary = buildDailySummary(db, {
+					const summary = await buildDailySummary(sqlDb, {
 						harnessId,
 						baselines: baselinePrices(cfg.report.baselines, (s) => catalog.find(s)),
-						spikes: ledger.softFailureSpikes?.() ?? [],
+						spikes: await ledger.softFailureSpikes(),
 						ollama: ollamaSummary,
 					});
 					if (auto && !summaryHasNews(summary)) return json({ due: false, reason: "nothing to report", summary: null });
@@ -651,7 +699,7 @@ export function startServer(cfg: RouterConfig): StartedServer {
 					const sinceRaw = Number.parseInt(url.searchParams.get("since") ?? "", 10);
 					const daysRaw = url.searchParams.get("days");
 					const sinceMs = Number.isFinite(sinceRaw) ? sinceRaw : daysRaw === null ? 0 : Date.now() - clampDays(daysRaw, 30) * 86_400_000;
-					const entries = decisionEntries(db, {
+					const entries = await decisionEntries(sqlDb, {
 						sinceMs,
 						harness: harnessScopeParam(url.searchParams.get("harness")),
 						limit,
@@ -717,7 +765,7 @@ export function startServer(cfg: RouterConfig): StartedServer {
 					// ledger file by design, so this route is the only way it can act
 					// on its own retention policy — and the once-an-hour floor is the
 					// runner's, not the caller's, so calling it in a loop is harmless.
-					const result = retention.runNow();
+					const result = await retention.runNow();
 					if (result.deleted > 0) log.info("pruned ledger rows past retention", { deleted: result.deleted, retentionDays: cfg.ledger.retentionDays });
 					return json({ ...result, retentionDays: cfg.ledger.retentionDays });
 				}
@@ -853,10 +901,10 @@ export function startServer(cfg: RouterConfig): StartedServer {
 					}
 					const target =
 						typeof body?.ledgerId === "string"
-							? (ledger.recentEntries(1_000).find((e) => e.id === body.ledgerId) ?? null)
-							: (ledger.latestForSession?.(session) ?? null);
+							? ((await ledger.recentEntries(1_000)).find((e) => e.id === body.ledgerId) ?? null)
+							: ((await ledger.latestForSession(session)) ?? null);
 					if (target === null) return wireErrorResponse({ status: 404, code: "not_found", message: "no routed turn for that session yet" });
-					const id = feedback.record({
+					const id = await feedback.record({
 						ledgerId: target.id,
 						ompSessionId: session,
 						slug: target.servedSlug ?? target.slug,
@@ -870,6 +918,12 @@ export function startServer(cfg: RouterConfig): StartedServer {
 					const snap = catalog.peek();
 					return json({
 						status: "ok",
+						// What a front door may rely on, by name rather than by version: a
+						// team edition that sends per-turn credentials to a router without
+						// `upstream-keys` would have them ignored, and its tenants served on
+						// the deployment's own credential — a silent cross-charge. A name it
+						// can check turns that into a refusal it can explain.
+						features: ["upstream-keys"],
 						apiKeyConfigured: cfg.openrouter.apiKey !== "",
 						// Which upstreams turns can actually be served from: OpenRouter needs
 						// its key; Ollama needs to be on and out of cooldown.
@@ -904,12 +958,12 @@ export function startServer(cfg: RouterConfig): StartedServer {
 										meter: ollamaMeter(ollamaUsage.peek(), cfg.ollama.planCreditsUsd),
 										// Ledger vs meter, and how long the credits last at the recent burn.
 										calibration: ollamaUsage.calibration(),
-										runway: ollamaRunway(ollamaMeter(ollamaUsage.peek(), cfg.ollama.planCreditsUsd), ledger.providerSpendSince?.("ollama/", Date.now() - 7 * 86_400_000) ?? 0, ollamaUsage.calibration()?.factor ?? 1),
+										runway: ollamaRunway(ollamaMeter(ollamaUsage.peek(), cfg.ollama.planCreditsUsd), await ledger.providerSpendSince("ollama/", Date.now() - 7 * 86_400_000), ollamaUsage.calibration()?.factor ?? 1),
 										costBias: { configured: cfg.ollama.costBias, effective: catalog.ollamaBias?.() ?? cfg.ollama.costBias, biasUntilUsage: cfg.ollama.biasUntilUsage },
 									},
 						// Models failing well above their own baseline in the last hour.
 						// Visibility only: nothing routes around a spike.
-						softFailures: { recentMs: 3_600_000, baselineDays: 7, spikes: ledger.softFailureSpikes?.() ?? [] },
+						softFailures: { recentMs: 3_600_000, baselineDays: 7, spikes: await ledger.softFailureSpikes() },
 						catalog: snap === null
 							? null
 							: {
@@ -1000,7 +1054,19 @@ export function startServer(cfg: RouterConfig): StartedServer {
 			// Drain queued agentdox write-backs before the DB closes under them.
 			context.close();
 			await context.flush();
+			// Both handles on the store, or Windows keeps the file locked and a
+			// caller that deletes its temp directory gets EBUSY.
+			await sqlDb.close();
 			db.close();
+			if (memoryStore !== null) {
+				// A `:memory:` ledger promised nothing durable, so its stand-in file
+				// goes with the server. Best-effort: Windows may still hold the WAL.
+				try {
+					rmSync(memoryStore, { recursive: true, force: true });
+				} catch {
+					/* the OS reclaims a temp directory soon enough */
+				}
+			}
 		},
 	};
 }

package/src/server/providers.ts

@@ -5,6 +5,7 @@
  */
 
 import type { Database } from "bun:sqlite";
+import { num, type SqlDb } from "../util/sql.ts";
 import { createCompositeCatalog } from "../catalog/composite.ts";
 import { createStaticCatalogSource } from "../catalog/static-catalog.ts";
 import { createAnthropicClient } from "../upstream/anthropic.ts";
@@ -15,7 +16,6 @@ import type { CatalogSnapshot, CatalogSource } from "../catalog/types.ts";
 import type { RouterConfig } from "../config/types.ts";
 import { createMultiUpstream } from "../upstream/multi.ts";
 import { createOllamaClient, type OllamaClient } from "../upstream/ollama.ts";
-import { createLedger } from "../cost/ledger.ts";
 import { setKnownUpstreamIds } from "../cost/report.ts";
 import { createOllamaUsageSource, type OllamaUsageSource } from "../upstream/ollama-usage.ts";
 import { createOpenRouterClient } from "../upstream/openrouter.ts";
@@ -39,7 +39,13 @@ export interface Providers {
 	namedServing(): string[];
 }
 
-export function createProviders(cfg: RouterConfig, db: Database, log: Logger = createLogger(cfg.logLevel)): Providers {
+export function createProviders(
+	cfg: RouterConfig,
+	db: Database,
+	/** The engine-agnostic handle on the same store, for the calibration samples. */
+	sqlDb: SqlDb,
+	log: Logger = createLogger(cfg.logLevel),
+): Providers {
 	const openrouter = createOpenRouterClient(cfg);
 	const openrouterCatalog = createCatalog(cfg, openrouter, db);
 	// Ollama Cloud is a second upstream ranked in the same catalog: `ollama/…`
@@ -51,7 +57,15 @@ export function createProviders(cfg: RouterConfig, db: Database, log: Logger = c
 	const ollamaServing = (): boolean => cfg.ollama.enabled && ollama.available();
 	// Plan usage lives on ollama.com whichever base URL dispatches; it needs the
 	// key, so the daemon path without `/login ollama-cloud` keeps a static bias.
-	const ledgerForCalibration = createLedger(db, cfg);
+	// Only the ledger's Ollama total is needed, so this reads through the shim
+	// rather than constructing a second full ledger.
+	const ollamaLedgerUsd = async (): Promise<number> => {
+		const row = await sqlDb.one<{ total: unknown }>(
+			"SELECT COALESCE(SUM(COALESCE(reported_usd, predicted_usd)), 0) AS total FROM ledger WHERE COALESCE(served_slug, slug) LIKE $prefix",
+			{ prefix: "ollama/%" },
+		);
+		return num(row?.total);
+	};
 	const ollamaUsage = createOllamaUsageSource({
 		apiKey: () => cfg.ollama.apiKey,
 		pollMs: cfg.ollama.usagePollMs,
@@ -59,7 +73,7 @@ export function createProviders(cfg: RouterConfig, db: Database, log: Logger = c
 		log,
 		// Each poll records the meter beside the ledger's Ollama total, so the
 		// estimate can be scaled to what ollama.com actually bills.
-		calibration: { db, ledgerUsd: () => ledgerForCalibration.providerSpendSince?.("ollama/", 0) ?? 0, planCreditsOverrideUsd: cfg.ollama.planCreditsUsd },
+		calibration: { db: sqlDb, ledgerUsd: ollamaLedgerUsd, planCreditsOverrideUsd: cfg.ollama.planCreditsUsd },
 	});
 	// Named upstreams (OpenAI, Azure, Anthropic, vLLM…): a client per id, built when
 	// first needed and kept — its breaker state must survive config reloads — while

package/src/server/turn.ts

@@ -14,7 +14,7 @@ import type { ContextBridge } from "../context/types.ts";
 import type { RouterConfig } from "../config/types.ts";
 import { estimateUnreportedCache } from "../cost/cache-estimate.ts";
 import { computeCost } from "../cost/forecast.ts";
-import { EMPTY_USAGE, type Ledger, type UsageCounts } from "../cost/types.ts";
+import { EMPTY_USAGE, type AsyncLedger, type UsageCounts } from "../cost/types.ts";
 import { createProbe, type Probe } from "../router/escalate.ts";
 import { resolveHoldTurns } from "../router/explore.ts";
 import { adjustPendingEstimate } from "../tokens/estimate.ts";
@@ -65,7 +65,7 @@ export interface TurnDeps {
 	config: RouterConfig;
 	router: Router;
 	upstream: UpstreamClient;
-	ledger: Ledger;
+	ledger: AsyncLedger;
 	conversations: ConversationStore;
 	catalog: CatalogSource;
 	/** agentdox bridge. The disabled bridge makes every call here a no-op. */
@@ -86,6 +86,17 @@ class SinkError extends Error {
 	}
 }
 
+/**
+ * A deliberate refusal from `route()` — `BudgetExceededError` today — reports
+ * the status and code it chose. Anything else is an internal failure and gets
+ * a 500, so a new throw never silently becomes a 4xx.
+ */
+function routerRefusal(err: unknown): { status: number; code: string } | null {
+	if (!(err instanceof Error) || !("status" in err) || !("code" in err)) return null;
+	const { status, code } = err as Error & { status: unknown; code: unknown };
+	return typeof status === "number" && typeof code === "string" ? { status, code } : null;
+}
+
 /** Latest user text, used to bias agentdox relevance ranking and as the recorded turn. */
 function lastUserText(req: NormRequest): string {
 	for (let i = req.messages.length - 1; i >= 0; i--) {
@@ -122,7 +133,7 @@ export async function runTurn(
 ): Promise<void> {
 	const { config, router, upstream, ledger, conversations, context: bridge } = deps;
 	const log = createLogger(config.logLevel);
-	const state = conversations.load(req.conversationKey);
+	const state = await conversations.load(req.conversationKey);
 	const turnNumber = state.turn + 1;
 	// Digest quality signal: the calls the agent just made, matched against
 	// recent digests of this session (a re-run of a digested read means the
@@ -131,7 +142,9 @@ export async function runTurn(
 		for (let i = req.messages.length - 1; i >= 0; i--) {
 			const m = req.messages[i];
 			if (m === undefined || m.role !== "assistant") continue;
-			if (m.toolCalls.length > 0) deps.digester.noteToolCalls(req.ompSessionId, m.toolCalls.map((c) => ({ name: c.name, argsJson: c.argsJson })));
+			if (m.toolCalls.length > 0) {
+				await deps.digester.noteToolCalls(req.ompSessionId, m.toolCalls.map((c) => ({ name: c.name, argsJson: c.argsJson })));
+			}
 			break;
 		}
 	}
@@ -191,7 +204,17 @@ export async function runTurn(
 				}
 				decision = await router.route(req, opts);
 			} catch (err) {
-				await sink.error({ status: 500, code: "router_error", message: err instanceof Error ? err.message : String(err) });
+				// A refusal the router MEANS carries its own status and code — a
+				// budget cap in `reject` mode is a 402 `budget_exceeded`, not an
+				// internal failure. Reported as a 500 it reads to the caller (and
+				// to a team front door relaying it) as "the router broke", which
+				// sent a capped deployment looking for a crash.
+				const refusal = routerRefusal(err);
+				await sink.error({
+					status: refusal?.status ?? 500,
+					code: refusal?.code ?? "router_error",
+					message: err instanceof Error ? err.message : String(err),
+				});
 				return;
 			}
 		}
@@ -314,7 +337,7 @@ export async function runTurn(
 				generationId = await dispatch.generationId().catch(() => null);
 			}
 			const priceModel = deps.catalog.find(servedSlug ?? decision.slug);
-			ledger.record({
+			await ledger.record({
 				id: crypto.randomUUID(),
 				createdAtMs: Date.now(),
 				conversationKey: req.conversationKey,
@@ -364,7 +387,7 @@ export async function runTurn(
 			// the committed path used to reach the state update below, so aborted
 			// dispatches (30% of real spend on live data) stayed invisible to the
 			// per-conversation budget guard.
-			conversations.accrue(req.conversationKey, { spentUsd: reportedUsd ?? decision.forecast.expectedUsd });
+			await conversations.accrue(req.conversationKey, { spentUsd: reportedUsd ?? decision.forecast.expectedUsd });
 		};
 
 		// Same-tier failover: re-route with every failed slug excluded and accept
@@ -449,7 +472,9 @@ export async function runTurn(
 		let streamEnded = false;
 
 		try {
-			dispatch = await upstream.dispatch({ body, sessionId: decision.sessionId, signal: attemptSignal });
+			// Inside the attempt loop, so a retry, a same-tier failover and a tier
+			// escalation all dispatch with the same per-turn credentials.
+			dispatch = await upstream.dispatch({ body, sessionId: decision.sessionId, signal: attemptSignal, ...(req.upstreamKeys === undefined ? {} : { upstreamKeys: req.upstreamKeys }) });
 		} catch (err) {
 			streamError = err;
 		}
@@ -641,7 +666,7 @@ export async function runTurn(
 		state.currentTier = decision.tier;
 		// Escalations accumulate in SQL for the same reason spend does: `save`
 		// below no longer writes this column, so a snapshot cannot clobber it.
-		conversations.accrue(req.conversationKey, { escalations });
+		await conversations.accrue(req.conversationKey, { escalations });
 		// Hysteresis window. Only re-arm when the served tier actually changed
 		// (or this turn escalated). Re-arming on EVERY turn — even a trivial one
 		// served by a held hard model — extends the lock forever: the classifier
@@ -672,7 +697,7 @@ export async function runTurn(
 			state.cacheWarmAtMs = Date.now();
 		}
 		state.updatedAtMs = Date.now();
-		conversations.save(state);
+		await conversations.save(state);
 
 		// Record the settled turn into agentdox, attributed to the model that
 		// actually served it. Queued and never awaited: the transcript is an

package/src/tokens/estimate.ts

@@ -6,7 +6,6 @@
  * `token_calibration` per tokenizer family on measurement.
  */
 
-import type { Ledger } from "../cost/types.ts";
 import type { NormRequest } from "../wire/types.ts";
 
 /** Chars-per-token when nothing better is known. Conservative-ish for mixed prose+code. */
@@ -39,9 +38,20 @@ function familyKey(tokenizer: string): string {
 	return tokenizer.trim().toLowerCase();
 }
 
-export function estimateTokens(bytes: number, tokenizer: string, ledger: Ledger | null): number {
-	const ratio = ledger?.tokenRatio(tokenizer) ?? FAMILY_BYTES_PER_TOKEN[familyKey(tokenizer)] ?? DEFAULT_BYTES_PER_TOKEN;
-	return Math.max(0, Math.ceil(bytes / ratio));
+/**
+ * A measured bytes-per-token ratio for a tokenizer family, or null when the
+ * ledger has too few samples to have calibrated one.
+ *
+ * Passed as a VALUE rather than read from the ledger here: estimation runs
+ * inside synchronous code (the Anthropic token counter, candidate scoring), and
+ * the ledger may be a shared database that cannot be read synchronously. The
+ * caller fetches the one ratio it needs before estimating.
+ */
+export type TokenRatio = number | null;
+
+export function estimateTokens(bytes: number, tokenizer: string, ratio: TokenRatio): number {
+	const perToken = ratio ?? FAMILY_BYTES_PER_TOKEN[familyKey(tokenizer)] ?? DEFAULT_BYTES_PER_TOKEN;
+	return Math.max(0, Math.ceil(bytes / perToken));
 }
 
 /**
@@ -53,10 +63,10 @@ export function estimateTokens(bytes: number, tokenizer: string, ledger: Ledger
 const PENDING_CAP = 1024;
 const pendingEstimates = new Map<string, { tokenizer: string; bytes: number }>();
 
-export function estimatePromptTokens(req: NormRequest, tokenizer: string, ledger: Ledger | null): number {
+export function estimatePromptTokens(req: NormRequest, tokenizer: string, ratio: TokenRatio): number {
 	let images = 0;
 	for (const message of req.messages) images += message.images;
-	const tokens = estimateTokens(req.promptBytes, tokenizer, ledger) + images * IMAGE_TOKEN_ALLOWANCE;
+	const tokens = estimateTokens(req.promptBytes, tokenizer, ratio) + images * IMAGE_TOKEN_ALLOWANCE;
 	if (pendingEstimates.size >= PENDING_CAP && !pendingEstimates.has(req.conversationKey)) {
 		// Map iteration order is insertion order: drop the eldest.
 		const eldest = pendingEstimates.keys().next();

package/src/upstream/anthropic.ts

@@ -410,13 +410,13 @@ export function createAnthropicClient(cfg: RouterConfig, id: string, fetchImpl:
 		if (caller && timeout) return AbortSignal.any([caller, timeout]);
 		return caller ?? timeout;
 	}
-	async function post(e: UpstreamEntry, body: Record<string, unknown>, signal: AbortSignal | undefined): Promise<Response> {
+	async function post(e: UpstreamEntry, body: Record<string, unknown>, signal: AbortSignal | undefined, apiKey: string = e.apiKey): Promise<Response> {
 		const headers: Record<string, string> = { "content-type": "application/json", "anthropic-version": ANTHROPIC_VERSION, ...e.headers };
 		if (e.auth === "oauth-bearer") {
 			// A Claude Pro/Max subscription token: Bearer auth at the first-party API, with the OAuth beta. No per-token cost is reported.
-			headers["authorization"] = `Bearer ${e.apiKey}`;
+			headers["authorization"] = `Bearer ${apiKey}`;
 			headers["anthropic-beta"] = e.headers["anthropic-beta"] ?? "oauth-2025-04-20,claude-code-20250219";
-		} else if (e.apiKey !== "") headers["x-api-key"] = e.apiKey;
+		} else if (apiKey !== "") headers["x-api-key"] = apiKey;
 		try {
 			return await fetchImpl(`${e.baseUrl.replace(/\/+$/, "")}/v1/messages`, { method: "POST", headers, body: JSON.stringify(body), signal: composeSignal(e, signal) });
 		} catch (err) {
@@ -444,7 +444,8 @@ export function createAnthropicClient(cfg: RouterConfig, id: string, fetchImpl:
 		async dispatch(opts: DispatchOptions): Promise<Dispatch> {
 			const e = entry();
 			const { rendered, servedSlug } = render(e, { ...opts.body, stream: true });
-			const res = await post(e, rendered, opts.signal);
+			// A per-turn credential for this upstream wins for this dispatch only; the shared entry is never touched.
+			const res = await post(e, rendered, opts.signal, opts.upstreamKeys?.[id] ?? e.apiKey);
 			if (!res.ok) throw await httpError(res);
 			if (!res.body) throw new UpstreamError("upstream_error", res.status, "response had no body", true);
 			const translator = createAnthropicTranslator(servedSlug);

package/src/upstream/compat.ts

@@ -113,15 +113,20 @@ function transportError(id: string, err: unknown): UpstreamError {
 	return new UpstreamError("network", 0, err instanceof Error ? err.message : String(err), true);
 }
 
-/** The chat-completions URL and auth for an entry: Azure names the deployment in the path and keys with `api-key`. */
-export function compatEndpoint(entry: UpstreamEntry, modelId: string): { url: string; headers: Record<string, string> } {
+/**
+ * The chat-completions URL and auth for an entry: Azure names the deployment in the path and keys with `api-key`.
+ *
+ * `apiKey` overrides the entry's own credential for one dispatch (a per-turn
+ * key from the front door). The entry itself is never written to.
+ */
+export function compatEndpoint(entry: UpstreamEntry, modelId: string, apiKey: string = entry.apiKey): { url: string; headers: Record<string, string> } {
 	const base = entry.baseUrl.replace(/\/+$/, "");
 	const headers: Record<string, string> = { "content-type": "application/json", ...entry.headers };
 	if (entry.kind === "azure") {
-		if (entry.apiKey !== "") headers["api-key"] = entry.apiKey;
+		if (apiKey !== "") headers["api-key"] = apiKey;
 		return { url: `${base}/openai/deployments/${encodeURIComponent(modelId)}/chat/completions?api-version=${encodeURIComponent(entry.apiVersion)}`, headers };
 	}
-	if (entry.apiKey !== "") headers.authorization = `Bearer ${entry.apiKey}`;
+	if (apiKey !== "") headers.authorization = `Bearer ${apiKey}`;
 	return { url: `${base}/chat/completions`, headers };
 }
 
@@ -195,8 +200,8 @@ export function createCompatClient(cfg: RouterConfig, id: string, fetchImpl: Fet
 		return err;
 	}
 
-	async function post(e: UpstreamEntry, body: Record<string, unknown>, signal: AbortSignal | undefined): Promise<Response> {
-		const { url, headers } = compatEndpoint(e, typeof body.model === "string" ? body.model : "");
+	async function post(e: UpstreamEntry, body: Record<string, unknown>, signal: AbortSignal | undefined, apiKey: string = e.apiKey): Promise<Response> {
+		const { url, headers } = compatEndpoint(e, typeof body.model === "string" ? body.model : "", apiKey);
 		try {
 			return await fetchImpl(url, { method: "POST", headers, body: JSON.stringify(body), signal: composeSignal(e, signal) });
 		} catch (err) {
@@ -212,7 +217,8 @@ export function createCompatClient(cfg: RouterConfig, id: string, fetchImpl: Fet
 
 		async dispatch(opts: DispatchOptions): Promise<Dispatch> {
 			const e = entry();
-			const res = await post(e, toCompatBody(id, { ...opts.body, stream: true }), opts.signal);
+			// A per-turn credential for this upstream wins for this dispatch only; the shared entry is never touched.
+			const res = await post(e, toCompatBody(id, { ...opts.body, stream: true }), opts.signal, opts.upstreamKeys?.[id] ?? e.apiKey);
 			if (!res.ok) throw await httpError(res);
 			if (!res.body) throw new UpstreamError("upstream_error", res.status, "response had no body", true);
 			const parsed = parseSse(res.body, (msg, fields) => log.warn(msg, fields));