auto-model-router 0.4.2 → 0.4.4

package/test/digest.test.ts

@@ -162,6 +162,50 @@ describe("createDigester", () => {
 		db2.close();
 	});
 
+	test("a compaction-sourced digest is gated on compaction.digestToolResults and judges the given tier", async () => {
+		const cfg = cfgWith({ enabled: false });
+		cfg.compaction.digestToolResults = true;
+		const db = openDb(":memory:");
+		const ledger = createLedger(db, cfg);
+		// No session rows at all: the tier comes from the request.
+		const { upstream, calls } = fakeUpstream(() => "Condensed.");
+		const d = createDigester({ cfg, catalog, ledger, upstream, log });
+		const base = { ompSessionId: "omp-9", harnessId: "", toolName: "read", input: { path: "x.ts" }, content: BIG, query: "q" };
+		// digest.enabled is off, so the tool_result path declines...
+		expect(await d.digest(base)).toMatchObject({ digested: false, reason: expect.stringContaining("disabled") });
+		// ...but compaction only needs compaction.digestToolResults, plus a tier at or above fromTier.
+		expect(await d.digest({ ...base, tier: "simple", source: "compaction" })).toMatchObject({ digested: false, reason: expect.stringContaining("below digest.fromTier") });
+		const r = await d.digest({ ...base, tier: "hard", source: "compaction" });
+		expect(r.digested).toBe(true);
+		expect(calls).toHaveLength(1);
+		expect(ledger.recentEntries(5).find((e) => e.requestedModel === "digest")?.reasons[0]).toContain("digest (compaction)");
+		cfg.compaction.digestToolResults = false;
+		expect(await d.digest({ ...base, tier: "hard", source: "compaction" })).toMatchObject({ digested: false });
+		db.close();
+	});
+
+	test("a later call of the same tool with the same primary argument marks the digest wasted", async () => {
+		const cfg = cfgWith();
+		const db = openDb(":memory:");
+		const ledger = createLedger(db, cfg);
+		seedSession(ledger, "hard");
+		const dg = createDigester({ cfg, catalog, ledger, upstream: fakeUpstream(() => "Condensed.").upstream, log });
+		const r = await dg.digest({ ompSessionId: "omp-1", harnessId: "", toolName: "read", input: { path: "src/a.ts", offset: 1 }, content: BIG, query: "" });
+		expect(r.digested).toBe(true);
+		const row = () => ledger.recentEntries(10).find((e) => e.requestedModel === "digest")!;
+		expect(row().wasted).toBe(false);
+		// A different file, a different tool, another session: no match.
+		expect(dg.noteToolCalls("omp-1", [{ name: "read", argsJson: '{"path":"src/b.ts"}' }, { name: "grep", argsJson: '{"pattern":"src/a.ts"}' }])).toBe(0);
+		expect(dg.noteToolCalls("omp-2", [{ name: "read", argsJson: '{"path":"src/a.ts"}' }])).toBe(0);
+		expect(row().wasted).toBe(false);
+		// The same read again (case-insensitive tool name, any other args): the agent wanted the full output.
+		expect(dg.noteToolCalls("omp-1", [{ name: "Read", argsJson: '{"path":"src/a.ts","limit":50}' }])).toBe(1);
+		expect(row().wasted).toBe(true);
+		// Marked once; a third read does not count again.
+		expect(dg.noteToolCalls("omp-1", [{ name: "read", argsJson: '{"path":"src/a.ts"}' }])).toBe(0);
+		db.close();
+	});
+
 	test("a pinned digest model is used as-is", async () => {
 		const pinned = MODELS.find((m) => m.price.prompt > 0)!.slug;
 		const cfg = cfgWith({ model: pinned });

package/test/failover.test.ts

@@ -46,7 +46,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
 			data: { axis: "intelligence", minQuality: 0 },
 			chat: { axis: "intelligence", minQuality: 0 },
 		},
-		filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, feedbackWeight: 0, minTrustSamples: 5, trustScopedByHarness: false, trustWindowDays: 0, contextHeadroom: 1.2, latencyWeight: 0, latencyReferenceMs: 5000, latencyReferenceTokensPerSec: 30, cacheReliabilityMinSamples: 10, latencyMinSamples: 20, escalationCostWeight: 0 },
+		filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, feedbackWeight: 0, feedbackByTask: false, minTrustSamples: 5, trustScopedByHarness: false, trustWindowDays: 0, contextHeadroom: 1.2, latencyWeight: 0, latencyReferenceMs: 5000, latencyReferenceTokensPerSec: 30, cacheReliabilityMinSamples: 10, latencyMinSamples: 20, escalationCostWeight: 0 },
 		classifier: {
 			ambiguityThreshold: 0,
 			model: "test/adjudicator", learnedModelPath: "",
@@ -74,12 +74,12 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
 		exploration: { enabled: false, rates: {}, stickyPolicy: "never", holdTurns: { enabled: false, values: [2, 3, 4] } },
 		cache: { injectBreakpoints: true, maxBreakpoints: 4, minPromptTokens: 1024, milestoneTokens: 20_000 },
 		context: { enabled: false, baseUrl: "", token: "", defaultScope: "", timeoutMs: 3_000, maxStalenessMs: 900_000, maxBlockChars: 24_000, memoryLimit: 8, docsLimit: 2, sessionLimit: 6, briefChars: 0, recordTurns: false, maxQueue: 64 },
-		compaction: { enabled: false, budgetTokens: 40_000, floorRatio: 1, fitToWindow: true, protectRecentTurns: 4, maxToolResultBytes: 4_096, keepHeadBytes: 512, keepTailBytes: 512, elideSupersededReads: true, collapseDuplicateResults: true, replanGrowthRatio: 1 },
+		compaction: { enabled: false, budgetTokens: 40_000, floorRatio: 1, fitToWindow: true, protectRecentTurns: 4, maxToolResultBytes: 4_096, keepHeadBytes: 512, keepTailBytes: 512, elideSupersededReads: true, collapseDuplicateResults: true, replanGrowthRatio: 1, digestToolResults: false, digestMaxPerTurn: 2 },
 		budget: { onExceeded: "downgrade" },
-		report: { baselines: [] },
+		report: { baselines: [], dailySummary: false },
 		digest: { enabled: false, minBytes: 12_000, maxBytes: 400_000, tools: ["read"], fromTier: "moderate", tier: "simple", model: "", maxOutputTokens: 700, maxCostUsd: 0.02, timeoutMs: 25_000 },
 		profiles: [],
-		ledger: { path: ":memory:", blendWindowDays: 7, blendMinSamples: 20, fallbackBlend: { inputPerMtok: 1, outputPerMtok: 4 }, conversationTtlMs: 86_400_000 },
+		ledger: { path: ":memory:", blendWindowDays: 7, blendMinSamples: 20, fallbackBlend: { inputPerMtok: 1, outputPerMtok: 4 }, conversationTtlMs: 86_400_000 , retentionDays: 0,},
 		adaptiveTierFloors: true,
 		adaptivePriceCeilings: false,
 		logLevel: "silent",

package/test/hot-reload.test.ts

@@ -3,7 +3,7 @@ import { mkdirSync, rmSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { DEFAULT_CONFIG } from "../src/config/defaults.ts";
 import type { RouterConfig } from "../src/config/types.ts";
-import { readValidatedConfig, watchConfig, type ConfigWatcher } from "../src/config/hot-reload.ts";
+import { PINNED_CONFIG_PATHS, readValidatedConfig, watchConfig, type ConfigWatcher } from "../src/config/hot-reload.ts";
 
 const DIR = join(import.meta.dir, ".tmp-hot-reload");
 const CFG = join(DIR, "config.yml");
@@ -129,3 +129,39 @@ describe("watchConfig", () => {
 		expect(live.filters.latencyWeight).not.toBe(9.9);
 	});
 });
+
+describe("watchConfig pins by path", () => {
+	const CFG2 = join(DIR, "config-paths.yml");
+	const live: RouterConfig = structuredClone(DEFAULT_CONFIG);
+	let watcher: ConfigWatcher | null = null;
+
+	beforeAll(() => {
+		writeFileSync(CFG2, "");
+		const pinned = structuredClone(DEFAULT_CONFIG);
+		pinned.ollama.apiKey = "pinned-key";
+		watcher = watchConfig(CFG2, live, pinned, PINNED_CONFIG_PATHS);
+	});
+	afterAll(() => watcher?.close());
+
+	test("a pinned key inside a block keeps its construction value while its siblings hot-reload", async () => {
+		writeFileSync(CFG2, yamlOf({ ollama: { apiKey: "from-file", costBias: 0.25, biasUntilUsage: 0.5 }, server: { port: 1, subagentProfile: "auto" }, ledger: { retentionDays: 30 } }));
+		await settle();
+		expect(live.ollama.costBias).toBe(0.25);
+		expect(live.ollama.biasUntilUsage).toBe(0.5);
+		expect(live.ollama.apiKey).toBe("pinned-key");
+		expect(live.server.port).toBe(DEFAULT_CONFIG.server.port);
+		expect(live.server.subagentProfile).toBe("auto");
+		expect(live.ledger.retentionDays).toBe(30);
+		expect(live.ledger.path).toBe(DEFAULT_CONFIG.ledger.path);
+	});
+
+	test("the pinned path list names only real config keys", () => {
+		const root = DEFAULT_CONFIG as unknown as Record<string, Record<string, unknown>>;
+		for (const p of PINNED_CONFIG_PATHS) {
+			const [block = "", key] = p.split(".");
+			expect(block in root).toBe(true);
+			// Optional keys (server.apiKey, server.harnessId) are absent from the defaults but real.
+			if (key !== undefined && !["apiKey", "harnessId"].includes(key)) expect(key in root[block]!).toBe(true);
+		}
+	});
+});

package/test/learned.test.ts

@@ -1,6 +1,6 @@
 import { describe, expect, test } from "bun:test";
 
-import { auc, FEATURE_NAMES, learnedVector, predictRisk, trainLogistic, type LearnedModel } from "../src/router/learned.ts";
+import { auc, FEATURE_NAMES, LEARNED_MODEL_VERSION, learnedRiskName, learnedVector, loadLearnedModel, predictRisk, resetLearnedModels, trainLogistic, type LearnedModel } from "../src/router/learned.ts";
 
 /**
  * The learned escalation-risk model: a deterministic logistic regression that
@@ -59,3 +59,23 @@ describe("trainLogistic", () => {
 		expect(() => trainLogistic([], [])).toThrow();
 	});
 });
+
+describe("learned label", () => {
+	test("a feedback-labelled model loads with its label and names its risk p(bad)", async () => {
+		const d = FEATURE_NAMES.length;
+		const base: LearnedModel = { version: LEARNED_MODEL_VERSION, trainedAtMs: 0, rows: 100, positives: 10, names: [...FEATURE_NAMES], means: new Array(d).fill(0), stds: new Array(d).fill(1), weights: new Array(d).fill(0), bias: 0, auc: 0.5 };
+		expect(learnedRiskName(base)).toBe("escalate");
+		expect(learnedRiskName({ ...base, label: "feedback" })).toBe("bad");
+		const path = `${import.meta.dir}/../.tmp-learned-feedback.json`;
+		await Bun.write(path, JSON.stringify({ ...base, label: "feedback" }));
+		try {
+			resetLearnedModels();
+			const loaded = await loadLearnedModel(path);
+			expect(loaded?.label).toBe("feedback");
+			expect(learnedRiskName(loaded!)).toBe("bad");
+		} finally {
+			resetLearnedModels();
+			await Bun.file(path).delete();
+		}
+	});
+});

package/test/migrations.test.ts

@@ -0,0 +1,84 @@
+import { describe, expect, test } from "bun:test";
+import { copyFileSync, mkdtempSync, readdirSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { DEFAULT_CONFIG } from "../src/config/defaults.ts";
+import { createFeedbackStore } from "../src/cost/feedback.ts";
+import { createLedger } from "../src/cost/ledger.ts";
+import { buildUsageReport } from "../src/cost/report.ts";
+import { buildDailySummary, createKv } from "../src/cost/summary.ts";
+import { createConversationStore } from "../src/router/state.ts";
+import { openDb } from "../src/util/sqlite.ts";
+
+/**
+ * Every ledger a past release wrote must open under the current bootstrap:
+ * the version lands on the current number, every prepared statement the
+ * router uses compiles against the migrated schema, the carried rows survive
+ * with their backfills, and the aggregates run. The fixtures come from
+ * tools/gen-migration-fixtures.ts (each tag's own bootstrap plus one row per
+ * table), so a column added without a guard, or a statement that assumes a
+ * column older files lack, fails here rather than on a user's install.
+ */
+
+const FIXTURES = join(import.meta.dir, "fixtures", "migrations");
+const files = readdirSync(FIXTURES).filter((f) => /^router-v\d+\.db$/.test(f)).sort((a, b) => Number(/\d+/.exec(a)![0]) - Number(/\d+/.exec(b)![0]));
+const CURRENT_VERSION = 17;
+
+describe("schema migrations from every shipped version", () => {
+	test("fixtures exist for the versions that shipped", () => {
+		expect(files.map((f) => Number(/\d+/.exec(f)![0]))).toEqual([4, 5, 10, 12, 13, 14, 16]);
+	});
+
+	for (const file of files) {
+		const from = Number(/\d+/.exec(file)![0]);
+		test(`v${from} → v${CURRENT_VERSION}: opens, migrates, keeps its rows, and every consumer runs`, () => {
+			const dir = mkdtempSync(join(tmpdir(), "amr-migrate-"));
+			const path = join(dir, "router.db");
+			copyFileSync(join(FIXTURES, file), path);
+			const cfg = structuredClone(DEFAULT_CONFIG);
+			cfg.ledger.path = path;
+			const db = openDb(path);
+			try {
+				expect((db.query("PRAGMA user_version").get() as { user_version: number }).user_version).toBe(CURRENT_VERSION);
+				// Every column the current code writes exists after migration.
+				const ledgerCols = new Set((db.query("PRAGMA table_info(ledger)").all() as { name: string }[]).map((c) => c.name));
+				for (const c of ["harness_id", "error_kind", "omp_session_id", "features", "explored_from", "hold_arm", "prompt_tokens_saved"]) expect(ledgerCols.has(c)).toBe(true);
+				const convCols = new Set((db.query("PRAGMA table_info(conversations)").all() as { name: string }[]).map((c) => c.name));
+				for (const c of ["context_version", "compaction_plan", "compaction_plan_tokens", "upgrade_deferred_tier"]) expect(convCols.has(c)).toBe(true);
+				// The fixture's ledger row survived the ALTERs with its values.
+				const row = db.query("SELECT id, error, slug FROM ledger").get() as { id: string; error: string | null; slug: string } | null;
+				expect(row).toEqual({ id: "fixture-id", error: "upstream_error: 502", slug: "fixture-slug" });
+				// Every prepared statement compiles and every consumer runs on the migrated file.
+				const ledger = createLedger(db, cfg);
+				const conversations = createConversationStore(db);
+				createFeedbackStore(db);
+				createKv(db);
+				expect(ledger.recentEntries(5)).toHaveLength(1);
+				expect(ledger.trust("fixture-slug")).not.toBeNull();
+				expect(ledger.softFailureSpikes?.()).toEqual([]);
+				expect(ledger.latestForSession?.("nope")).toBeNull();
+				expect(conversations.load("fixture-key").key).toBe("fixture-key");
+				expect(buildUsageReport(db, { windowDays: 3650 }).totals.dispatches).toBe(1);
+				expect(buildDailySummary(db, {}).current.dispatches).toBe(0);
+				expect(ledger.prune?.(0)).toBe(0);
+			} finally {
+				db.close();
+				try {
+					rmSync(dir, { recursive: true, force: true });
+				} catch {
+					// Windows keeps the file locked until the statements are collected; the temp dir is disposable.
+				}
+			}
+		});
+	}
+
+	test("a fresh database lands on the same version as a migrated one", () => {
+		const db = openDb(":memory:");
+		try {
+			expect((db.query("PRAGMA user_version").get() as { user_version: number }).user_version).toBe(CURRENT_VERSION);
+		} finally {
+			db.close();
+		}
+	});
+});

package/test/report-hub.test.ts

@@ -80,7 +80,7 @@ function report(over: Partial<UsageReport> = {}): UsageReport {
 			subagentSpendUsd: 0,
 			digests: 0,
 			digestSpendUsd: 0,
-			digestInputTokens: 0,
+			digestInputTokens: 0, digestReruns: 0, forecastSamples: 0, forecastMeanError: 0, forecastOverShare: 0,
 		},
 		providers: [row("openrouter", 2), row("ollama", 1)],
 		models: [

package/test/report-logic.test.ts

@@ -1,6 +1,6 @@
 import { describe, expect, test } from "bun:test";
 
-import { fetchReport, parseReportArgs, renderStatus, type HealthSnapshot } from "../omp-extension/report-logic.ts";
+import { fetchReport, parseReportArgs, renderSoftFailureSpikes, renderStatus, type HealthSnapshot } from "../omp-extension/report-logic.ts";
 
 describe("parseReportArgs", () => {
 	test("defaults to 7 days scoped to the harness", () => {
@@ -71,8 +71,11 @@ describe("renderStatus", () => {
 				costBias: { configured: 0.1, effective: 0.1, biasUntilUsage: 0.9 },
 			},
 			catalog: { models: 240, ageMs: 5 * 60_000, keyScoped: true, shrink: { fromModels: 300, toModels: 120, atMs: now } },
+			softFailures: { recentMs: 3_600_000, baselineDays: 7, spikes: [{ slug: "z-ai/glm-5.3", recentDispatches: 12, recentFailures: 5, recentRate: 5 / 12, baselineDispatches: 340, baselineRate: 0.08 }] },
 		};
 		const text = renderStatus("http://127.0.0.1:8788", h, now);
+		expect(text).toContain("soft failures SPIKING (1):");
+		expect(text).toContain("  z-ai/glm-5.3: 42% of 12 failed in the last 1h (7d baseline 8% of 340)");
 		expect(text).toContain("configured (omp)");
 		expect(text).toContain("240 models");
 		expect(text).toContain("refreshed 5m ago");
@@ -86,6 +89,13 @@ describe("renderStatus", () => {
 		expect(text).toContain("recording turns");
 	});
 
+	test("reports a quiet hour and omits the line for routers that predate the check", () => {
+		expect(renderStatus("http://h", { status: "ok", softFailures: { spikes: [] } })).toContain("soft failures: no model spiking in the last hour");
+		expect(renderStatus("http://h", { status: "ok" })).not.toContain("soft failures");
+		expect(renderSoftFailureSpikes(null)).toEqual([]);
+		expect(renderSoftFailureSpikes([{ slug: "a/b", recentRate: 0.5, recentDispatches: 6 }], 30 * 60_000, 7)).toEqual(["a/b: 50% of 6 failed in the last 30m (7d baseline 0% of 0)"]);
+	});
+
 	test("degrades cleanly when sections are absent", () => {
 		const text = renderStatus("http://h", { status: "ok", apiKeyConfigured: false });
 		expect(text).toContain("key MISSING");

package/test/report.test.ts

@@ -229,7 +229,7 @@ describe("buildUsageReport", () => {
 			subagentSpendUsd: 0,
 			digests: 0,
 			digestSpendUsd: 0,
-			digestInputTokens: 0,
+			digestInputTokens: 0, digestReruns: 0, forecastSamples: 0, forecastMeanError: 0, forecastOverShare: 0,
 		});
 		expect(r.providers).toEqual([]);
 		expect(r.models).toEqual([]);
@@ -291,3 +291,31 @@ describe("renderUsageReport", () => {
 		db.close();
 	});
 });
+
+describe("digest re-runs and forecast accuracy", () => {
+	test("wasted digest rows count as re-runs; forecast error is judged on clean kept rows only", () => {
+		const { db, ledger } = seeded();
+		try {
+			ledger.record(entry({ requestedModel: "digest", conversationKey: "d1", reportedUsd: 0.001, predictedUsd: 0.001 }));
+			ledger.record(entry({ requestedModel: "digest", conversationKey: "d2", reportedUsd: 0.001, predictedUsd: 0.001, wasted: true }));
+			// Two clean turns: one predicted double, one predicted half.
+			ledger.record(entry({ predictedUsd: 0.02, reportedUsd: 0.01 }));
+			ledger.record(entry({ predictedUsd: 0.005, reportedUsd: 0.01 }));
+			// Excluded from the forecast judgement: wasted, errored, no reported cost.
+			ledger.record(entry({ predictedUsd: 1, reportedUsd: 0.01, wasted: true }));
+			ledger.record(entry({ predictedUsd: 1, reportedUsd: 0.01, error: "upstream_error: 500" }));
+			ledger.record(entry({ predictedUsd: 1, reportedUsd: null }));
+			const t = buildUsageReport(db, { windowDays: 1, nowMs: NOW }).totals;
+			expect(t.digests).toBe(2);
+			expect(t.digestReruns).toBe(1);
+			expect(t.forecastSamples).toBe(2);
+			expect(t.forecastMeanError).toBeCloseTo((1 + 0.5) / 2, 6);
+			expect(t.forecastOverShare).toBeCloseTo(0.5, 6);
+			const text = renderUsageReport(buildUsageReport(db, { windowDays: 1, nowMs: NOW }));
+			expect(text).toContain("re-run rate 50% (1 fetched again in full)");
+			expect(text).toContain("forecast: mean error 75% of reported cost over 2 turns · 50% over-predicted");
+		} finally {
+			db.close();
+		}
+	});
+});

package/test/select.test.ts

@@ -637,7 +637,7 @@ describe("context compaction", () => {
 			keepHeadBytes: 20,
 			keepTailBytes: 20,
 			elideSupersededReads: true,
-			collapseDuplicateResults: true,
+			collapseDuplicateResults: true, digestToolResults: false, digestMaxPerTurn: 2,
 		},
 	};
 
@@ -954,7 +954,7 @@ describe("compaction.replanGrowthRatio (review 2026-09-05 §7)", () => {
 				keepHeadBytes: 20,
 				keepTailBytes: 20,
 				elideSupersededReads: false,
-				collapseDuplicateResults: false,
+				collapseDuplicateResults: false, digestToolResults: false, digestMaxPerTurn: 2,
 			},
 		};
 	}
@@ -1052,6 +1052,30 @@ describe("hysteresis.confirmUpgradesBelowConfidence", () => {
 	});
 });
 
+describe("recorded forecast is the expected price, not the cold worst case", () => {
+	const warmSlug = "x-ai/grok-4.6";
+	test("a warm stay prices the previous prompt as cache reads; coldUsd keeps the cold figure", () => {
+		const cfg: RouterConfig = { ...BASE, hysteresis: { ...BASE.hysteresis, switchMargin: 1e6 } };
+		const d = run({
+			tier: "hard",
+			promptTokens: 80_000,
+			cfg,
+			st: state({ currentSlug: warmSlug, currentTier: "hard", cacheWarmSlug: warmSlug, cacheWarmAtMs: Date.now(), lastPromptTokens: 60_000 }),
+		});
+		expect(d.slug).toBe(warmSlug);
+		// 60k of the 80k prompt is the cached prefix; no reliability sample ⇒ assumed reliable.
+		expect(d.forecast.assumedCacheHitRate).toBeCloseTo(0.75, 6);
+		expect(d.forecast.expectedUsd).toBeLessThan(d.forecast.coldUsd);
+		expect(d.forecast.breakdown.cacheRead).toBeGreaterThan(0);
+	});
+
+	test("a cold turn records the cold price", () => {
+		const d = run({ tier: "hard", promptTokens: 80_000 });
+		expect(d.forecast.assumedCacheHitRate).toBe(0);
+		expect(d.forecast.expectedUsd).toBeLessThanOrEqual(d.forecast.coldUsd);
+	});
+});
+
 describe("cache reliability in the stay/switch comparison", () => {
 	const warmSlug = "x-ai/grok-4.6";
 	function ledgerWithReliability(rate: number | null, samples = 50): Ledger {

package/test/summary.test.ts

@@ -0,0 +1,171 @@
+import { describe, expect, test } from "bun:test";
+
+import { DEFAULT_CONFIG } from "../src/config/defaults.ts";
+import { createLedger } from "../src/cost/ledger.ts";
+import { buildDailySummary, countTierChanges, createKv, DAILY_SUMMARY_INTERVAL_MS, markSummaryShown, renderDailySummary, summaryDue, summaryHasNews } from "../src/cost/summary.ts";
+import type { LedgerEntry } from "../src/cost/types.ts";
+import { openDb } from "../src/util/sqlite.ts";
+import { fetchSummary } from "../omp-extension/report-logic.ts";
+
+/**
+ * The daily summary: the last 24h against the day before, top models, tier
+ * moves, the once-a-day gate in router_kv, and the text the transcript gets.
+ */
+
+const HOUR = 3_600_000;
+const NOW = Date.UTC(2026, 8, 7, 9, 0, 0);
+
+function entry(over: Partial<LedgerEntry>): LedgerEntry {
+	return {
+		id: crypto.randomUUID(),
+		createdAtMs: NOW - HOUR,
+		conversationKey: "k",
+		sessionId: "s",
+		turn: 1,
+		requestedModel: "auto",
+		harnessId: "",
+		ompSessionId: "",
+		slug: "vendor/model",
+		servedSlug: "vendor/model",
+		tier: "simple",
+		classificationSource: "heuristic",
+		reasons: [],
+		features: null,
+		score: null,
+		confidence: null,
+		task: null,
+		classifierReasons: null,
+		exploredFrom: null,
+		holdArm: null,
+		predictedUsd: 0.01,
+		reportedUsd: 0.01,
+		usage: { promptTokens: 1000, cachedTokens: 500, cacheWriteTokens: 0, completionTokens: 100, reasoningTokens: 0, images: 0 },
+		attempt: 0,
+		escalationSignal: null,
+		latencyMs: 1_100,
+		ttftMs: 100,
+		finishReason: "stop",
+		wasted: false,
+		upstreamGenerationId: null,
+		error: null,
+		promptTokensSaved: null,
+		...over,
+	} as LedgerEntry;
+}
+
+function seeded() {
+	const cfg = structuredClone(DEFAULT_CONFIG);
+	cfg.ledger.path = ":memory:";
+	const db = openDb(":memory:");
+	const ledger = createLedger(db, cfg);
+	return { db, ledger };
+}
+
+describe("buildDailySummary", () => {
+	test("compares the last 24h with the day before and names the top models and tier moves", () => {
+		const { db, ledger } = seeded();
+		try {
+			// Yesterday: one conversation climbing simple → hard, then dropping to moderate; a second model.
+			ledger.record(entry({ createdAtMs: NOW - 5 * HOUR, tier: "simple", turn: 1 }));
+			ledger.record(entry({ createdAtMs: NOW - 4 * HOUR, tier: "hard", turn: 2, slug: "vendor/big", servedSlug: "vendor/big", reportedUsd: 0.5 }));
+			ledger.record(entry({ createdAtMs: NOW - 3 * HOUR, tier: "moderate", turn: 3, escalationSignal: "empty_completion", wasted: true }));
+			ledger.record(entry({ createdAtMs: NOW - 3 * HOUR + 1, tier: "moderate", turn: 3, attempt: 1 }));
+			ledger.record(entry({ createdAtMs: NOW - 2 * HOUR, requestedModel: "digest", slug: "vendor/tiny", servedSlug: "vendor/tiny", conversationKey: "d", reportedUsd: 0.001 }));
+			// The day before: pricier.
+			for (let i = 0; i < 4; i++) ledger.record(entry({ createdAtMs: NOW - 30 * HOUR - i, conversationKey: "old", reportedUsd: 0.4 }));
+			const s = buildDailySummary(db, { nowMs: NOW, baselines: [{ slug: "anthropic/claude-opus-5", prompt: 15e-6, completion: 75e-6, cacheRead: 1.5e-6 }] });
+			expect(s.current.dispatches).toBe(5);
+			expect(s.current.spendUsd).toBeCloseTo(0.531, 6);
+			expect(s.current.escalations).toBe(1);
+			expect(s.current.digests).toBe(1);
+			expect(s.current.modelSwitches).toBe(2);
+			expect(s.previous.dispatches).toBe(4);
+			expect(s.previous.spendUsd).toBeCloseTo(1.6, 6);
+			expect(s.topModels[0]).toMatchObject({ slug: "vendor/big", dispatches: 1 });
+			expect(s.topModels[0]!.share).toBeCloseTo(0.5 / 0.531, 3);
+			expect(s.tierChanges).toEqual({ up: 1, down: 1 });
+			expect(s.baseline?.slug).toBe("anthropic/claude-opus-5");
+			expect(summaryHasNews(s)).toBe(true);
+
+			const text = renderDailySummary({ ...s, spikes: [{ slug: "vendor/big", recentDispatches: 8, recentFailures: 4, recentRate: 0.5, baselineDispatches: 90, baselineFailures: 3, baselineRate: 3 / 90 }], ollama: { plan: "pro", usedUsd: 25.2, creditsUsd: 60, runwayDays: 23.4 } });
+			expect(text).toContain("auto-model-router daily summary — last 24h (all harnesses)");
+			expect(text).toContain("spend $0.531 (prev 24h $1.60, −67%) · 5 turns · 2 conversations · $0.106/turn");
+			expect(text).toContain("cache hit 50% · 1 escalations · 0 errors · 2 model switches (1 tier up, 1 down)");
+			expect(text).toContain("top models: vendor/big $0.500 (94%, 1 turns)");
+			// Tiny seeded turns cost more than Opus would have at list price: the honest branch renders.
+			expect(text).toContain("cost 574% MORE than anthropic/claude-opus-5 ($0.079 at list)");
+			expect(renderDailySummary({ ...s, baseline: { slug: "anthropic/claude-opus-5", usd: 2.5, savedShare: 0.7876 } })).toContain("saved 79% vs anthropic/claude-opus-5 ($2.50 at list)");
+			expect(text).toContain("1 digests for $0.001");
+			expect(text).toContain("soft failures SPIKING (1):\n  vendor/big: 50% of 8 failed in the last 1h (7d baseline 3% of 90)");
+			expect(text).toContain("ollama: pro plan $25.20 of $60 · ~23 days of credits left");
+		} finally {
+			db.close();
+		}
+	});
+
+	test("an empty day renders as such and carries no news unless a model is spiking", () => {
+		const { db } = seeded();
+		try {
+			const s = buildDailySummary(db, { nowMs: NOW });
+			expect(summaryHasNews(s)).toBe(false);
+			const text = renderDailySummary(s);
+			expect(text).toContain("no routed turns in the last 24h");
+			expect(text).toContain("soft failures: no model spiking in the last hour");
+			expect(text).not.toContain("ollama:");
+			expect(summaryHasNews({ ...s, spikes: [{ slug: "a/b", recentDispatches: 5, recentFailures: 3, recentRate: 0.6, baselineDispatches: 0, baselineFailures: 0, baselineRate: 0 }] })).toBe(true);
+			expect(countTierChanges(db, 0, "")).toEqual({ up: 0, down: 0 });
+		} finally {
+			db.close();
+		}
+	});
+
+	test("harness scope narrows both windows", () => {
+		const { db, ledger } = seeded();
+		try {
+			ledger.record(entry({ harnessId: "a" }));
+			ledger.record(entry({ harnessId: "b", reportedUsd: 5 }));
+			expect(buildDailySummary(db, { nowMs: NOW, harnessId: "a" }).current).toMatchObject({ dispatches: 1, spendUsd: 0.01 });
+			expect(buildDailySummary(db, { nowMs: NOW }).current.dispatches).toBe(2);
+		} finally {
+			db.close();
+		}
+	});
+});
+
+describe("once-a-day gate", () => {
+	test("router_kv marks the summary shown per harness for 20h, surviving a reopen", () => {
+		const db = openDb(":memory:");
+		try {
+			const kv = createKv(db);
+			expect(summaryDue(kv, "", NOW)).toBe(true);
+			markSummaryShown(kv, "", NOW);
+			expect(summaryDue(kv, "", NOW + HOUR)).toBe(false);
+			expect(summaryDue(kv, "other", NOW + HOUR)).toBe(true);
+			expect(summaryDue(kv, "", NOW + DAILY_SUMMARY_INTERVAL_MS)).toBe(true);
+			// Same table, fresh handle: the marker is durable.
+			expect(summaryDue(createKv(db), "", NOW + HOUR)).toBe(false);
+			kv.set("k", "v1");
+			kv.set("k", "v2");
+			expect(kv.get("k")).toBe("v2");
+			expect(kv.get("missing")).toBeNull();
+		} finally {
+			db.close();
+		}
+	});
+});
+
+describe("fetchSummary", () => {
+	test("passes harness and auto through and surfaces the router's verdict", async () => {
+		const calls: string[] = [];
+		const fetchImpl = async (url: string): Promise<Response> => {
+			calls.push(url);
+			return new Response(JSON.stringify({ due: false, reason: "posted in the last 20h", summary: null }), { status: 200 });
+		};
+		const r = await fetchSummary("http://h", "omp", true, {}, fetchImpl);
+		expect(r).toEqual({ due: false, reason: "posted in the last 20h", summary: null });
+		expect(calls).toEqual(["http://h/v1/router/summary?harness=omp&auto=1"]);
+		await fetchSummary("http://h", "", false, {}, fetchImpl);
+		expect(calls[1]).toBe("http://h/v1/router/summary");
+		await expect(fetchSummary("http://h", "", false, {}, async () => new Response("no", { status: 503 }))).rejects.toThrow("router returned 503");
+	});
+});

package/test/support/preload.ts

@@ -0,0 +1,19 @@
+/**
+ * Test preload (bunfig.toml `[test] preload`): isolate every test from the
+ * developer's live router home BEFORE any module loads.
+ *
+ * Five test files build their config with `loadConfig({})`, which layers
+ * `$AUTO_MODEL_ROUTER_HOME/config.yml` over the defaults. Run alone they read
+ * the real config and fail on whatever the developer has tuned; run in the
+ * full suite they happened to pass because an earlier file had already
+ * pointed the home at a temp dir. Doing it here makes both cases the same.
+ * Tests that want a specific home (config.test.ts, embed-lifecycle) still
+ * set their own; this only supplies the default.
+ */
+import { mkdtempSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+if (process.env.AUTO_MODEL_ROUTER_HOME === undefined) {
+	process.env.AUTO_MODEL_ROUTER_HOME = mkdtempSync(join(tmpdir(), "amr-test-home-"));
+}

package/test/tokens.test.ts

@@ -237,3 +237,71 @@ describe("ledger.escalationCost", () => {
 	});
 });
 
+
+describe("ledger.softFailureSpikes", () => {
+	test("flags a model whose last-hour failure rate is a spike against its own 7-day baseline", () => {
+		const db = openDb(":memory:");
+		try {
+			const ledger = createLedger(db, cfg);
+			const now = 1_800_000_000_000;
+			const H = 3_600_000;
+			// Baseline: 100 dispatches over the prior week at 5% soft failures.
+			for (let i = 0; i < 100; i++) {
+				ledger.record(entry({ createdAtMs: now - 2 * H - i * 60 * 60_000, escalationSignal: i % 20 === 0 ? "empty_completion" : null, wasted: i % 20 === 0 }));
+			}
+			// Last hour: 10 dispatches, 4 soft failures (40%): a spike.
+			for (let i = 0; i < 10; i++) {
+				ledger.record(entry({ createdAtMs: now - 5 * 60_000 - i * 60_000, escalationSignal: i < 4 ? "repeat_tool_call" : null, wasted: i < 4 }));
+			}
+			// A second model with plenty of failures but a matching baseline is not spiking.
+			for (let i = 0; i < 100; i++) {
+				ledger.record(entry({ slug: "x/steady", servedSlug: "x/steady", createdAtMs: now - 2 * H - i * 60 * 60_000, error: i % 2 === 0 ? "upstream_error: 502" : null }));
+			}
+			for (let i = 0; i < 10; i++) {
+				ledger.record(entry({ slug: "x/steady", servedSlug: "x/steady", createdAtMs: now - 5 * 60_000 - i * 60_000, error: i % 2 === 0 ? "upstream_error: 502" : null }));
+			}
+			// Aborted and quota errors are not attributable; digest rows are side calls.
+			for (let i = 0; i < 10; i++) {
+				ledger.record(entry({ slug: "x/aborted", servedSlug: "x/aborted", createdAtMs: now - 5 * 60_000 - i * 60_000, error: "aborted: client closed" }));
+				ledger.record(entry({ slug: "x/digest", servedSlug: "x/digest", requestedModel: "digest", createdAtMs: now - 5 * 60_000 - i * 60_000, error: "upstream_error: 500" }));
+			}
+			const spikes = ledger.softFailureSpikes?.(now) ?? [];
+			expect(spikes.map((s) => s.slug)).toEqual(["openai/gpt-5-mini"]);
+			const s = spikes[0]!;
+			expect(s.recentDispatches).toBe(10);
+			expect(s.recentFailures).toBe(4);
+			expect(s.recentRate).toBeCloseTo(0.4, 6);
+			expect(s.baselineDispatches).toBe(100);
+			expect(s.baselineFailures).toBe(5);
+			expect(s.baselineRate).toBeCloseTo(0.05, 6);
+			// Too few recent dispatches: nothing spikes, however high the rate.
+			expect(ledger.softFailureSpikes?.(now, 3 * 60_000)).toEqual([]);
+		} finally {
+			db.close();
+		}
+	});
+});
+
+describe("ledger.prune and markWasted", () => {
+	test("prune deletes rows past retention and 0 keeps everything; markWasted flips one row", () => {
+		const db = openDb(":memory:");
+		try {
+			const ledger = createLedger(db, cfg);
+			const now = 1_800_000_000_000;
+			const DAY = 86_400_000;
+			for (let i = 0; i < 5; i++) ledger.record(entry({ createdAtMs: now - i * 100 * DAY }));
+			expect(ledger.prune?.(0, now)).toBe(0);
+			expect(ledger.recentEntries(10)).toHaveLength(5);
+			expect(ledger.prune?.(365, now)).toBe(1); // only the 400-day-old row
+			expect(ledger.recentEntries(10)).toHaveLength(4);
+			expect(ledger.prune?.(150, now)).toBe(2); // 200 and 300 days old
+			const left = ledger.recentEntries(10);
+			expect(left).toHaveLength(2);
+			expect(left.every((e) => e.wasted === false)).toBe(true);
+			ledger.markWasted?.(left[0]!.id);
+			expect(ledger.recentEntries(10).find((e) => e.id === left[0]!.id)?.wasted).toBe(true);
+		} finally {
+			db.close();
+		}
+	});
+});