auto-model-router 0.1.3 → 0.2.0

package/test/toast-logic.test.ts

@@ -141,6 +141,38 @@ describe("selectToasts", () => {
 		];
 		expect(selectToasts(entries, "", "")).toHaveLength(2);
 	});
+
+	test("filters to the requesting omp session when one is set", () => {
+		// Two interactive omp sessions sharing one router's ledger: session-a's
+		// toast must not surface session-b's decisions.
+		const entries = [
+			dec({ id: "d3", slug: "mine", ompSessionId: "sess-a" }),
+			dec({ id: "d2", slug: "other", ompSessionId: "sess-b" }),
+			dec({ id: "d1", slug: "prior", ompSessionId: "sess-a" }),
+		];
+		const toasts = selectToasts(entries, "d1", "", "sess-a");
+		expect(toasts).toHaveLength(1);
+		expect(toasts[0]?.model).toBe("mine");
+	});
+
+	test("empty omp session id toasts every session", () => {
+		const entries = [
+			dec({ id: "d2", slug: "a", ompSessionId: "sess-a" }),
+			dec({ id: "d1", slug: "b", ompSessionId: "sess-b" }),
+		];
+		expect(selectToasts(entries, "", "", "")).toHaveLength(2);
+	});
+
+	test("harness and session filters compose", () => {
+		const entries = [
+			dec({ id: "d3", slug: "keep", harnessId: "h", ompSessionId: "sess-a" }),
+			dec({ id: "d2", slug: "wrong-session", harnessId: "h", ompSessionId: "sess-b" }),
+			dec({ id: "d1", slug: "wrong-harness", harnessId: "other", ompSessionId: "sess-a" }),
+		];
+		const toasts = selectToasts(entries, "", "h", "sess-a");
+		expect(toasts).toHaveLength(1);
+		expect(toasts[0]?.model).toBe("keep");
+	});
 });
 
 describe("toToastText", () => {

package/test/tokens.test.ts

@@ -18,11 +18,19 @@ function entry(over: Partial<LedgerEntry>): LedgerEntry {
 		turn: 1,
 		requestedModel: "auto",
 		harnessId: "",
+		ompSessionId: "",
 		slug: "openai/gpt-5-mini",
 		servedSlug: "openai/gpt-5-mini",
 		tier: "simple",
 		classificationSource: "heuristic",
 		reasons: [],
+		features: null,
+		score: null,
+		confidence: null,
+		task: null,
+		classifierReasons: null,
+		exploredFrom: null,
+		holdArm: null,
 		predictedUsd: 0.001,
 		reportedUsd: 0.001,
 		usage: EMPTY_USAGE,

package/test/trust-attribution.test.ts

@@ -16,11 +16,19 @@ function entry(over: Partial<LedgerEntry>): LedgerEntry {
 		turn: 1,
 		requestedModel: "auto",
 		harnessId: "",
+		ompSessionId: "",
 		slug: "vendor/model",
 		servedSlug: "vendor/model",
 		tier: "simple",
 		classificationSource: "heuristic",
 		reasons: [],
+		features: null,
+		score: null,
+		confidence: null,
+		task: null,
+		classifierReasons: null,
+		exploredFrom: null,
+		holdArm: null,
 		predictedUsd: 0.001,
 		reportedUsd: 0.001,
 		usage: EMPTY_USAGE,
@@ -163,11 +171,111 @@ describe("v4 migration", () => {
 		}
 	});
 
-	test("schema is at user_version 4", () => {
+	test("schema is at user_version 10", () => {
 		const db = openDb(":memory:");
 		try {
 			const row = db.query("PRAGMA user_version").get() as { user_version: number };
-			expect(row.user_version).toBe(4);
+			expect(row.user_version).toBe(10);
+		} finally {
+			db.close();
+		}
+	});
+
+	test("persists omp_session_id and returns it via recentEntries", () => {
+		const db = openDb(":memory:");
+		try {
+			const ledger = createLedger(db, cfg);
+			ledger.record(entry({ ompSessionId: "sess-a" }));
+			ledger.record(entry({ ompSessionId: "" }));
+			const got = ledger.recentEntries(10).map((e) => e.ompSessionId).sort();
+			expect(got).toEqual(["", "sess-a"]);
+		} finally {
+			db.close();
+		}
+	});
+});
+
+describe("v6 classifier instrumentation", () => {
+	const FEATURES = {
+		promptTokens: 1234,
+		isToolResultContinuation: true,
+		toolLoopDepth: 3,
+		complexityKeywords: ["race", "debug"],
+	};
+
+	test("round-trips the feature vector and classifier outputs", () => {
+		const db = openDb(":memory:");
+		try {
+			const ledger = createLedger(db, cfg);
+			ledger.record(
+				entry({
+					features: FEATURES,
+					score: 0.42,
+					confidence: 0.75,
+					task: "coding",
+					classifierReasons: ["-0.28 tool-result continuation"],
+				}),
+			);
+
+			const got = ledger.recentEntries(1)[0];
+			expect(got?.features).toEqual(FEATURES);
+			expect(got?.score).toBe(0.42);
+			expect(got?.confidence).toBe(0.75);
+			expect(got?.task).toBe("coding");
+			expect(got?.classifierReasons).toEqual(["-0.28 tool-result continuation"]);
+		} finally {
+			db.close();
+		}
+	});
+
+	test("an uninstrumented row reads back as null, not as invented data", () => {
+		const db = openDb(":memory:");
+		try {
+			const ledger = createLedger(db, cfg);
+			ledger.record(entry({}));
+			const got = ledger.recentEntries(1)[0];
+			expect(got?.features).toBeNull();
+			expect(got?.score).toBeNull();
+			expect(got?.confidence).toBeNull();
+			expect(got?.task).toBeNull();
+			expect(got?.classifierReasons).toBeNull();
+		} finally {
+			db.close();
+		}
+	});
+
+	test("records which tier exploration dropped from, and NULL otherwise", () => {
+		const db = openDb(":memory:");
+		try {
+			const ledger = createLedger(db, cfg);
+			ledger.record(entry({ tier: "simple", exploredFrom: "moderate" }));
+			ledger.record(entry({ tier: "moderate" }));
+
+			const got = ledger.recentEntries(10);
+			expect(got.map((e) => e.exploredFrom).sort()).toEqual(["moderate", null] as unknown as string[]);
+
+			// The counterfactual query this whole column exists to make possible:
+			// of the turns we deliberately under-routed, how many had to escalate?
+			const counted = db
+				.query("SELECT COUNT(*) n FROM ledger WHERE explored_from IS NOT NULL")
+				.get() as { n: number };
+			expect(counted.n).toBe(1);
+		} finally {
+			db.close();
+		}
+	});
+	test("features land in the column as queryable JSON", () => {
+		const db = openDb(":memory:");
+		try {
+			const ledger = createLedger(db, cfg);
+			ledger.record(entry({ features: FEATURES, score: 0.9, confidence: 0.1, task: "vision" }));
+			// SQLite json_extract proves the blob is real JSON, not a stringified object.
+			const row = db
+				.query("SELECT json_extract(features, '$.toolLoopDepth') AS depth, score, task FROM ledger")
+				.get() as { depth: number; score: number; task: string };
+			expect(row.depth).toBe(3);
+			expect(row.score).toBe(0.9);
+			expect(row.task).toBe("vision");
 		} finally {
 			db.close();
 		}

package/test/turn.test.ts

@@ -29,6 +29,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
 	return {
 		server: { host: "127.0.0.1", port: 8787 },
 		openrouter: { baseUrl: "https://openrouter.ai/api/v1", apiKey: "", title: "test", timeoutMs: 30_000, catalogTtlMs: 3_600_000, catalogRefreshMs: 0 },
+		benchmarks: { enabled: false, artificialAnalysisApiKey: "", benchlm: true, refreshMs: 86_400_000, timeoutMs: 30_000, useLocalScores: false },
 		tiers: {
 			trivial: { minQuality: 0, maxInputPerMtok: 0.3, qualityExponent: 0, pin: [] },
 			simple: { minQuality: 40, maxInputPerMtok: 1.5, qualityExponent: 0, pin: [] },
@@ -65,11 +66,13 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
 			...escalation,
 		},
 		hysteresis: { holdTurns: 2, holdTurnsAfterEscalation: 4, switchMargin: 1.5, cacheWarmTtlMs: 600_000, maxDowngradePerTurn: 1 },
+		exploration: { enabled: false, rates: {}, stickyPolicy: "never", holdTurns: { enabled: false, values: [2, 3, 4] } },
 		cache: { injectBreakpoints: true, maxBreakpoints: 4, minPromptTokens: 1024 },
 		budget: { onExceeded: "downgrade" },
 		profiles: [],
 		ledger: { path: ":memory:", blendWindowDays: 7, blendMinSamples: 20, fallbackBlend: { inputPerMtok: 1, outputPerMtok: 4 }, conversationTtlMs: 86_400_000 },
 		adaptiveTierFloors: true,
+		adaptivePriceCeilings: false,
 		logLevel: "silent",
 	};
 }
@@ -79,6 +82,7 @@ function mkReq(): NormRequest {
 		protocol: "openai-chat",
 		conversationKey: "conv-test",
 		harnessId: "",
+		ompSessionId: "",
 		requestedModel: "auto",
 		messages: [{ role: "user", text: "hi", images: 0, textBytes: 2, toolCalls: [] }],
 		tools: [],
@@ -140,6 +144,7 @@ function mkDecision(tier: Tier, slug: string, probe: Partial<ProbePlan> = {}): D
 		considered: [],
 		rejected: [],
 		reasons: ["test decision"],
+		explored: null,
 		budgetDowngraded: false,
 	};
 }
@@ -496,3 +501,44 @@ describe("runTurn", () => {
 		expect(errors).toHaveLength(0);
 	});
 });
+
+describe("exploration reaches the ledger", () => {
+	test("an explored turn records the tier it was dropped from", async () => {
+		const explored = { ...mkDecision("simple", "cheap/model"), explored: { from: "moderate" as Tier, to: "simple" as Tier } };
+		const { router } = mkRouter([explored]);
+		const { upstream } = mkUpstream([
+			{
+				kind: "chunks",
+				chunks: [startChunk("cheap/model"), textChunk("ok"), finishChunk("stop"), usageChunk({ promptTokens: 10, completionTokens: 2 }, 0.0001)],
+			},
+		]);
+		const { ledger, entries } = mkLedger();
+		const { store } = mkConversations();
+		const { sink } = mkSink();
+
+		await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog }, new AbortController().signal);
+
+		expect(entries).toHaveLength(1);
+		// The counterfactual pair: what the classifier wanted, and what actually ran.
+		expect(entries[0]?.exploredFrom).toBe("moderate");
+		expect(entries[0]?.tier).toBe("simple");
+	});
+
+	test("a normally routed turn leaves it null", async () => {
+		const { router } = mkRouter([mkDecision("simple", "cheap/model")]);
+		const { upstream } = mkUpstream([
+			{
+				kind: "chunks",
+				chunks: [startChunk("cheap/model"), textChunk("ok"), finishChunk("stop"), usageChunk({ promptTokens: 10, completionTokens: 2 }, 0.0001)],
+			},
+		]);
+		const { ledger, entries } = mkLedger();
+		const { store } = mkConversations();
+		const { sink } = mkSink();
+
+		await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog }, new AbortController().signal);
+
+		expect(entries).toHaveLength(1);
+		expect(entries[0]?.exploredFrom).toBeNull();
+	});
+});

package/test/wire-request.test.ts

@@ -58,6 +58,17 @@ describe("parseChatRequest normalization", () => {
 		expect(prefixed.requestedModel).toBe("auto");
 	});
 
+	test("reads harness and omp session ids from headers, trimmed", () => {
+		const headers = new Headers({ "x-omp-harness": " prod-a ", "x-omp-session": " sess-1 " });
+		const req = parseChatRequest(userBody("hi"), headers);
+		expect(req.harnessId).toBe("prod-a");
+		expect(req.ompSessionId).toBe("sess-1");
+	});
+
+	test("omp session id defaults to empty when the header is absent", () => {
+		expect(parseChatRequest(userBody("hi"), HEADERS).ompSessionId).toBe("");
+	});
+
 	test("tool schemas, names, and descriptions contribute to promptBytes", () => {
 		const parameters = { type: "object", properties: { path: { type: "string" } } };
 		const withTools = parseChatRequest(

package/tools/smoke.ts

@@ -160,6 +160,8 @@ cfg.ledger.path = join(home, "router.db");
 cfg.logLevel = "warn";
 // The adjudicator would call the mock and obscure which tier the heuristic chose.
 cfg.classifier.ambiguityThreshold = 0;
+// External benchmark feeds hit the real internet; keep the smoke test hermetic.
+cfg.benchmarks.enabled = false;
 
 const app = startServer(cfg);
 const base = `http://127.0.0.1:${app.server.port}`;

package/tools/sync-marketplace-version.ts

@@ -0,0 +1,60 @@
+#!/usr/bin/env bun
+/**
+ * Syncs the Git-marketplace catalog version with package.json.
+ *
+ * The marketplace (`.omp-plugin/marketplace.json`) serves the repo tip, and omp
+ * uses the catalog `version` for upgrade comparisons. If it lags the published
+ * package version, marketplace users are never offered new releases.
+ *
+ * This runs automatically as the npm `version` lifecycle script (`npm version`
+ * → this → commit), so every release keeps the catalog in step. It rewrites
+ * `metadata.version` and every `plugins[].version` to package.json's version and
+ * stages the file so it lands in the same version commit.
+ *
+ * Idempotent: a no-op when already in sync (nothing to stage). Also runnable by
+ * hand: `bun tools/sync-marketplace-version.ts`.
+ */
+
+import { spawnSync } from "node:child_process";
+import { readFileSync, writeFileSync } from "node:fs";
+import { join, resolve } from "node:path";
+
+const ROOT = resolve(import.meta.dir, "..");
+const PKG_PATH = join(ROOT, "package.json");
+const CATALOG_PATH = join(ROOT, ".omp-plugin", "marketplace.json");
+
+function main(): void {
+	const pkg = JSON.parse(readFileSync(PKG_PATH, "utf8")) as { version?: unknown };
+	const version = pkg.version;
+	if (typeof version !== "string" || version === "") {
+		throw new Error(`package.json has no usable version: ${JSON.stringify(version)}`);
+	}
+
+	const before = readFileSync(CATALOG_PATH, "utf8");
+	// Replace only the value of every `"version": "..."` field, preserving all
+	// other formatting (single-line arrays, spacing, key order) so the tool is a
+	// true no-op once in sync. marketplace.json carries exactly two: the
+	// top-level metadata.version and the single plugin's version.
+	if (!/"version"\s*:\s*"/.test(before)) {
+		throw new Error(`no "version" field found in ${CATALOG_PATH}`);
+	}
+	const after = before.replace(
+		/("version"\s*:\s*")[^"]*(")/g,
+		(_m, head: string, tail: string) => `${head}${version}${tail}`,
+	);
+	if (after === before) {
+		console.log(`marketplace.json already at ${version}`);
+		return;
+	}
+
+	writeFileSync(CATALOG_PATH, after, "utf8");
+	console.log(`marketplace.json → ${version}`);
+
+	// Stage it so `npm version`'s commit includes the sync.
+	const add = spawnSync("git", ["add", CATALOG_PATH], { cwd: ROOT, stdio: "inherit" });
+	if (add.status !== 0) {
+		throw new Error(`git add failed for ${CATALOG_PATH} (exit ${add.status ?? "signal"})`);
+	}
+}
+
+main();