auto-model-router 0.2.21 → 0.2.23

package/tools/verify-embed-port.ts

@@ -0,0 +1,91 @@
+/**
+ * Verifies the embed port contract that omp's model resolution depends on.
+ *
+ * omp resolves `modelRoles.default` from `models.yml` during startup — BEFORE
+ * extensions load, so before the router can bind and rewrite that file. With an
+ * ephemeral port the block therefore names the PREVIOUS session's port, which
+ * is dead once that session exits: every main-agent turn fails with "Unable to
+ * connect" while utility calls (resolved later, from the live registration)
+ * still work. This asserts the properties that make that impossible:
+ *
+ *  1. The bind port is deterministic across restarts (configured port wins).
+ *  2. A second session finds the first one healthy, so it can REUSE it.
+ *  3. A port held by something that is not our router is not mistaken for one,
+ *     and binding falls back instead of leaving the session with no provider.
+ *
+ * Run: bun tools/verify-embed-port.ts
+ */
+import { mkdtempSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { probeEmbed, resolveEmbedPort } from "../omp-extension/embed-logic.ts";
+import { loadConfig } from "../src/config/load.ts";
+import { startServer, type StartedServer } from "../src/server/http.ts";
+import type { RouterConfig } from "../src/config/types.ts";
+
+let failures = 0;
+const check = (label: string, ok: boolean, detail?: unknown): void => {
+	console.log(`  ${ok ? "PASS" : "FAIL"}  ${label}${ok || detail === undefined ? "" : `\n        ${JSON.stringify(detail)}`}`);
+	if (!ok) failures++;
+};
+
+const home = mkdtempSync(join(tmpdir(), "verify-embed-port-"));
+function baseCfg(): RouterConfig {
+	const cfg = loadConfig({ overrides: { server: { host: "127.0.0.1" } } });
+	cfg.ledger.path = join(home, "router.db");
+	cfg.logLevel = "error";
+	cfg.benchmarks.enabled = false;
+	cfg.context.enabled = false;
+	return cfg;
+}
+
+// 1. Determinism: two independent startups must choose the same port.
+const cfg1 = baseCfg();
+const configured = cfg1.server.port;
+const portA = resolveEmbedPort(undefined, configured);
+const portB = resolveEmbedPort(undefined, baseCfg().server.port);
+check("bind port is deterministic across restarts", portA === portB && portA !== 0, { portA, portB });
+check("deterministic port comes from config.yml server.port", portA === configured, { portA, configured });
+check("an explicit env port still wins", resolveEmbedPort("8812", configured) === 8812);
+check("an explicit env 0 still requests an ephemeral port", resolveEmbedPort("0", configured) === 0);
+
+// 2. First session binds it; a second must see it healthy (=> reuse, no bind war).
+cfg1.server.port = portA;
+let first: StartedServer | null = null;
+try {
+	first = startServer(cfg1);
+} catch (err) {
+	check("first session can bind the deterministic port", false, String(err));
+}
+
+if (first !== null) {
+	check("first session bound the deterministic port", first.server.port === portA, { bound: first.server.port, wanted: portA });
+	check("router answers /health there (so a peer session reuses it)", await probeEmbed(portA), { port: portA });
+
+	// 3. A non-router occupant must not be mistaken for our router.
+	const squatter = Bun.serve({ port: 0, hostname: "127.0.0.1", fetch: () => new Response("not a router", { status: 404 }) });
+	const squatted = squatter.port as number;
+	check("a non-router occupant fails the health probe", (await probeEmbed(squatted)) === false, { port: squatted });
+
+	// ...and binding over it must still leave the session with a usable router.
+	const cfgFallback = baseCfg();
+	cfgFallback.server.port = squatted;
+	let fell: StartedServer | null = null;
+	try {
+		fell = startServer(cfgFallback);
+		check("bind on an occupied port yields a different, usable port", fell.server.port !== squatted, { got: fell.server.port });
+	} catch {
+		cfgFallback.server.port = 0;
+		fell = startServer(cfgFallback);
+		check("falls back to an ephemeral port when the desired one is taken", fell.server.port !== undefined && fell.server.port !== squatted, {
+			got: fell.server.port,
+		});
+	}
+	if (fell !== null) await fell.stop();
+	squatter.stop(true);
+	await first.stop();
+}
+
+console.log(failures === 0 ? "\nAll checks passed." : `\n${failures} check(s) FAILED.`);
+process.exit(failures === 0 ? 0 : 1);

package/tools/verify-plan-persist.ts

@@ -0,0 +1,127 @@
+/**
+ * End-to-end verification of compaction plan stability.
+ *
+ * Two properties, both measured against a real router process over a growing
+ * agentic conversation:
+ *
+ *  1. STABILITY — an edit applied on one turn is re-applied byte-identically on
+ *     every later turn. Any change to already-sent bytes invalidates the
+ *     upstream prompt cache from that message onward.
+ *  2. CHURN — how many turns change the plan at all. Each change is a cache
+ *     invalidation; live ledger data puts a changed-plan turn at 15.4% cold vs
+ *     8.9% when the plan holds, and a cold prompt costs 4.34x a warm one per
+ *     token. `compaction.floorRatio` trades a little extra elision for far
+ *     fewer changes.
+ *
+ * Run: bun tools/verify-plan-persist.ts [floorRatio]
+ */
+import { mkdtempSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { loadConfig } from "../src/config/load.ts";
+import { startServer } from "../src/server/http.ts";
+import { startMockOpenRouter } from "./mock-openrouter.ts";
+
+const floorRatio = Number.parseFloat(process.argv[2] ?? "0.75");
+const home = mkdtempSync(join(tmpdir(), "verify-plan-persist-"));
+const mock = await startMockOpenRouter("test/fixtures/openrouter-models.json");
+
+const cfg = loadConfig({});
+cfg.server = { host: "127.0.0.1", port: 0 };
+cfg.openrouter.baseUrl = `${mock.url}/api/v1`;
+cfg.openrouter.apiKey = "sk-mock";
+cfg.ledger.path = join(home, "router.db");
+cfg.logLevel = "error";
+cfg.classifier.ambiguityThreshold = 0;
+cfg.benchmarks.enabled = false;
+cfg.context.enabled = false;
+// Scaled-down budget so the fixture behaves like a 40k-budget real conversation.
+cfg.compaction.enabled = true;
+cfg.compaction.budgetTokens = 1_500;
+cfg.compaction.floorRatio = floorRatio;
+cfg.compaction.maxToolResultBytes = 256;
+cfg.compaction.keepHeadBytes = 16;
+cfg.compaction.keepTailBytes = 16;
+
+const app = startServer(cfg);
+const base = `http://127.0.0.1:${app.server.port}`;
+
+const big = (marker: string): string => `${marker}: ${"payload ".repeat(60)}`;
+const call = (id: string, name: string, args: unknown): unknown => ({
+	role: "assistant",
+	content: null,
+	tool_calls: [{ id, type: "function", function: { name, arguments: JSON.stringify(args) } }],
+});
+const result = (id: string, content: string): unknown => ({ role: "tool", tool_call_id: id, content });
+const cycle = (n: number): unknown[] => [
+	call(`c${n}`, "read", { path: `src/file${n}.ts` }),
+	result(`c${n}`, big(`READ${n}`)),
+	{ role: "assistant", content: `read file${n}` },
+];
+
+async function dispatch(messages: unknown[]): Promise<{ role: string; content: unknown }[]> {
+	const res = await fetch(`${base}/v1/chat/completions`, {
+		method: "POST",
+		headers: { "content-type": "application/json" },
+		body: JSON.stringify({ model: "auto", messages, stream: true }),
+	});
+	await res.text();
+	const body = mock.requests.at(-1)?.body as Record<string, unknown>;
+	return body.messages as { role: string; content: unknown }[];
+}
+
+const fail = (label: string, detail?: unknown): never => {
+	console.error(`FAIL ${label}`, detail === undefined ? "" : JSON.stringify(detail).slice(0, 500));
+	process.exit(1);
+};
+const shrunkOf = (msgs: { content: unknown }[]): Map<number, string> => {
+	const out = new Map<number, string>();
+	msgs.forEach((m, i) => {
+		if (typeof m.content === "string" && m.content.includes("omp-router: elided")) out.set(i, m.content);
+	});
+	return out;
+};
+
+// A 20-cycle conversation, dispatched turn by turn exactly as omp would: the
+// full history every time, one cycle longer each turn.
+const TURNS = 20;
+let history: unknown[] = [{ role: "user", content: "audit the project" }];
+let prev = new Map<number, string>();
+let changes = 0;
+let firstPlanTurn = 0;
+
+for (let n = 1; n <= TURNS; n++) {
+	history = [...history, ...cycle(n)];
+	const msgs = await dispatch(history);
+	const shrunk = shrunkOf(msgs);
+
+	// STABILITY: every previously-shrunk message must still be shrunk, with the
+	// same bytes. A dropped or altered edit rewrites the cached prefix.
+	for (const [i, content] of prev) {
+		const now = shrunk.get(i);
+		if (now === undefined) fail(`turn ${n}: edit at message ${i} was DROPPED (bytes re-inflated)`, { turn: n, i });
+		if (now !== content) fail(`turn ${n}: edit at message ${i} changed bytes`, { was: content.slice(0, 90), now: now.slice(0, 90) });
+	}
+
+	const added = [...shrunk.keys()].filter((i) => !prev.has(i));
+	if (added.length > 0) {
+		changes++;
+		if (firstPlanTurn === 0) firstPlanTurn = n;
+		console.log(`turn ${String(n).padStart(2)}: plan CHANGED (+${added.length} edits, ${shrunk.size} total)`);
+	} else if (shrunk.size > 0) {
+		console.log(`turn ${String(n).padStart(2)}: plan held (${shrunk.size} edits)`);
+	} else {
+		console.log(`turn ${String(n).padStart(2)}: no compaction`);
+	}
+	prev = shrunk;
+}
+
+const planningTurns = TURNS - firstPlanTurn + 1;
+console.log(`\nfloorRatio ${floorRatio}`);
+console.log(`PASS stability: no edit was ever dropped or rewritten across ${TURNS} turns`);
+console.log(`plan changes: ${changes} over ${planningTurns} compacting turns (${((changes / planningTurns) * 100).toFixed(0)}% of turns invalidate cache)`);
+
+app.stop(true);
+await mock.stop();
+process.exit(0);