auto-model-router 0.30.0 → 0.30.2

package/.omp-plugin/marketplace.json

@@ -7,14 +7,14 @@
   },
   "metadata": {
     "description": "auto-model-router: a local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
-    "version": "0.30.0",
+    "version": "0.30.2",
     "pluginRoot": "."
   },
   "plugins": [
     {
       "name": "auto-model-router",
       "description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter. Runs in-process, routes per turn by price and task complexity, with budget caps, mid-stream escalation, and cache-aware hysteresis.",
-      "version": "0.30.0",
+      "version": "0.30.2",
       "author": {
         "name": "drewappling",
         "email": "drewappling@gmail.com"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "auto-model-router",
-  "version": "0.30.0",
+  "version": "0.30.2",
   "private": false,
   "description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
   "type": "module",

package/src/eval/tasks.ts

@@ -307,6 +307,46 @@ export const EVAL_TASKS: readonly EvalTask[] = [
 			return words.every((w) => known.includes(w)) ? 1 : 0.5;
 		},
 	},
+
+	// ---- harder still. The first hard band was aced 6/6 by deepseek-v4.1-flash, so it was
+	// not measuring a ceiling either. These need multi-step state tracking with no shortcut:
+	// the answer cannot be recalled, only computed, and one slip anywhere changes it.
+	{
+		id: "coding/stack-machine",
+		axis: "coding",
+		complexity: "hard",
+		system: JSON_ONLY,
+		user: "A stack machine starts with an empty stack. Execute: PUSH 4, PUSH 7, ADD, PUSH 3, SWAP, SUB, PUSH 2, MUL, DUP, ADD.\nSUB pops a then b and pushes b-a. SWAP exchanges the top two. DUP duplicates the top. Reply with the final stack, bottom to top, comma separated.",
+		// 4 | 4,7 | 11 | 11,3 | 3,11 | 3-11=-8 | -8,2 | -16 | -16,-16 | -32
+		grade: (o) => answerScore(o, "-32"),
+	},
+	{
+		id: "intel/ledger-balance",
+		axis: "intelligence",
+		complexity: "hard",
+		system: JSON_ONLY,
+		user: "An account starts at 0. Apply in order: +120, -45, then double the balance, -30, then halve the balance (round DOWN to a whole number), +7, then subtract a tenth of the balance (round DOWN), finally -1. Reply with the final balance alone.",
+		// 0→120→75→150→120→60→67; a tenth of 67 floors to 6 ⇒ 61; −1 ⇒ 60
+		grade: (o) => answerScore(o, "60"),
+	},
+	{
+		id: "intel/constraint-conflict",
+		axis: "intelligence",
+		complexity: "hard",
+		system: "Follow every constraint. If two constraints cannot both hold, say IMPOSSIBLE and nothing else.",
+		user: "Give a single whole number that is greater than 10, less than 20, divisible by 4, and odd. Reply with the number, or IMPOSSIBLE.",
+		// No odd multiple of 4 exists: recognising the contradiction is the capability.
+		grade: (o) => (/\bimpossible\b/i.test(o) ? 1 : 0),
+	},
+	{
+		id: "coding/regex-backtrack",
+		axis: "coding",
+		complexity: "hard",
+		system: JSON_ONLY,
+		user: "In JavaScript, give the result of each, one per line, in order:\n(1) 'aaa'.replace(/a*/g, 'X')\n(2) 'a1b2'.match(/[a-z](?=\\d)/g).join('')\n(3) /^(a+)+$/.test('aaab')\n(4) 'x.y.z'.split('.', 2).join('|')\n(5) 'abc'.replace(/(b)/, '[$1$$]')",
+		// Empty match at end ⇒ "XX"; lookahead ⇒ "ab"; false; "x|y"; "$$" is a literal $ ⇒ "a[b$]c"
+		grade: (o) => multiAnswerCoverage(o, ["XX", "ab", "false", "x|y", "a[b$]c"]),
+	},
 ];
 
 /**

package/src/server/http.ts

@@ -248,6 +248,11 @@ export function startServer(cfg: RouterConfig): StartedServer {
 	const overrides = createSessionOverrides();
 	const feedback = createFeedbackStore(db);
 	const kv = createKv(db);
+	/**
+	 * Background local-benchmark runs. In memory and not persisted: a run is an explicit,
+	 * attended admin action, and its lasting output is the `local_scores` row it writes.
+	 */
+	const benchmarkJobs = new Map<string, { state: "running" | "done" | "error"; slug: string; startedAtMs: number; result?: Record<string, unknown>; error?: string }>();
 	const digester = createDigester({ cfg, catalog, ledger, upstream, log });
 	const turnDeps = { config: cfg, router, upstream, ledger, conversations, catalog, context, overrides, ollamaCostScale, digester };
 
@@ -768,28 +773,52 @@ export function startServer(cfg: RouterConfig): StartedServer {
 					};
 					const judgeSlug = typeof body?.judge === "string" && body.judge !== "" ? body.judge : "";
 					const asked = typeof body?.concurrency === "number" ? Math.floor(body.concurrency) : 2;
-					const results = await runEval({
-						slugs: [slug, ...anchors],
-						complete,
-						toolComplete,
-						repeats: Math.min(Math.max(1, typeof body?.repeats === "number" ? Math.floor(body.repeats) : 1), 20),
-						concurrency: Math.min(Math.max(1, asked), 8),
-						...(judgeSlug === "" ? {} : { judge: makeJudge(complete, judgeSlug) }),
-					});
-					const target = results[0]!;
-					const anchorResults = results.slice(1);
-					const published = (s: string, axis: QualityAxis): number | undefined => models.find((m) => m.slug === s)?.quality[axis];
-					const cal = fitCalibration(anchorResults, published);
-					const authorOf = (s: string): string => models.find((m) => m.slug === s)?.author ?? "";
-					const fresh = toLocalFeedScores([target], cal, authorOf);
-					if (fresh.length === 0) {
-						return json({ slug, anchors, calibrated: null, raw: target.axes, byComplexity: target.byComplexity, repeats: target.repeats, spread: target.spread, errors: target.errors, applied: false, reason: "no axis produced a usable fit; try more or better-spread anchors" });
-					}
-					// Merge, never replace: other models' measurements are not this run's to discard.
-					const kept = loadLocalScores(db).filter((s) => s.key !== fresh[0]!.key);
-					saveLocalScores(db, [...kept, ...fresh]);
-					log.info("benchmarked a model with the local eval suite", { slug, anchors: anchors.length, errors: target.errors, useLocalScores: cfg.benchmarks.useLocalScores });
-					return json({ slug, anchors, raw: target.axes, byComplexity: target.byComplexity, repeats: target.repeats, spread: target.spread, calibrated: fresh[0], errors: target.errors, applied: cfg.benchmarks.useLocalScores });
+					// The run is a BACKGROUND job, not a long request. Bun caps `idleTimeout` at 255
+					// seconds, so a suite of any size over ten repeats outlives the socket: measured,
+					// a single pass returned in 193s and two passes died at ~350s with the response
+					// lost. The caller polls instead.
+					const jobId = `bench_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`;
+					const started = Date.now();
+					benchmarkJobs.set(jobId, { state: "running", slug, startedAtMs: started });
+					void (async () => {
+						try {
+							const results = await runEval({
+								slugs: [slug, ...anchors],
+								complete,
+								toolComplete,
+								repeats: Math.min(Math.max(1, typeof body?.repeats === "number" ? Math.floor(body.repeats) : 1), 20),
+								concurrency: Math.min(Math.max(1, asked), 8),
+								...(judgeSlug === "" ? {} : { judge: makeJudge(complete, judgeSlug) }),
+							});
+							const target = results[0]!;
+							const published = (s: string, axis: QualityAxis): number | undefined => models.find((m) => m.slug === s)?.quality[axis];
+							const cal = fitCalibration(results.slice(1), published);
+							const authorOf = (s: string): string => models.find((m) => m.slug === s)?.author ?? "";
+							const fresh = toLocalFeedScores([target], cal, authorOf);
+							const shared = { slug, anchors, raw: target.axes, byComplexity: target.byComplexity, repeats: target.repeats, spread: target.spread, errors: target.errors, tookMs: Date.now() - started };
+							if (fresh.length === 0) {
+								benchmarkJobs.set(jobId, { state: "done", slug, startedAtMs: started, result: { ...shared, calibrated: null, applied: false, reason: "no axis produced a usable fit; try more or better-spread anchors" } });
+								return;
+							}
+							// Merge, never replace: other models' measurements are not this run's to discard.
+							const kept = loadLocalScores(db).filter((s) => s.key !== fresh[0]!.key);
+							saveLocalScores(db, [...kept, ...fresh]);
+							log.info("benchmarked a model with the local eval suite", { slug, anchors: anchors.length, repeats: target.repeats, errors: target.errors, useLocalScores: cfg.benchmarks.useLocalScores });
+							benchmarkJobs.set(jobId, { state: "done", slug, startedAtMs: started, result: { ...shared, calibrated: fresh[0], applied: cfg.benchmarks.useLocalScores } });
+						} catch (err) {
+							const message = err instanceof Error ? err.message : String(err);
+							log.warn("local benchmark run failed", { slug, error: message });
+							benchmarkJobs.set(jobId, { state: "error", slug, startedAtMs: started, error: message });
+						}
+					})();
+					return json({ jobId, state: "running", slug, anchors });
+				}
+				if (req.method === "GET" && url.pathname === "/v1/router/benchmark") {
+					const jobId = url.searchParams.get("job");
+					if (jobId === null) return json({ jobs: [...benchmarkJobs.entries()].map(([id, j]) => ({ id, slug: j.slug, state: j.state, startedAtMs: j.startedAtMs })) });
+					const job = benchmarkJobs.get(jobId);
+					if (job === undefined) return wireErrorResponse({ status: 404, code: "invalid_request_error", message: `no benchmark job ${jobId}` });
+					return json({ id: jobId, slug: job.slug, state: job.state, startedAtMs: job.startedAtMs, ...(job.result === undefined ? {} : { result: job.result }), ...(job.error === undefined ? {} : { error: job.error }) });
 				}
 				if (req.method === "POST" && url.pathname === "/v1/router/feedback") {
 					// A user verdict on the newest routed turn of an omp session.

package/test/eval.test.ts

@@ -117,6 +117,14 @@ describe("calibration", () => {
 		expect(hard.find((t) => t.id === "coding/sort-lexicographic")!.grade("[9, 10, 80]")).toBeLessThan(1);
 		expect(hard.find((t) => t.id === "intel/collatz-steps")!.grade("1")).toBeLessThan(1);
 		expect(hard.find((t) => t.id === "intel/strict-format")!.grade("Red, blue, and yellow.")).toBeLessThan(1);
+		// The second hard band: computable only, no recall shortcut.
+		expect(hard.find((t) => t.id === "coding/stack-machine")!.grade("-32")).toBe(1);
+		expect(hard.find((t) => t.id === "coding/stack-machine")!.grade("-16")).toBeLessThan(1);
+		expect(hard.find((t) => t.id === "intel/ledger-balance")!.grade("60")).toBe(1);
+		expect(hard.find((t) => t.id === "intel/ledger-balance")!.grade("61")).toBeLessThan(1);
+		expect(hard.find((t) => t.id === "intel/constraint-conflict")!.grade("IMPOSSIBLE")).toBe(1);
+		expect(hard.find((t) => t.id === "intel/constraint-conflict")!.grade("12")).toBe(0);
+		expect(hard.find((t) => t.id === "coding/regex-backtrack")!.grade("XX\nab\nfalse\nx|y\na[b$]c")).toBe(1);
 	});
 
 	test("fitCalibration + toLocalFeedScores place a target on the AA scale", () => {