auto-model-router 0.30.1 → 0.30.3

package/.omp-plugin/marketplace.json

@@ -7,14 +7,14 @@
   },
   "metadata": {
     "description": "auto-model-router: a local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
-    "version": "0.30.1",
+    "version": "0.30.3",
     "pluginRoot": "."
   },
   "plugins": [
     {
       "name": "auto-model-router",
       "description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter. Runs in-process, routes per turn by price and task complexity, with budget caps, mid-stream escalation, and cache-aware hysteresis.",
-      "version": "0.30.1",
+      "version": "0.30.3",
       "author": {
         "name": "drewappling",
         "email": "drewappling@gmail.com"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "auto-model-router",
-  "version": "0.30.1",
+  "version": "0.30.3",
   "private": false,
   "description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
   "type": "module",

package/src/eval/run.ts

@@ -67,6 +67,12 @@ export interface RunEvalArgs {
 	concurrency?: number;
 	/** Called as each model finishes, for progress logging. */
 	onProgress?: (result: EvalResult, done: number, total: number) => void;
+	/**
+	 * Called as each PASS of each model finishes. A run is `models x repeats` passes and takes
+	 * the better part of an hour at ten repeats, so per-model progress is too coarse to show
+	 * anyone: without this the only observable states are "running" and "finished".
+	 */
+	onPass?: (slug: string, pass: number, of: number) => void;
 	/**
 	 * A tool-capable completer. Absent ⇒ the agentic SCENARIOS are skipped and the axis falls
 	 * back to the text tasks, which only ever measured whether a model can format JSON.
@@ -171,6 +177,7 @@ async function scoreModel(slug: string, args: RunEvalArgs): Promise<EvalResult>
 	let errors = 0;
 	for (let i = 0; i < passes; i++) {
 		const pass = await scorePass(slug, args);
+		args.onPass?.(slug, i + 1, passes);
 		errors += pass.errors;
 		for (const axis of AXES) {
 			axes[axis].sum += pass.axes[axis].sum;

package/src/server/http.ts

@@ -182,6 +182,19 @@ export function ollamaRunway(
 	return { dailyBurnUsd, creditsLeftUsd, days: dailyBurnUsd > 0 ? creditsLeftUsd / dailyBurnUsd : null };
 }
 
+/**
+ * Progress for a local benchmark run, as the dashboard needs it. The ETA is measured from
+ * this run's own completed passes rather than assumed — passes vary with the model's speed,
+ * and a guess from a fixed per-pass constant would be wrong by minutes on a slow upstream.
+ * Null until a pass has finished, because until then there is nothing to extrapolate from.
+ */
+function benchmarkProgress(job: { state: string; startedAtMs: number; donePasses: number; totalPasses: number; current: string }): Record<string, unknown> {
+	const elapsedMs = Date.now() - job.startedAtMs;
+	const fraction = job.totalPasses > 0 ? job.donePasses / job.totalPasses : 0;
+	const etaMs = job.state === "running" && job.donePasses > 0 ? Math.round((elapsedMs / job.donePasses) * (job.totalPasses - job.donePasses)) : null;
+	return { donePasses: job.donePasses, totalPasses: job.totalPasses, fraction, current: job.current, elapsedMs, etaMs };
+}
+
 function json(data: unknown, status = 200): Response {
 	return new Response(JSON.stringify(data), {
 		status,
@@ -248,6 +261,11 @@ export function startServer(cfg: RouterConfig): StartedServer {
 	const overrides = createSessionOverrides();
 	const feedback = createFeedbackStore(db);
 	const kv = createKv(db);
+	/**
+	 * Background local-benchmark runs. In memory and not persisted: a run is an explicit,
+	 * attended admin action, and its lasting output is the `local_scores` row it writes.
+	 */
+	const benchmarkJobs = new Map<string, { state: "running" | "done" | "error"; slug: string; startedAtMs: number; donePasses: number; totalPasses: number; current: string; result?: Record<string, unknown>; error?: string }>();
 	const digester = createDigester({ cfg, catalog, ledger, upstream, log });
 	const turnDeps = { config: cfg, router, upstream, ledger, conversations, catalog, context, overrides, ollamaCostScale, digester };
 
@@ -768,28 +786,59 @@ export function startServer(cfg: RouterConfig): StartedServer {
 					};
 					const judgeSlug = typeof body?.judge === "string" && body.judge !== "" ? body.judge : "";
 					const asked = typeof body?.concurrency === "number" ? Math.floor(body.concurrency) : 2;
-					const results = await runEval({
-						slugs: [slug, ...anchors],
-						complete,
-						toolComplete,
-						repeats: Math.min(Math.max(1, typeof body?.repeats === "number" ? Math.floor(body.repeats) : 1), 20),
-						concurrency: Math.min(Math.max(1, asked), 8),
-						...(judgeSlug === "" ? {} : { judge: makeJudge(complete, judgeSlug) }),
-					});
-					const target = results[0]!;
-					const anchorResults = results.slice(1);
-					const published = (s: string, axis: QualityAxis): number | undefined => models.find((m) => m.slug === s)?.quality[axis];
-					const cal = fitCalibration(anchorResults, published);
-					const authorOf = (s: string): string => models.find((m) => m.slug === s)?.author ?? "";
-					const fresh = toLocalFeedScores([target], cal, authorOf);
-					if (fresh.length === 0) {
-						return json({ slug, anchors, calibrated: null, raw: target.axes, byComplexity: target.byComplexity, repeats: target.repeats, spread: target.spread, errors: target.errors, applied: false, reason: "no axis produced a usable fit; try more or better-spread anchors" });
-					}
-					// Merge, never replace: other models' measurements are not this run's to discard.
-					const kept = loadLocalScores(db).filter((s) => s.key !== fresh[0]!.key);
-					saveLocalScores(db, [...kept, ...fresh]);
-					log.info("benchmarked a model with the local eval suite", { slug, anchors: anchors.length, errors: target.errors, useLocalScores: cfg.benchmarks.useLocalScores });
-					return json({ slug, anchors, raw: target.axes, byComplexity: target.byComplexity, repeats: target.repeats, spread: target.spread, calibrated: fresh[0], errors: target.errors, applied: cfg.benchmarks.useLocalScores });
+					// The run is a BACKGROUND job, not a long request. Bun caps `idleTimeout` at 255
+					// seconds, so a suite of any size over ten repeats outlives the socket: measured,
+					// a single pass returned in 193s and two passes died at ~350s with the response
+					// lost. The caller polls instead.
+					const jobId = `bench_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`;
+					const started = Date.now();
+					const repeats = Math.min(Math.max(1, typeof body?.repeats === "number" ? Math.floor(body.repeats) : 1), 20);
+					// A pass of one model is the unit of progress: models x repeats, known up front.
+					const totalPasses = repeats * (1 + anchors.length);
+					benchmarkJobs.set(jobId, { state: "running", slug, startedAtMs: started, donePasses: 0, totalPasses, current: slug });
+					void (async () => {
+						try {
+							const results = await runEval({
+								slugs: [slug, ...anchors],
+								complete,
+								toolComplete,
+								repeats,
+								concurrency: Math.min(Math.max(1, asked), 8),
+								onPass: (passSlug, _pass, _of) => {
+									const job = benchmarkJobs.get(jobId);
+									if (job !== undefined) benchmarkJobs.set(jobId, { ...job, donePasses: job.donePasses + 1, current: passSlug });
+								},
+								...(judgeSlug === "" ? {} : { judge: makeJudge(complete, judgeSlug) }),
+							});
+							const target = results[0]!;
+							const published = (s: string, axis: QualityAxis): number | undefined => models.find((m) => m.slug === s)?.quality[axis];
+							const cal = fitCalibration(results.slice(1), published);
+							const authorOf = (s: string): string => models.find((m) => m.slug === s)?.author ?? "";
+							const fresh = toLocalFeedScores([target], cal, authorOf);
+							const shared = { slug, anchors, raw: target.axes, byComplexity: target.byComplexity, repeats: target.repeats, spread: target.spread, errors: target.errors, tookMs: Date.now() - started };
+							if (fresh.length === 0) {
+								benchmarkJobs.set(jobId, { ...benchmarkJobs.get(jobId)!, state: "done", result: { ...shared, calibrated: null, applied: false, reason: "no axis produced a usable fit; try more or better-spread anchors" } });
+								return;
+							}
+							// Merge, never replace: other models' measurements are not this run's to discard.
+							const kept = loadLocalScores(db).filter((s) => s.key !== fresh[0]!.key);
+							saveLocalScores(db, [...kept, ...fresh]);
+							log.info("benchmarked a model with the local eval suite", { slug, anchors: anchors.length, repeats: target.repeats, errors: target.errors, useLocalScores: cfg.benchmarks.useLocalScores });
+							benchmarkJobs.set(jobId, { ...benchmarkJobs.get(jobId)!, state: "done", result: { ...shared, calibrated: fresh[0], applied: cfg.benchmarks.useLocalScores } });
+						} catch (err) {
+							const message = err instanceof Error ? err.message : String(err);
+							log.warn("local benchmark run failed", { slug, error: message });
+							benchmarkJobs.set(jobId, { ...benchmarkJobs.get(jobId)!, state: "error", error: message });
+						}
+					})();
+					return json({ jobId, state: "running", slug, anchors, totalPasses });
+				}
+				if (req.method === "GET" && url.pathname === "/v1/router/benchmark") {
+					const jobId = url.searchParams.get("job");
+					if (jobId === null) return json({ jobs: [...benchmarkJobs.entries()].map(([id, j]) => ({ id, slug: j.slug, state: j.state, startedAtMs: j.startedAtMs, ...benchmarkProgress(j) })) });
+					const job = benchmarkJobs.get(jobId);
+					if (job === undefined) return wireErrorResponse({ status: 404, code: "invalid_request_error", message: `no benchmark job ${jobId}` });
+					return json({ id: jobId, slug: job.slug, state: job.state, startedAtMs: job.startedAtMs, ...benchmarkProgress(job), ...(job.result === undefined ? {} : { result: job.result }), ...(job.error === undefined ? {} : { error: job.error }) });
 				}
 				if (req.method === "POST" && url.pathname === "/v1/router/feedback") {
 					// A user verdict on the newest routed turn of an omp session.