auto-model-router 0.2.31 → 0.2.32

package/.omp-plugin/marketplace.json

@@ -7,14 +7,14 @@
   },
   "metadata": {
     "description": "auto-model-router: a local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
-    "version": "0.2.31",
+    "version": "0.2.32",
     "pluginRoot": "."
   },
   "plugins": [
     {
       "name": "auto-model-router",
       "description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter. Runs in-process, routes per turn by price and task complexity, with budget caps, mid-stream escalation, and cache-aware hysteresis.",
-      "version": "0.2.31",
+      "version": "0.2.32",
       "author": {
         "name": "drewappling",
         "email": "drewappling@gmail.com"

package/README.md

@@ -577,6 +577,9 @@ Each task (`coding`, `vision`, `documentation`, `data`, `chat`) is a
 | `minTrustSamples` | `12` | Attempts before trust is enforced. |
 | `trustScopedByHarness` | `false` | `true` = each harness reads only its own trust rows. |
 | `contextHeadroom` | `1.25` | Fraction of context kept free (a model must fit prompt × this). |
+| `latencyWeight` | `0` | How hard to penalise slow models in scoring (soft multiplier on effective cost). `0` disables it. |
+| `latencyMinSamples` | `20` | Streamed samples before latency is judged against a model. |
+| `maxExpectedWaitMs` | unset | Absolute expected-wait ceiling (ms): a hard drop for models *proven* slower (≥ `latencyMinSamples`), regardless of price. The soft penalty is multiplicative and capped, so it cannot demote a slow-but-cheap model — this can. New models keep their cold-start turns; relaxed with trust in tier rescue. Undefined ⇒ off. |
 
 ### `classifier` — complexity adjudication
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "auto-model-router",
-  "version": "0.2.31",
+  "version": "0.2.32",
   "private": false,
   "description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
   "type": "module",

package/src/config/schema.ts

@@ -73,6 +73,7 @@ const filters = z.strictObject({
 	latencyReferenceMs: z.number().positive().optional(),
 	latencyReferenceTokensPerSec: z.number().positive().optional(),
 	latencyMinSamples: z.number().int().nonnegative().optional(),
+	maxExpectedWaitMs: z.number().positive().optional(),
 });
 
 const classifier = z.strictObject({

package/src/config/types.ts

@@ -206,6 +206,19 @@ export interface FilterConfig {
 	latencyReferenceTokensPerSec: number;
 	/** Streamed samples required before latency is scored against a model. */
 	latencyMinSamples: number;
+	/**
+	 * Absolute expected-wait ceiling (ms). A hard drop, mirroring the price
+	 * ceiling: any model whose expected total wait (TTFT + streaming the expected
+	 * completion at its measured throughput) exceeds this is rejected outright,
+	 * regardless of tier or price. This is the gate the latency *penalty* cannot
+	 * be — the penalty is multiplicative on cost and capped, so on an ultra-cheap
+	 * model even the capped multiple leaves it cheapest; a slow-but-cheap model is
+	 * never demoted by scoring alone. Only models with at least `latencyMinSamples`
+	 * observations are dropped, so a new model still gets its cold-start turns.
+	 * Relaxed alongside trust in tier rescue so a narrowed catalog never 500s.
+	 * Undefined ⇒ off (the default).
+	 */
+	maxExpectedWaitMs?: number;
 }
 
 export interface ClassifierConfig {

package/src/router/candidates.ts

@@ -71,6 +71,12 @@ const UNMEASURED_TRUST = 0.9;
 /** Excess-ratio cap so one very slow model cannot be penalised into oblivion. */
 const LATENCY_EXCESS_CAP = 3;
 
+/** Expected total wait: time to first token plus streaming the expected completion at measured throughput. */
+function expectedWaitMs(latency: ModelLatency, expectedCompletionTokens: number): number {
+	const streamMs = latency.tokensPerSec > 0 ? (expectedCompletionTokens / latency.tokensPerSec) * 1000 : 0;
+	return latency.ttftMs + streamMs;
+}
+
 /**
  * Latency penalty as a multiplier on effective cost (>= 1; 1 = no penalty).
  *
@@ -81,13 +87,15 @@ const LATENCY_EXCESS_CAP = 3;
  * model of equal quality and price outranks a sluggish one. Capturing throughput,
  * not just TTFT, is what catches a model that starts fast but streams slowly
  * (deepseek-v4-flash: ~2s TTFT yet ~20 tok/s → ~38s total). Inert when the weight
- * is 0 or the model has too few streamed samples to judge; when throughput is
- * unmeasured it degrades to a TTFT-only comparison against the reference wait.
+ * is 0 or the model has too few streamed samples to judge.
+ *
+ * The penalty CANNOT discipline a slow-but-cheap model: it is multiplicative on a
+ * tiny cost and capped at LATENCY_EXCESS_CAP, so the model stays cheapest. That is
+ * `filters.maxExpectedWaitMs`'s job — a hard drop, applied in buildCandidates.
  */
 function latencyMultiplier(latency: ModelLatency | null, filters: FilterConfig, expectedCompletionTokens: number): number {
 	if (latency === null || filters.latencyWeight <= 0 || latency.samples < filters.latencyMinSamples) return 1;
-	const streamMs = latency.tokensPerSec > 0 ? (expectedCompletionTokens / latency.tokensPerSec) * 1000 : 0;
-	const waitMs = latency.ttftMs + streamMs;
+	const waitMs = expectedWaitMs(latency, expectedCompletionTokens);
 	const refWaitMs = filters.latencyReferenceMs + (expectedCompletionTokens / filters.latencyReferenceTokensPerSec) * 1000;
 	const excess = refWaitMs > 0 ? Math.max(0, (waitMs - refWaitMs) / refWaitMs) : 0;
 	return 1 + filters.latencyWeight * Math.min(excess, LATENCY_EXCESS_CAP);
@@ -233,6 +241,33 @@ export function buildCandidates(args: BuildCandidatesArgs): { candidates: Candid
 			continue;
 		}
 
+		// Fetch latency ONCE for both the ceiling gate here and the scoring
+		// multiplier below. Absolute latency ceiling: a hard drop, mirroring the
+		// price ceiling, for models PROVEN slow (>= latencyMinSamples). The penalty
+		// alone cannot demote a slow-but-cheap model (see latencyMultiplier); this
+		// gate can. Only measured models are dropped, so a new model still gets its
+		// cold-start turns to accumulate samples. Relaxed with trust in rescue.
+		const needLatency = filters.latencyWeight > 0 || filters.maxExpectedWaitMs !== undefined;
+		const latency = needLatency
+			? (ledger?.latency(slug, filters.trustScopedByHarness ? req.harnessId : undefined) ?? null)
+			: null;
+		if (
+			!relaxTrust &&
+			filters.maxExpectedWaitMs !== undefined &&
+			latency !== null &&
+			latency.samples >= filters.latencyMinSamples
+		) {
+			const waitMs = expectedWaitMs(latency, expectedCompletionTokens);
+			if (waitMs > filters.maxExpectedWaitMs) {
+				rejected.push({
+					slug,
+					reason: "over_latency_ceiling",
+					detail: `expected wait ${Math.round(waitMs)}ms > ceiling ${filters.maxExpectedWaitMs}ms (ttft ${Math.round(latency.ttftMs)}ms, ${latency.tokensPerSec.toFixed(0)} tok/s over ${latency.samples} samples)`,
+				});
+				continue;
+			}
+		}
+
 		// Every candidate is priced COLD, deliberately, and this has been measured
 		// rather than assumed. Two reasons:
 		//  1. `coldUsd` feeds the budget guard in select.ts, and a budget must
@@ -259,10 +294,6 @@ export function buildCandidates(args: BuildCandidatesArgs): { candidates: Candid
 		// 20% of the time really costs ~25% more in retries. Latency does the same
 		// for slowness (TTFT over the reference). qualityExponent 0 makes this
 		// "cheapest above the floor"; the floor does the quality work.
-		const latency =
-			filters.latencyWeight > 0
-				? (ledger?.latency(slug, filters.trustScopedByHarness ? req.harnessId : undefined) ?? null)
-				: null;
 		const latencyMult = latencyMultiplier(latency, filters, expectedCompletionTokens);
 		const effectiveUsd = (fc.expectedUsd / Math.max(trustScore, 0.5)) * latencyMult;
 		// Score is assigned in a SECOND PASS below: both qualityNormalization and

package/src/router/types.ts

@@ -127,6 +127,7 @@ export type RejectionReason =
 	| "no_image_support"
 	| "below_quality_floor"
 	| "over_price_ceiling"
+	| "over_latency_ceiling"
 	| "over_budget"
 	| "denylisted"
 	| "not_allowlisted"

package/test/select.test.ts

@@ -594,6 +594,31 @@ describe("latency scoring", () => {
 		const d = run({ tier: "simple", cfg: withWeight(2), ledger });
 		expect(d.slug).not.toBe(slow);
 	});
+
+	const withCeiling = (maxExpectedWaitMs: number, latencyWeight = 0): RouterConfig => ({
+		...BASE,
+		filters: { ...BASE.filters, latencyWeight, latencyReferenceMs: 5000, latencyMinSamples: 20, maxExpectedWaitMs },
+	});
+
+	test("ceiling hard-drops a proven-slow model the penalty cannot, even at weight 0", () => {
+		const slow = run({ tier: "simple" }).slug;
+		const ledger = ledgerWithLatency({ [slow]: { ttftMs: 60_000, samples: 50 } });
+		// latencyWeight 0 → the multiplier is inert; only the hard ceiling can act.
+		const d = run({ tier: "simple", cfg: withCeiling(20_000), ledger });
+		expect(d.slug).not.toBe(slow);
+	});
+
+	test("ceiling spares an under-sampled slow model (cold-start grace)", () => {
+		const slow = run({ tier: "simple" }).slug;
+		const ledger = ledgerWithLatency({ [slow]: { ttftMs: 60_000, samples: 5 } });
+		expect(run({ tier: "simple", cfg: withCeiling(20_000), ledger }).slug).toBe(slow);
+	});
+
+	test("ceiling unset ⇒ no latency gate (proven-slow model still wins on price)", () => {
+		const slow = run({ tier: "simple" }).slug;
+		const ledger = ledgerWithLatency({ [slow]: { ttftMs: 60_000, samples: 50 } });
+		expect(run({ tier: "simple", cfg: withWeight(0), ledger }).slug).toBe(slow);
+	});
 });
 
 describe("context compaction", () => {

package/tools/build-site.ts

@@ -298,6 +298,7 @@ const configBody = `<h2>Configuration reference</h2>
   <dt>filters.minTrust / minTrustSamples</dt><dd>Demote models whose measured reliability falls below the floor once enough samples exist. <span class="default">Default: 0.7 over 12 samples.</span></dd>
   <dt>filters.contextHeadroom</dt><dd>Require a context window this multiple of the estimated prompt. <span class="default">Default: 1.25.</span></dd>
   <dt>filters.latencyWeight</dt><dd>Inflate a model's effective cost by expected wait (TTFT + completion time). <span class="default">Default: 0 (off) \u2014 opt in after establishing a baseline.</span></dd>
+  <dt>filters.maxExpectedWaitMs</dt><dd>Absolute expected-wait ceiling: a hard drop for models <em>proven</em> slower than this (≥ latencyMinSamples), regardless of price \u2014 the soft penalty above is multiplicative and capped, so it cannot demote a slow-but-cheap model. New models keep their cold-start turns. <span class="default">Default: unset (off).</span></dd>
 </dl>
 
 <h3>escalation \u2014 mid-stream recovery</h3>