auto-model-router 0.2.31 → 0.2.33

package/.omp-plugin/marketplace.json

@@ -7,14 +7,14 @@
   },
   "metadata": {
     "description": "auto-model-router: a local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
-    "version": "0.2.31",
+    "version": "0.2.32",
     "pluginRoot": "."
   },
   "plugins": [
     {
       "name": "auto-model-router",
       "description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter. Runs in-process, routes per turn by price and task complexity, with budget caps, mid-stream escalation, and cache-aware hysteresis.",
-      "version": "0.2.31",
+      "version": "0.2.32",
       "author": {
         "name": "drewappling",
         "email": "drewappling@gmail.com"

package/README.md

@@ -577,6 +577,9 @@ Each task (`coding`, `vision`, `documentation`, `data`, `chat`) is a
 | `minTrustSamples` | `12` | Attempts before trust is enforced. |
 | `trustScopedByHarness` | `false` | `true` = each harness reads only its own trust rows. |
 | `contextHeadroom` | `1.25` | Fraction of context kept free (a model must fit prompt × this). |
+| `latencyWeight` | `0` | How hard to penalise slow models in scoring (soft multiplier on effective cost). `0` disables it. |
+| `latencyMinSamples` | `20` | Streamed samples before latency is judged against a model. |
+| `maxExpectedWaitMs` | unset | Absolute expected-wait ceiling (ms): a hard drop for models *proven* slower (≥ `latencyMinSamples`), regardless of price. The soft penalty is multiplicative and capped, so it cannot demote a slow-but-cheap model — this can. New models keep their cold-start turns; relaxed with trust in tier rescue. Undefined ⇒ off. |
 
 ### `classifier` — complexity adjudication
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "auto-model-router",
-  "version": "0.2.31",
+  "version": "0.2.33",
   "private": false,
   "description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
   "type": "module",

package/src/config/defaults.ts

@@ -101,8 +101,11 @@ export const DEFAULT_CONFIG: RouterConfig = {
 		toolAxis: "coding",
 		chatAxis: "intelligence",
 		agenticLoopDepth: 3,
-		// Shipped values, unchanged. See ClassifierConfig.reasoningWeights: a
-		// harness that pins the level for a whole session turns these into a
+		// A mechanical retry (failed tool call + tool-result continuation) keeps
+		// only a fifth of the +0.26; a user-visible failure keeps the full weight.
+		mechanicalRetryFactor: 0.2,
+		// Shipped reasoning values, unchanged. See ClassifierConfig.reasoningWeights:
+		// a harness that pins the level for a whole session turns these into a
 		// constant tier offset, in which case `medium` belongs near 0.
 		reasoningWeights: { medium: 0.14, high: 0.24, xhigh: 0.3, max: 0.34 },
 	},

package/src/config/schema.ts

@@ -73,6 +73,7 @@ const filters = z.strictObject({
 	latencyReferenceMs: z.number().positive().optional(),
 	latencyReferenceTokensPerSec: z.number().positive().optional(),
 	latencyMinSamples: z.number().int().nonnegative().optional(),
+	maxExpectedWaitMs: z.number().positive().optional(),
 });
 
 const classifier = z.strictObject({
@@ -85,6 +86,7 @@ const classifier = z.strictObject({
 	toolAxis: qualityAxis.optional(),
 	chatAxis: qualityAxis.optional(),
 	agenticLoopDepth: z.number().int().nonnegative().optional(),
+	mechanicalRetryFactor: z.number().min(0).max(1).optional(),
 	reasoningWeights: z
 		.strictObject({
 			medium: z.number().nonnegative().optional(),

package/src/config/types.ts

@@ -206,6 +206,19 @@ export interface FilterConfig {
 	latencyReferenceTokensPerSec: number;
 	/** Streamed samples required before latency is scored against a model. */
 	latencyMinSamples: number;
+	/**
+	 * Absolute expected-wait ceiling (ms). A hard drop, mirroring the price
+	 * ceiling: any model whose expected total wait (TTFT + streaming the expected
+	 * completion at its measured throughput) exceeds this is rejected outright,
+	 * regardless of tier or price. This is the gate the latency *penalty* cannot
+	 * be — the penalty is multiplicative on cost and capped, so on an ultra-cheap
+	 * model even the capped multiple leaves it cheapest; a slow-but-cheap model is
+	 * never demoted by scoring alone. Only models with at least `latencyMinSamples`
+	 * observations are dropped, so a new model still gets its cold-start turns.
+	 * Relaxed alongside trust in tier rescue so a narrowed catalog never 500s.
+	 * Undefined ⇒ off (the default).
+	 */
+	maxExpectedWaitMs?: number;
 }
 
 export interface ClassifierConfig {
@@ -229,6 +242,13 @@ export interface ClassifierConfig {
 	chatAxis: QualityAxis;
 	/** Tool-loop depth above which the agentic axis takes over. */
 	agenticLoopDepth: number;
+	/**
+	 * Fraction of the failed-tool weight that survives when the turn is a
+	 * mechanical tool-result continuation. A retry after a failed tool call is
+	 * the most mechanical turn there is; the flat weight let automated retry
+	 * loops buy the hard tier. 1 preserves the shipped behaviour.
+	 */
+	mechanicalRetryFactor: number;
 	/**
 	 * Score added when the CLIENT asks for a reasoning effort, per level. The
 	 * premise is that asking for reasoning states expected difficulty directly.

package/src/router/candidates.ts

@@ -71,6 +71,12 @@ const UNMEASURED_TRUST = 0.9;
 /** Excess-ratio cap so one very slow model cannot be penalised into oblivion. */
 const LATENCY_EXCESS_CAP = 3;
 
+/** Expected total wait: time to first token plus streaming the expected completion at measured throughput. */
+function expectedWaitMs(latency: ModelLatency, expectedCompletionTokens: number): number {
+	const streamMs = latency.tokensPerSec > 0 ? (expectedCompletionTokens / latency.tokensPerSec) * 1000 : 0;
+	return latency.ttftMs + streamMs;
+}
+
 /**
  * Latency penalty as a multiplier on effective cost (>= 1; 1 = no penalty).
  *
@@ -81,13 +87,15 @@ const LATENCY_EXCESS_CAP = 3;
  * model of equal quality and price outranks a sluggish one. Capturing throughput,
  * not just TTFT, is what catches a model that starts fast but streams slowly
  * (deepseek-v4-flash: ~2s TTFT yet ~20 tok/s → ~38s total). Inert when the weight
- * is 0 or the model has too few streamed samples to judge; when throughput is
- * unmeasured it degrades to a TTFT-only comparison against the reference wait.
+ * is 0 or the model has too few streamed samples to judge.
+ *
+ * The penalty CANNOT discipline a slow-but-cheap model: it is multiplicative on a
+ * tiny cost and capped at LATENCY_EXCESS_CAP, so the model stays cheapest. That is
+ * `filters.maxExpectedWaitMs`'s job — a hard drop, applied in buildCandidates.
  */
 function latencyMultiplier(latency: ModelLatency | null, filters: FilterConfig, expectedCompletionTokens: number): number {
 	if (latency === null || filters.latencyWeight <= 0 || latency.samples < filters.latencyMinSamples) return 1;
-	const streamMs = latency.tokensPerSec > 0 ? (expectedCompletionTokens / latency.tokensPerSec) * 1000 : 0;
-	const waitMs = latency.ttftMs + streamMs;
+	const waitMs = expectedWaitMs(latency, expectedCompletionTokens);
 	const refWaitMs = filters.latencyReferenceMs + (expectedCompletionTokens / filters.latencyReferenceTokensPerSec) * 1000;
 	const excess = refWaitMs > 0 ? Math.max(0, (waitMs - refWaitMs) / refWaitMs) : 0;
 	return 1 + filters.latencyWeight * Math.min(excess, LATENCY_EXCESS_CAP);
@@ -233,6 +241,33 @@ export function buildCandidates(args: BuildCandidatesArgs): { candidates: Candid
 			continue;
 		}
 
+		// Fetch latency ONCE for both the ceiling gate here and the scoring
+		// multiplier below. Absolute latency ceiling: a hard drop, mirroring the
+		// price ceiling, for models PROVEN slow (>= latencyMinSamples). The penalty
+		// alone cannot demote a slow-but-cheap model (see latencyMultiplier); this
+		// gate can. Only measured models are dropped, so a new model still gets its
+		// cold-start turns to accumulate samples. Relaxed with trust in rescue.
+		const needLatency = filters.latencyWeight > 0 || filters.maxExpectedWaitMs !== undefined;
+		const latency = needLatency
+			? (ledger?.latency(slug, filters.trustScopedByHarness ? req.harnessId : undefined) ?? null)
+			: null;
+		if (
+			!relaxTrust &&
+			filters.maxExpectedWaitMs !== undefined &&
+			latency !== null &&
+			latency.samples >= filters.latencyMinSamples
+		) {
+			const waitMs = expectedWaitMs(latency, expectedCompletionTokens);
+			if (waitMs > filters.maxExpectedWaitMs) {
+				rejected.push({
+					slug,
+					reason: "over_latency_ceiling",
+					detail: `expected wait ${Math.round(waitMs)}ms > ceiling ${filters.maxExpectedWaitMs}ms (ttft ${Math.round(latency.ttftMs)}ms, ${latency.tokensPerSec.toFixed(0)} tok/s over ${latency.samples} samples)`,
+				});
+				continue;
+			}
+		}
+
 		// Every candidate is priced COLD, deliberately, and this has been measured
 		// rather than assumed. Two reasons:
 		//  1. `coldUsd` feeds the budget guard in select.ts, and a budget must
@@ -259,10 +294,6 @@ export function buildCandidates(args: BuildCandidatesArgs): { candidates: Candid
 		// 20% of the time really costs ~25% more in retries. Latency does the same
 		// for slowness (TTFT over the reference). qualityExponent 0 makes this
 		// "cheapest above the floor"; the floor does the quality work.
-		const latency =
-			filters.latencyWeight > 0
-				? (ledger?.latency(slug, filters.trustScopedByHarness ? req.harnessId : undefined) ?? null)
-				: null;
 		const latencyMult = latencyMultiplier(latency, filters, expectedCompletionTokens);
 		const effectiveUsd = (fc.expectedUsd / Math.max(trustScore, 0.5)) * latencyMult;
 		// Score is assigned in a SECOND PASS below: both qualityNormalization and

package/src/router/classify.ts

@@ -118,7 +118,21 @@ export function scoreHeuristic(f: Features, cfg: RouterConfig): Classification {
 		Math.max(f.trivialityKeywords.length * W_TRIVIALITY_KEYWORD, CAP_TRIVIALITY),
 		`triviality keywords [${f.trivialityKeywords.join(", ")}]`,
 	);
-	if (f.lastToolFailed) add(W_TOOL_FAILED, "last tool result failed");
+	if (f.lastToolFailed) {
+		// A retry after a failed tool call is the MOST mechanical turn there is:
+		// no new user intent, same prompt prefix, the harness just re-asks. The
+		// flat +0.26 let an automated retry loop buy the hard tier ($7.02 of one
+		// measured day vs $0.19 for the same rows as moderate picks). A
+		// continuation keeps only a small nudge; a genuine user-visible failure
+		// (NOT a tool-result continuation) keeps the full weight.
+		const failed = f.isToolResultContinuation ? W_TOOL_FAILED * cfg.classifier.mechanicalRetryFactor : W_TOOL_FAILED;
+		add(
+			failed,
+			f.isToolResultContinuation
+				? `last tool result failed (mechanical retry, damped x${cfg.classifier.mechanicalRetryFactor})`
+				: "last tool result failed",
+		);
+	}
 	if (f.circularToolCall) add(W_CIRCULAR_LOOP, "circular tool call (re-issued a prior call; stuck)");
 	const rw = reasoningWeight(f.requestedReasoning, cfg);
 	if (rw > 0) add(rw, `client requested reasoning=${f.requestedReasoning ?? ""}`);

package/src/router/types.ts

@@ -127,6 +127,7 @@ export type RejectionReason =
 	| "no_image_support"
 	| "below_quality_floor"
 	| "over_price_ceiling"
+	| "over_latency_ceiling"
 	| "over_budget"
 	| "denylisted"
 	| "not_allowlisted"

package/test/classify.test.ts

@@ -269,9 +269,30 @@ describe("scoreHeuristic", () => {
 		expect(deepCircular.tier).toBe("hard");
 	});
 
-	test("a failing tool result on a deep loop is at least moderate", () => {
+	test("a failing tool result on a deep loop is at least simple", () => {
+		// Was 'at least moderate' before the mechanical-retry damp: the flat
+		// +0.26 pushed deep mechanical retry loops into hard. A damped retry
+		// still clears trivial.
 		const deepAndFailing = scoreHeuristic(contFeatures(20, { lastToolFailed: true }), BASE);
-		expect(tierIdx(deepAndFailing.tier)).toBeGreaterThanOrEqual(tierIdx("moderate"));
+		expect(tierIdx(deepAndFailing.tier)).toBeGreaterThanOrEqual(tierIdx("simple"));
+	});
+
+	test("a failed-tool retry on a mechanical continuation is damped, not hard", () => {
+		// A retry after a failed tool call is the most mechanical turn there is;
+		// the flat +0.26 let automated retry loops buy the hard tier ($7.02 of one
+		// measured day vs $0.19 for the same rows as moderate picks). The
+		// continuation keeps only mechanicalRetryFactor of the weight.
+		const retry = scoreHeuristic(contFeatures(20, { lastToolFailed: true }), BASE);
+		const quiet = scoreHeuristic(contFeatures(20), BASE);
+		expect(tierIdx(retry.tier)).toBeLessThan(tierIdx("hard"));
+		expect(retry.score - quiet.score).toBeCloseTo(
+			BASE.classifier.mechanicalRetryFactor * 0.26,
+			5,
+		);
+		// A failure the USER sees (not a tool-result continuation) keeps the full
+		// weight: that genuinely changes what the turn needs.
+		const userSeen = scoreHeuristic(contFeatures(2, { isToolResultContinuation: false, lastToolFailed: true }), BASE);
+		expect(userSeen.score - scoreHeuristic(contFeatures(2, { isToolResultContinuation: false }), BASE).score).toBeCloseTo(0.26, 5);
 	});
 });
 

package/test/failover.test.ts

@@ -55,6 +55,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
 			toolAxis: "coding",
 			chatAxis: "intelligence",
 			agenticLoopDepth: 3,
+			mechanicalRetryFactor: 0.2,
 			reasoningWeights: { medium: 0.14, high: 0.24, xhigh: 0.3, max: 0.34 },
 		},
 		escalation: {

package/test/select.test.ts

@@ -594,6 +594,31 @@ describe("latency scoring", () => {
 		const d = run({ tier: "simple", cfg: withWeight(2), ledger });
 		expect(d.slug).not.toBe(slow);
 	});
+
+	const withCeiling = (maxExpectedWaitMs: number, latencyWeight = 0): RouterConfig => ({
+		...BASE,
+		filters: { ...BASE.filters, latencyWeight, latencyReferenceMs: 5000, latencyMinSamples: 20, maxExpectedWaitMs },
+	});
+
+	test("ceiling hard-drops a proven-slow model the penalty cannot, even at weight 0", () => {
+		const slow = run({ tier: "simple" }).slug;
+		const ledger = ledgerWithLatency({ [slow]: { ttftMs: 60_000, samples: 50 } });
+		// latencyWeight 0 → the multiplier is inert; only the hard ceiling can act.
+		const d = run({ tier: "simple", cfg: withCeiling(20_000), ledger });
+		expect(d.slug).not.toBe(slow);
+	});
+
+	test("ceiling spares an under-sampled slow model (cold-start grace)", () => {
+		const slow = run({ tier: "simple" }).slug;
+		const ledger = ledgerWithLatency({ [slow]: { ttftMs: 60_000, samples: 5 } });
+		expect(run({ tier: "simple", cfg: withCeiling(20_000), ledger }).slug).toBe(slow);
+	});
+
+	test("ceiling unset ⇒ no latency gate (proven-slow model still wins on price)", () => {
+		const slow = run({ tier: "simple" }).slug;
+		const ledger = ledgerWithLatency({ [slow]: { ttftMs: 60_000, samples: 50 } });
+		expect(run({ tier: "simple", cfg: withWeight(0), ledger }).slug).toBe(slow);
+	});
 });
 
 describe("context compaction", () => {

package/test/turn.test.ts

@@ -56,6 +56,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
 			toolAxis: "coding",
 			chatAxis: "intelligence",
 			agenticLoopDepth: 3,
+			mechanicalRetryFactor: 0.2,
 			reasoningWeights: { medium: 0.14, high: 0.24, xhigh: 0.3, max: 0.34 },
 		},
 		escalation: {

package/tools/build-site.ts

@@ -298,6 +298,7 @@ const configBody = `<h2>Configuration reference</h2>
   <dt>filters.minTrust / minTrustSamples</dt><dd>Demote models whose measured reliability falls below the floor once enough samples exist. <span class="default">Default: 0.7 over 12 samples.</span></dd>
   <dt>filters.contextHeadroom</dt><dd>Require a context window this multiple of the estimated prompt. <span class="default">Default: 1.25.</span></dd>
   <dt>filters.latencyWeight</dt><dd>Inflate a model's effective cost by expected wait (TTFT + completion time). <span class="default">Default: 0 (off) \u2014 opt in after establishing a baseline.</span></dd>
+  <dt>filters.maxExpectedWaitMs</dt><dd>Absolute expected-wait ceiling: a hard drop for models <em>proven</em> slower than this (≥ latencyMinSamples), regardless of price \u2014 the soft penalty above is multiplicative and capped, so it cannot demote a slow-but-cheap model. New models keep their cold-start turns. <span class="default">Default: unset (off).</span></dd>
 </dl>
 
 <h3>escalation \u2014 mid-stream recovery</h3>