auto-model-router 0.2.1 → 0.2.2

package/.omp-plugin/marketplace.json

@@ -7,14 +7,14 @@
   },
   "metadata": {
     "description": "auto-model-router: a local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
-    "version": "0.2.1",
+    "version": "0.2.2",
     "pluginRoot": "."
   },
   "plugins": [
     {
       "name": "auto-model-router",
       "description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter. Runs in-process, routes per turn by price and task complexity, with budget caps, mid-stream escalation, and cache-aware hysteresis.",
-      "version": "0.2.1",
+      "version": "0.2.2",
       "author": {
         "name": "drewappling",
         "email": "drewappling@gmail.com"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "auto-model-router",
-  "version": "0.2.1",
+  "version": "0.2.2",
   "private": false,
   "description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
   "type": "module",

package/src/config/defaults.ts

@@ -68,6 +68,11 @@ export const DEFAULT_CONFIG: RouterConfig = {
 		// even with a tiny guardrail-narrowed catalog.
 		trustScopedByHarness: false,
 		contextHeadroom: 1.25,
+		// Latency scoring is off by default (weight 0): opt in after establishing a
+		// baseline. TTFT above the reference inflates a model's effective cost.
+		latencyWeight: 0,
+		latencyReferenceMs: 5000,
+		latencyMinSamples: 20,
 	},
 	classifier: {
 		ambiguityThreshold: 0.6,

package/src/config/schema.ts

@@ -65,6 +65,9 @@ const filters = z.strictObject({
 	minTrustSamples: z.number().int().nonnegative().optional(),
 	trustScopedByHarness: z.boolean().optional(),
 	contextHeadroom: z.number().positive().optional(),
+	latencyWeight: z.number().nonnegative().optional(),
+	latencyReferenceMs: z.number().positive().optional(),
+	latencyMinSamples: z.number().int().nonnegative().optional(),
 });
 
 const classifier = z.strictObject({

package/src/config/types.ts

@@ -138,6 +138,17 @@ export interface FilterConfig {
 	 * model's context window, absorbing token-estimate error and the response.
 	 */
 	contextHeadroom: number;
+	/**
+	 * How hard to penalise slow models in candidate scoring. A model's mean
+	 * time-to-first-token above `latencyReferenceMs` inflates its effective
+	 * cost — the same lever trust uses for flakiness — so a faster model of
+	 * equal quality and price wins. 0 disables latency scoring entirely.
+	 */
+	latencyWeight: number;
+	/** TTFT (ms) below which a model is considered fully responsive (no penalty). */
+	latencyReferenceMs: number;
+	/** Streamed samples required before latency is scored against a model. */
+	latencyMinSamples: number;
 }
 
 export interface ClassifierConfig {

package/src/cost/ledger.ts

@@ -18,7 +18,7 @@ import type { RouterConfig } from "../config/types.ts";
 import { consumePendingEstimate } from "../tokens/estimate.ts";
 import { computeBlendedRate } from "./blended.ts";
 import { computeCost } from "./forecast.ts";
-import type { BlendedRate, Ledger, LedgerEntry, ModelTrust, UsageCounts } from "./types.ts";
+import type { BlendedRate, Ledger, LedgerEntry, ModelLatency, ModelTrust, UsageCounts } from "./types.ts";
 
 /** Estimates below this many samples are noise; the default ratio is better. */
 const MIN_CALIBRATION_SAMPLES = 20;
@@ -67,6 +67,11 @@ interface TrustRow {
 	mean_cost_error: number | null;
 }
 
+interface LatencyRow {
+	samples: number;
+	ttft_ms: number | null;
+}
+
 interface CalibrationRow {
 	est_bytes: number;
 	actual_tokens: number;
@@ -98,6 +103,16 @@ const TRUST_SELECT = `COUNT(*) AS attempts,
 		AVG(CASE WHEN reported_usd IS NOT NULL AND reported_usd > 0
 			THEN ABS(reported_usd - predicted_usd) / reported_usd END) AS mean_cost_error`;
 
+/**
+ * Time-to-first-token, averaged over streamed, non-errored turns. TTFT (not
+ * total latency) isolates model+provider responsiveness from answer length: a
+ * model is "slow" when it takes a long time to START, not when it was asked for
+ * a long answer. Errored/aborted rows and non-streaming rows (null ttft) are
+ * excluded — they carry no responsiveness signal.
+ */
+const LATENCY_SELECT = `COUNT(CASE WHEN ttft_ms IS NOT NULL AND ttft_ms > 0 AND error IS NULL THEN 1 END) AS samples,
+		AVG(CASE WHEN ttft_ms IS NOT NULL AND ttft_ms > 0 AND error IS NULL THEN ttft_ms END) AS ttft_ms`;
+
 /**
  * Recovers the `UpstreamErrorKind` from the text turn.ts stored.
  *
@@ -127,6 +142,11 @@ function toTrust(slug: string, row: TrustRow): ModelTrust {
 	};
 }
 
+function toLatency(slug: string, row: LatencyRow): ModelLatency | null {
+	if (row.samples <= 0 || row.ttft_ms === null) return null;
+	return { slug, samples: row.samples, ttftMs: row.ttft_ms };
+}
+
 function toEntry(row: LedgerRow): LedgerEntry {
 	return {
 		id: row.id,
@@ -192,6 +212,8 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
 	const trustStmt = db.query(`SELECT ${TRUST_SELECT} FROM ledger WHERE slug = ?`);
 	const trustHarnessStmt = db.query(`SELECT ${TRUST_SELECT} FROM ledger WHERE slug = ? AND harness_id = ?`);
 	const allTrustStmt = db.query(`SELECT slug, ${TRUST_SELECT} FROM ledger GROUP BY slug`);
+	const latencyStmt = db.query(`SELECT ${LATENCY_SELECT} FROM ledger WHERE slug = ?`);
+	const latencyHarnessStmt = db.query(`SELECT ${LATENCY_SELECT} FROM ledger WHERE slug = ? AND harness_id = ?`);
 	const ratioStmt = db.query("SELECT est_bytes, actual_tokens, samples FROM token_calibration WHERE tokenizer = ?");
 	const recentStmt = db.query("SELECT * FROM ledger ORDER BY created_at_ms DESC LIMIT ?");
 	const cacheMetaStmt = db.query("SELECT fetched_at_ms FROM catalog_cache WHERE id = 1");
@@ -301,6 +323,15 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
 			return rows.map((row) => toTrust(row.slug, row));
 		},
 
+		latency(slug: string, harnessId?: string): ModelLatency | null {
+			const row =
+				harnessId !== undefined && harnessId !== ""
+					? (latencyHarnessStmt.get(slug, harnessId) as LatencyRow | null)
+					: (latencyStmt.get(slug) as LatencyRow | null);
+			if (row === null) return null;
+			return toLatency(slug, row);
+		},
+
 		tokenRatio(tokenizer: string): number | null {
 			const row = ratioStmt.get(tokenizer.trim().toLowerCase()) as CalibrationRow | null;
 			if (row === null || row.samples < MIN_CALIBRATION_SAMPLES || row.actual_tokens <= 0) return null;

package/src/cost/types.ts

@@ -159,6 +159,14 @@ export interface ModelTrust {
 	meanCostError: number;
 }
 
+/** Per-model responsiveness learned from our own traffic. Feeds candidate scoring. */
+export interface ModelLatency {
+	slug: string;
+	samples: number;
+	/** Mean time-to-first-token, ms, over streamed non-errored turns. */
+	ttftMs: number;
+}
+
 export interface Ledger {
 	record(entry: LedgerEntry): void;
 	/** Total reported (or predicted, when reported is null) spend for a conversation. */
@@ -172,6 +180,13 @@ export interface Ledger {
 	/** Per-model reliability over the ledger, optionally scoped to a harness. */
 	trust(slug: string, harnessId?: string): ModelTrust | null;
 	allTrust(): ModelTrust[];
+	/**
+	 * Per-model mean time-to-first-token (ms), optionally scoped to a harness.
+	 * Null until `filters.latencyMinSamples` streamed samples exist. TTFT, not
+	 * total latency: it measures model+provider responsiveness independent of
+	 * how many tokens the answer happened to need.
+	 */
+	latency(slug: string, harnessId?: string): ModelLatency | null;
 	/** Observed chars-per-token ratio for a tokenizer family; null until calibrated. */
 	tokenRatio(tokenizer: string): number | null;
 	recentEntries(limit: number): LedgerEntry[];

package/src/router/candidates.ts

@@ -5,9 +5,9 @@
  */
 
 import type { CatalogModel, CatalogSnapshot } from "../catalog/types.ts";
-import type { QualityAxis, RouterConfig } from "../config/types.ts";
+import type { FilterConfig, QualityAxis, RouterConfig } from "../config/types.ts";
 import { forecast, priceAt } from "../cost/forecast.ts";
-import type { Ledger } from "../cost/types.ts";
+import type { Ledger, ModelLatency } from "../cost/types.ts";
 import type { NormRequest } from "../wire/types.ts";
 import { effectivePriceCeiling, effectiveQualityFloor, tierPlanFor } from "./tier-plan.ts";
 import type { Candidate, Features, Rejection, TaskType, Tier } from "./types.ts";
@@ -68,6 +68,22 @@ function resolveQuality(model: CatalogModel, axis: QualityAxis): { score: number
 /** Neutral trust prior for models our ledger has never observed. */
 const UNMEASURED_TRUST = 0.9;
 
+/** Excess-ratio cap so one very slow model cannot be penalised into oblivion. */
+const LATENCY_EXCESS_CAP = 3;
+
+/**
+ * Latency penalty as a multiplier on effective cost (>= 1; 1 = no penalty).
+ * Mean TTFT above `latencyReferenceMs` inflates the model's effective cost, the
+ * same lever trust uses for flakiness, so a faster model of equal quality and
+ * price outranks a sluggish one. Inert when the weight is 0 or the model has
+ * too few streamed samples to judge.
+ */
+function latencyMultiplier(latency: ModelLatency | null, filters: FilterConfig): number {
+	if (latency === null || filters.latencyWeight <= 0 || latency.samples < filters.latencyMinSamples) return 1;
+	const excess = Math.max(0, (latency.ttftMs - filters.latencyReferenceMs) / filters.latencyReferenceMs);
+	return 1 + filters.latencyWeight * Math.min(excess, LATENCY_EXCESS_CAP);
+}
+
 export function buildCandidates(args: BuildCandidatesArgs): { candidates: Candidate[]; rejected: Rejection[] } {
 	const { req, features, tier, task, snapshot, ledger, cfg, expectedCompletionTokens, warmSlug, relaxLevel = 0 } = args;
 	// A Set only when non-empty: the common path allocates nothing.
@@ -215,9 +231,15 @@ export function buildCandidates(args: BuildCandidatesArgs): { candidates: Candid
 		const trustScore = trust !== null && trust.attempts > 0 ? trust.successRate : UNMEASURED_TRUST;
 		const qualityScore = quality?.score ?? 0;
 		// Shared scoring: trust converts flakiness into money — a model failing
-		// 20% of the time really costs ~25% more in retries. qualityExponent 0
-		// makes this "cheapest above the floor"; the floor does the quality work.
-		const effectiveUsd = fc.expectedUsd / Math.max(trustScore, 0.5);
+		// 20% of the time really costs ~25% more in retries. Latency does the same
+		// for slowness (TTFT over the reference). qualityExponent 0 makes this
+		// "cheapest above the floor"; the floor does the quality work.
+		const latency =
+			filters.latencyWeight > 0
+				? (ledger?.latency(slug, filters.trustScopedByHarness ? req.harnessId : undefined) ?? null)
+				: null;
+		const latencyMult = latencyMultiplier(latency, filters);
+		const effectiveUsd = (fc.expectedUsd / Math.max(trustScore, 0.5)) * latencyMult;
 		const score = Math.pow(qualityScore / 100, tierCfg.qualityExponent) / Math.max(effectiveUsd, 1e-9);
 
 		const reasons: string[] = [
@@ -229,6 +251,9 @@ export function buildCandidates(args: BuildCandidatesArgs): { candidates: Candid
 				: `trust ${trustScore.toFixed(2)} over ${trust.attempts} attempts`,
 			`expected $${fc.expectedUsd.toFixed(6)}`,
 		];
+		if (latencyMult > 1 && latency !== null) {
+			reasons.push(`latency penalty ×${latencyMult.toFixed(2)} (ttft ${Math.round(latency.ttftMs)}ms over ${latency.samples} samples)`);
+		}
 		if (pinned) reasons.push("pinned into tier");
 		candidates.push({ model, forecast: fc, qualityScore, trustScore, score, reasons });
 	}

package/test/failover.test.ts

@@ -43,7 +43,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
 			data: { axis: "intelligence", minQuality: 0 },
 			chat: { axis: "intelligence", minQuality: 0 },
 		},
-		filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, minTrustSamples: 5, trustScopedByHarness: false, contextHeadroom: 1.2 },
+		filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, minTrustSamples: 5, trustScopedByHarness: false, contextHeadroom: 1.2, latencyWeight: 0, latencyReferenceMs: 5000, latencyMinSamples: 20 },
 		classifier: {
 			ambiguityThreshold: 0,
 			model: "test/adjudicator",
@@ -229,6 +229,7 @@ function mkLedger(): { ledger: Ledger; entries: LedgerEntry[] } {
 		conversationSpend: () => 0,
 		spendSince: () => 0,
 		blendedRate: () => null,
+		latency: () => null,
 		trust: () => null,
 		allTrust: () => [],
 		tokenRatio: () => null,

package/test/select.test.ts

@@ -257,6 +257,7 @@ describe("budget guard", () => {
 			conversationSpend: () => 0,
 			spendSince: (_sinceMs, harnessId) => (harnessId === undefined ? 1.0 : spendByHarness[harnessId] ?? 0),
 			blendedRate: () => null,
+		latency: () => null,
 			trust: () => null,
 			allTrust: () => [],
 			tokenRatio: () => null,
@@ -283,6 +284,7 @@ describe("per-harness trust scoping", () => {
 		conversationSpend: () => 0,
 		spendSince: () => 0,
 		blendedRate: () => null,
+		latency: () => null,
 		trust: (_slug, harnessId) => {
 			// Harness A has burned the model; harness B has never tried it.
 			if (harnessId === "harness-a") {
@@ -418,6 +420,7 @@ describe("tier rescue under a guardrail-constrained catalog", () => {
 			conversationSpend: () => 0,
 			spendSince: () => 0,
 			blendedRate: () => null,
+		latency: () => null,
 			trust: (slug) => ({
 				slug,
 				attempts: 40,
@@ -536,3 +539,46 @@ describe("task-type routing", () => {
 		}
 	});
 });
+
+describe("latency scoring", () => {
+	function ledgerWithLatency(ttftBySlug: Record<string, { ttftMs: number; samples: number }>): Ledger {
+		return {
+			record: () => {},
+			conversationSpend: () => 0,
+			spendSince: () => 0,
+			blendedRate: () => null,
+			trust: () => null,
+			allTrust: () => [],
+			latency: (slug) => {
+				const v = ttftBySlug[slug];
+				return v === undefined ? null : { slug, samples: v.samples, ttftMs: v.ttftMs };
+			},
+			tokenRatio: () => null,
+			recentEntries: () => [],
+		};
+	}
+
+	const withWeight = (latencyWeight: number): RouterConfig => ({
+		...BASE,
+		filters: { ...BASE.filters, latencyWeight, latencyReferenceMs: 5000, latencyMinSamples: 20 },
+	});
+
+	test("penalises a chronically slow model out of the top slot", () => {
+		const slow = run({ tier: "simple" }).slug;
+		const ledger = ledgerWithLatency({ [slow]: { ttftMs: 60_000, samples: 50 } });
+		const d = run({ tier: "simple", cfg: withWeight(2), ledger });
+		expect(d.slug).not.toBe(slow);
+	});
+
+	test("latencyWeight 0 disables the penalty", () => {
+		const slow = run({ tier: "simple" }).slug;
+		const ledger = ledgerWithLatency({ [slow]: { ttftMs: 60_000, samples: 50 } });
+		expect(run({ tier: "simple", cfg: withWeight(0), ledger }).slug).toBe(slow);
+	});
+
+	test("a model with too few samples is not penalised", () => {
+		const slow = run({ tier: "simple" }).slug;
+		const ledger = ledgerWithLatency({ [slow]: { ttftMs: 60_000, samples: 5 } });
+		expect(run({ tier: "simple", cfg: withWeight(2), ledger }).slug).toBe(slow);
+	});
+});

package/test/trust-attribution.test.ts

@@ -145,6 +145,39 @@ describe("trust attribution", () => {
 	});
 });
 
+describe("latency signal", () => {
+	function latencyOf(rows: Array<Partial<LedgerEntry>>): { samples: number; ttftMs: number } | null {
+		const db = openDb(":memory:");
+		try {
+			const ledger = createLedger(db, cfg);
+			for (const r of rows) ledger.record(entry(r));
+			const l = ledger.latency("vendor/model");
+			return l === null ? null : { samples: l.samples, ttftMs: l.ttftMs };
+		} finally {
+			db.close();
+		}
+	}
+
+	test("averages TTFT over streamed, non-errored turns", () => {
+		expect(latencyOf([{ ttftMs: 50 }, { ttftMs: 100 }, { ttftMs: 150 }])).toEqual({ samples: 3, ttftMs: 100 });
+	});
+
+	test("excludes errored, aborted, and non-streamed (null TTFT) rows", () => {
+		expect(
+			latencyOf([
+				{ ttftMs: 100 },
+				{ ttftMs: 9999, error: "upstream_error: boom" },
+				{ ttftMs: 9999, error: "request aborted" },
+				{ ttftMs: null },
+			]),
+		).toEqual({ samples: 1, ttftMs: 100 });
+	});
+
+	test("null when no streamed sample exists", () => {
+		expect(latencyOf([{ ttftMs: null }, { ttftMs: 0 }])).toBeNull();
+	});
+});
+
 describe("v4 migration", () => {
 	test("backfills error_kind from stored error text", () => {
 		const db = openDb(":memory:");

package/test/turn.test.ts

@@ -43,7 +43,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
 			data: { axis: "intelligence", minQuality: 0 },
 			chat: { axis: "intelligence", minQuality: 0 },
 		},
-		filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, minTrustSamples: 5, trustScopedByHarness: false, contextHeadroom: 1.2 },
+		filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, minTrustSamples: 5, trustScopedByHarness: false, contextHeadroom: 1.2, latencyWeight: 0, latencyReferenceMs: 5000, latencyMinSamples: 20 },
 		classifier: {
 			ambiguityThreshold: 0,
 			model: "test/adjudicator",
@@ -223,6 +223,7 @@ function mkLedger(): { ledger: Ledger; entries: LedgerEntry[] } {
 		conversationSpend: () => 0,
 		spendSince: () => 0,
 		blendedRate: () => null,
+		latency: () => null,
 		trust: () => null,
 		allTrust: () => [],
 		tokenRatio: () => null,