auto-model-router 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -7,14 +7,14 @@
7
7
  },
8
8
  "metadata": {
9
9
  "description": "auto-model-router: a local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
10
- "version": "0.2.0",
10
+ "version": "0.2.2",
11
11
  "pluginRoot": "."
12
12
  },
13
13
  "plugins": [
14
14
  {
15
15
  "name": "auto-model-router",
16
16
  "description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter. Runs in-process, routes per turn by price and task complexity, with budget caps, mid-stream escalation, and cache-aware hysteresis.",
17
- "version": "0.2.0",
17
+ "version": "0.2.2",
18
18
  "author": {
19
19
  "name": "drewappling",
20
20
  "email": "drewappling@gmail.com"
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "auto-model-router",
3
- "version": "0.2.0",
3
+ "version": "0.2.2",
4
4
  "private": false,
5
5
  "description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
6
6
  "type": "module",
@@ -68,6 +68,11 @@ export const DEFAULT_CONFIG: RouterConfig = {
68
68
  // even with a tiny guardrail-narrowed catalog.
69
69
  trustScopedByHarness: false,
70
70
  contextHeadroom: 1.25,
71
+ // Latency scoring is off by default (weight 0): opt in after establishing a
72
+ // baseline. TTFT above the reference inflates a model's effective cost.
73
+ latencyWeight: 0,
74
+ latencyReferenceMs: 5000,
75
+ latencyMinSamples: 20,
71
76
  },
72
77
  classifier: {
73
78
  ambiguityThreshold: 0.6,
@@ -65,6 +65,9 @@ const filters = z.strictObject({
65
65
  minTrustSamples: z.number().int().nonnegative().optional(),
66
66
  trustScopedByHarness: z.boolean().optional(),
67
67
  contextHeadroom: z.number().positive().optional(),
68
+ latencyWeight: z.number().nonnegative().optional(),
69
+ latencyReferenceMs: z.number().positive().optional(),
70
+ latencyMinSamples: z.number().int().nonnegative().optional(),
68
71
  });
69
72
 
70
73
  const classifier = z.strictObject({
@@ -138,6 +138,17 @@ export interface FilterConfig {
138
138
  * model's context window, absorbing token-estimate error and the response.
139
139
  */
140
140
  contextHeadroom: number;
141
+ /**
142
+ * How hard to penalise slow models in candidate scoring. A model's mean
143
+ * time-to-first-token above `latencyReferenceMs` inflates its effective
144
+ * cost — the same lever trust uses for flakiness — so a faster model of
145
+ * equal quality and price wins. 0 disables latency scoring entirely.
146
+ */
147
+ latencyWeight: number;
148
+ /** TTFT (ms) below which a model is considered fully responsive (no penalty). */
149
+ latencyReferenceMs: number;
150
+ /** Streamed samples required before latency is scored against a model. */
151
+ latencyMinSamples: number;
141
152
  }
142
153
 
143
154
  export interface ClassifierConfig {
@@ -18,7 +18,7 @@ import type { RouterConfig } from "../config/types.ts";
18
18
  import { consumePendingEstimate } from "../tokens/estimate.ts";
19
19
  import { computeBlendedRate } from "./blended.ts";
20
20
  import { computeCost } from "./forecast.ts";
21
- import type { BlendedRate, Ledger, LedgerEntry, ModelTrust, UsageCounts } from "./types.ts";
21
+ import type { BlendedRate, Ledger, LedgerEntry, ModelLatency, ModelTrust, UsageCounts } from "./types.ts";
22
22
 
23
23
  /** Estimates below this many samples are noise; the default ratio is better. */
24
24
  const MIN_CALIBRATION_SAMPLES = 20;
@@ -67,6 +67,11 @@ interface TrustRow {
67
67
  mean_cost_error: number | null;
68
68
  }
69
69
 
70
+ interface LatencyRow {
71
+ samples: number;
72
+ ttft_ms: number | null;
73
+ }
74
+
70
75
  interface CalibrationRow {
71
76
  est_bytes: number;
72
77
  actual_tokens: number;
@@ -98,6 +103,16 @@ const TRUST_SELECT = `COUNT(*) AS attempts,
98
103
  AVG(CASE WHEN reported_usd IS NOT NULL AND reported_usd > 0
99
104
  THEN ABS(reported_usd - predicted_usd) / reported_usd END) AS mean_cost_error`;
100
105
 
106
+ /**
107
+ * Time-to-first-token, averaged over streamed, non-errored turns. TTFT (not
108
+ * total latency) isolates model+provider responsiveness from answer length: a
109
+ * model is "slow" when it takes a long time to START, not when it was asked for
110
+ * a long answer. Errored/aborted rows and non-streaming rows (null ttft) are
111
+ * excluded — they carry no responsiveness signal.
112
+ */
113
+ const LATENCY_SELECT = `COUNT(CASE WHEN ttft_ms IS NOT NULL AND ttft_ms > 0 AND error IS NULL THEN 1 END) AS samples,
114
+ AVG(CASE WHEN ttft_ms IS NOT NULL AND ttft_ms > 0 AND error IS NULL THEN ttft_ms END) AS ttft_ms`;
115
+
101
116
  /**
102
117
  * Recovers the `UpstreamErrorKind` from the text turn.ts stored.
103
118
  *
@@ -127,6 +142,11 @@ function toTrust(slug: string, row: TrustRow): ModelTrust {
127
142
  };
128
143
  }
129
144
 
145
+ function toLatency(slug: string, row: LatencyRow): ModelLatency | null {
146
+ if (row.samples <= 0 || row.ttft_ms === null) return null;
147
+ return { slug, samples: row.samples, ttftMs: row.ttft_ms };
148
+ }
149
+
130
150
  function toEntry(row: LedgerRow): LedgerEntry {
131
151
  return {
132
152
  id: row.id,
@@ -192,6 +212,8 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
192
212
  const trustStmt = db.query(`SELECT ${TRUST_SELECT} FROM ledger WHERE slug = ?`);
193
213
  const trustHarnessStmt = db.query(`SELECT ${TRUST_SELECT} FROM ledger WHERE slug = ? AND harness_id = ?`);
194
214
  const allTrustStmt = db.query(`SELECT slug, ${TRUST_SELECT} FROM ledger GROUP BY slug`);
215
+ const latencyStmt = db.query(`SELECT ${LATENCY_SELECT} FROM ledger WHERE slug = ?`);
216
+ const latencyHarnessStmt = db.query(`SELECT ${LATENCY_SELECT} FROM ledger WHERE slug = ? AND harness_id = ?`);
195
217
  const ratioStmt = db.query("SELECT est_bytes, actual_tokens, samples FROM token_calibration WHERE tokenizer = ?");
196
218
  const recentStmt = db.query("SELECT * FROM ledger ORDER BY created_at_ms DESC LIMIT ?");
197
219
  const cacheMetaStmt = db.query("SELECT fetched_at_ms FROM catalog_cache WHERE id = 1");
@@ -301,6 +323,15 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
301
323
  return rows.map((row) => toTrust(row.slug, row));
302
324
  },
303
325
 
326
+ latency(slug: string, harnessId?: string): ModelLatency | null {
327
+ const row =
328
+ harnessId !== undefined && harnessId !== ""
329
+ ? (latencyHarnessStmt.get(slug, harnessId) as LatencyRow | null)
330
+ : (latencyStmt.get(slug) as LatencyRow | null);
331
+ if (row === null) return null;
332
+ return toLatency(slug, row);
333
+ },
334
+
304
335
  tokenRatio(tokenizer: string): number | null {
305
336
  const row = ratioStmt.get(tokenizer.trim().toLowerCase()) as CalibrationRow | null;
306
337
  if (row === null || row.samples < MIN_CALIBRATION_SAMPLES || row.actual_tokens <= 0) return null;
package/src/cost/types.ts CHANGED
@@ -159,6 +159,14 @@ export interface ModelTrust {
159
159
  meanCostError: number;
160
160
  }
161
161
 
162
+ /** Per-model responsiveness learned from our own traffic. Feeds candidate scoring. */
163
+ export interface ModelLatency {
164
+ slug: string;
165
+ samples: number;
166
+ /** Mean time-to-first-token, ms, over streamed non-errored turns. */
167
+ ttftMs: number;
168
+ }
169
+
162
170
  export interface Ledger {
163
171
  record(entry: LedgerEntry): void;
164
172
  /** Total reported (or predicted, when reported is null) spend for a conversation. */
@@ -172,6 +180,13 @@ export interface Ledger {
172
180
  /** Per-model reliability over the ledger, optionally scoped to a harness. */
173
181
  trust(slug: string, harnessId?: string): ModelTrust | null;
174
182
  allTrust(): ModelTrust[];
183
+ /**
184
+ * Per-model mean time-to-first-token (ms), optionally scoped to a harness.
185
+ * Null until `filters.latencyMinSamples` streamed samples exist. TTFT, not
186
+ * total latency: it measures model+provider responsiveness independent of
187
+ * how many tokens the answer happened to need.
188
+ */
189
+ latency(slug: string, harnessId?: string): ModelLatency | null;
175
190
  /** Observed chars-per-token ratio for a tokenizer family; null until calibrated. */
176
191
  tokenRatio(tokenizer: string): number | null;
177
192
  recentEntries(limit: number): LedgerEntry[];
@@ -5,9 +5,9 @@
5
5
  */
6
6
 
7
7
  import type { CatalogModel, CatalogSnapshot } from "../catalog/types.ts";
8
- import type { QualityAxis, RouterConfig } from "../config/types.ts";
8
+ import type { FilterConfig, QualityAxis, RouterConfig } from "../config/types.ts";
9
9
  import { forecast, priceAt } from "../cost/forecast.ts";
10
- import type { Ledger } from "../cost/types.ts";
10
+ import type { Ledger, ModelLatency } from "../cost/types.ts";
11
11
  import type { NormRequest } from "../wire/types.ts";
12
12
  import { effectivePriceCeiling, effectiveQualityFloor, tierPlanFor } from "./tier-plan.ts";
13
13
  import type { Candidate, Features, Rejection, TaskType, Tier } from "./types.ts";
@@ -68,6 +68,22 @@ function resolveQuality(model: CatalogModel, axis: QualityAxis): { score: number
68
68
  /** Neutral trust prior for models our ledger has never observed. */
69
69
  const UNMEASURED_TRUST = 0.9;
70
70
 
71
+ /** Excess-ratio cap so one very slow model cannot be penalised into oblivion. */
72
+ const LATENCY_EXCESS_CAP = 3;
73
+
74
+ /**
75
+ * Latency penalty as a multiplier on effective cost (>= 1; 1 = no penalty).
76
+ * Mean TTFT above `latencyReferenceMs` inflates the model's effective cost, the
77
+ * same lever trust uses for flakiness, so a faster model of equal quality and
78
+ * price outranks a sluggish one. Inert when the weight is 0 or the model has
79
+ * too few streamed samples to judge.
80
+ */
81
+ function latencyMultiplier(latency: ModelLatency | null, filters: FilterConfig): number {
82
+ if (latency === null || filters.latencyWeight <= 0 || latency.samples < filters.latencyMinSamples) return 1;
83
+ const excess = Math.max(0, (latency.ttftMs - filters.latencyReferenceMs) / filters.latencyReferenceMs);
84
+ return 1 + filters.latencyWeight * Math.min(excess, LATENCY_EXCESS_CAP);
85
+ }
86
+
71
87
  export function buildCandidates(args: BuildCandidatesArgs): { candidates: Candidate[]; rejected: Rejection[] } {
72
88
  const { req, features, tier, task, snapshot, ledger, cfg, expectedCompletionTokens, warmSlug, relaxLevel = 0 } = args;
73
89
  // A Set only when non-empty: the common path allocates nothing.
@@ -215,9 +231,15 @@ export function buildCandidates(args: BuildCandidatesArgs): { candidates: Candid
215
231
  const trustScore = trust !== null && trust.attempts > 0 ? trust.successRate : UNMEASURED_TRUST;
216
232
  const qualityScore = quality?.score ?? 0;
217
233
  // Shared scoring: trust converts flakiness into money — a model failing
218
- // 20% of the time really costs ~25% more in retries. qualityExponent 0
219
- // makes this "cheapest above the floor"; the floor does the quality work.
220
- const effectiveUsd = fc.expectedUsd / Math.max(trustScore, 0.5);
234
+ // 20% of the time really costs ~25% more in retries. Latency does the same
235
+ // for slowness (TTFT over the reference). qualityExponent 0 makes this
236
+ // "cheapest above the floor"; the floor does the quality work.
237
+ const latency =
238
+ filters.latencyWeight > 0
239
+ ? (ledger?.latency(slug, filters.trustScopedByHarness ? req.harnessId : undefined) ?? null)
240
+ : null;
241
+ const latencyMult = latencyMultiplier(latency, filters);
242
+ const effectiveUsd = (fc.expectedUsd / Math.max(trustScore, 0.5)) * latencyMult;
221
243
  const score = Math.pow(qualityScore / 100, tierCfg.qualityExponent) / Math.max(effectiveUsd, 1e-9);
222
244
 
223
245
  const reasons: string[] = [
@@ -229,6 +251,9 @@ export function buildCandidates(args: BuildCandidatesArgs): { candidates: Candid
229
251
  : `trust ${trustScore.toFixed(2)} over ${trust.attempts} attempts`,
230
252
  `expected $${fc.expectedUsd.toFixed(6)}`,
231
253
  ];
254
+ if (latencyMult > 1 && latency !== null) {
255
+ reasons.push(`latency penalty ×${latencyMult.toFixed(2)} (ttft ${Math.round(latency.ttftMs)}ms over ${latency.samples} samples)`);
256
+ }
232
257
  if (pinned) reasons.push("pinned into tier");
233
258
  candidates.push({ model, forecast: fc, qualityScore, trustScore, score, reasons });
234
259
  }
@@ -43,9 +43,18 @@ const W_LARGE_CONTENT = 0.05;
43
43
  const W_HUGE_CONTENT = 0.1;
44
44
  const W_TURN_DEPTH = 0.004; // long conversations accumulate entangled context
45
45
  const CAP_TURN_DEPTH = 0.08;
46
- const W_LOOP_DEPTH = -0.008; // deep loops are mechanical
46
+ const W_LOOP_DEPTH = -0.008; // a single deep step is mechanical
47
47
  const CAP_LOOP_DEPTH = -0.06;
48
- const W_AUTONOMOUS_LOOP = 0.06; // but a loop this long implies a substantial underlying task
48
+ // A sustained autonomous loop is the signal the underlying task is substantial:
49
+ // the agent keeps grinding without human input. Unlike the mechanical-step
50
+ // penalty above, this term ACCUMULATES with depth past the agentic threshold so
51
+ // long/hard loops climb out of trivial instead of scoring like a one-line edit.
52
+ // Capped so pure depth tops out in `simple` (a competent-but-cheap model);
53
+ // reaching `moderate`/`hard` still requires real complexity signals (tool
54
+ // failure, repeated call, keywords) to stack on top.
55
+ const W_AUTONOMOUS_LOOP = 0.06; // base bonus at the threshold
56
+ const W_AUTONOMOUS_LOOP_PER_DEPTH = 0.018; // added per loop step beyond the threshold
57
+ const CAP_AUTONOMOUS_LOOP = 0.34;
49
58
  const W_IMAGES = 0.04;
50
59
  const W_TOOLS_OFFERED = 0.03;
51
60
 
@@ -100,7 +109,11 @@ export function scoreHeuristic(f: Features, cfg: RouterConfig): Classification {
100
109
  add(Math.min(f.turnDepth * W_TURN_DEPTH, CAP_TURN_DEPTH), `conversation depth ${f.turnDepth}`);
101
110
  add(Math.max(f.toolLoopDepth * W_LOOP_DEPTH, CAP_LOOP_DEPTH), `tool loop depth ${f.toolLoopDepth}`);
102
111
  if (f.toolLoopDepth >= cfg.classifier.agenticLoopDepth && f.toolLoopDepth > 0) {
103
- add(W_AUTONOMOUS_LOOP, "long autonomous loop implies a substantial task");
112
+ const excessDepth = f.toolLoopDepth - cfg.classifier.agenticLoopDepth;
113
+ add(
114
+ Math.min(W_AUTONOMOUS_LOOP + excessDepth * W_AUTONOMOUS_LOOP_PER_DEPTH, CAP_AUTONOMOUS_LOOP),
115
+ `autonomous loop depth ${f.toolLoopDepth} (sustained task)`,
116
+ );
104
117
  }
105
118
  if (f.hasImages) add(W_IMAGES, "image input");
106
119
  if (f.toolCount > 0) add(W_TOOLS_OFFERED, `${f.toolCount} tools offered`);
@@ -37,6 +37,32 @@ function featuresFor(messages: unknown[], tools?: unknown[] | undefined): Featur
37
37
  return extractFeatures(r, 5000);
38
38
  }
39
39
 
40
+ /** A tool-result continuation at a given autonomous-loop depth, no other signals. */
41
+ function contFeatures(toolLoopDepth: number, over: Partial<Features> = {}): Features {
42
+ return {
43
+ promptTokens: 30000,
44
+ newContentTokens: 200,
45
+ turnDepth: toolLoopDepth,
46
+ toolCount: 12,
47
+ toolSchemaBytes: 9783,
48
+ isToolResultContinuation: true,
49
+ toolLoopDepth,
50
+ distinctToolsUsed: 3,
51
+ lastToolFailed: false,
52
+ repeatedToolCall: false,
53
+ hasImages: false,
54
+ codeBlocks: 0,
55
+ codeBytes: 0,
56
+ looksLikeDiff: false,
57
+ complexityKeywords: [],
58
+ trivialityKeywords: [],
59
+ requestedReasoning: undefined,
60
+ questionCount: 0,
61
+ isTerseInstruction: false,
62
+ ...over,
63
+ };
64
+ }
65
+
40
66
  const SYSTEM = { role: "system", content: "You are a coding agent." };
41
67
 
42
68
  /** Upstream double that fails loudly if the adjudicator is consulted. */
@@ -160,6 +186,39 @@ describe("scoreHeuristic", () => {
160
186
  expect(c.confidence).toBeGreaterThanOrEqual(0);
161
187
  expect(c.confidence).toBeLessThanOrEqual(1);
162
188
  });
189
+
190
+ test("a shallow tool-result continuation stays trivial", () => {
191
+ const shallow = scoreHeuristic(contFeatures(2), BASE);
192
+ expect(shallow.tier).toBe("trivial");
193
+ });
194
+
195
+ test("a sustained autonomous loop climbs out of trivial", () => {
196
+ // The failure mode this fixes: a long coding loop pinned to the cheapest
197
+ // tier for dozens of turns because agentic complexity never accumulated.
198
+ const shallow = scoreHeuristic(contFeatures(2), BASE);
199
+ const deep = scoreHeuristic(contFeatures(20), BASE);
200
+ expect(tierIdx(deep.tier)).toBeGreaterThan(tierIdx(shallow.tier));
201
+ expect(deep.tier).not.toBe("trivial");
202
+ });
203
+
204
+ test("score increases monotonically with loop depth past the agentic threshold", () => {
205
+ const depths = [4, 6, 8, 10, 15, 20, 30];
206
+ let prev = -1;
207
+ for (const d of depths) {
208
+ const s = scoreHeuristic(contFeatures(d), BASE).score;
209
+ expect(s).toBeGreaterThanOrEqual(prev);
210
+ prev = s;
211
+ }
212
+ });
213
+
214
+ test("pure loop depth tops out at simple; complexity signals stack to moderate", () => {
215
+ // Depth alone means competent-but-cheap (simple), not a frontier model.
216
+ const veryDeep = scoreHeuristic(contFeatures(90), BASE);
217
+ expect(tierIdx(veryDeep.tier)).toBeLessThanOrEqual(tierIdx("simple"));
218
+ // A failing tool result on top of a deep loop is genuinely hard.
219
+ const deepAndFailing = scoreHeuristic(contFeatures(20, { lastToolFailed: true }), BASE);
220
+ expect(tierIdx(deepAndFailing.tier)).toBeGreaterThanOrEqual(tierIdx("moderate"));
221
+ });
163
222
  });
164
223
 
165
224
  describe("pickQualityAxis", () => {
@@ -43,7 +43,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
43
43
  data: { axis: "intelligence", minQuality: 0 },
44
44
  chat: { axis: "intelligence", minQuality: 0 },
45
45
  },
46
- filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, minTrustSamples: 5, trustScopedByHarness: false, contextHeadroom: 1.2 },
46
+ filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, minTrustSamples: 5, trustScopedByHarness: false, contextHeadroom: 1.2, latencyWeight: 0, latencyReferenceMs: 5000, latencyMinSamples: 20 },
47
47
  classifier: {
48
48
  ambiguityThreshold: 0,
49
49
  model: "test/adjudicator",
@@ -229,6 +229,7 @@ function mkLedger(): { ledger: Ledger; entries: LedgerEntry[] } {
229
229
  conversationSpend: () => 0,
230
230
  spendSince: () => 0,
231
231
  blendedRate: () => null,
232
+ latency: () => null,
232
233
  trust: () => null,
233
234
  allTrust: () => [],
234
235
  tokenRatio: () => null,
@@ -257,6 +257,7 @@ describe("budget guard", () => {
257
257
  conversationSpend: () => 0,
258
258
  spendSince: (_sinceMs, harnessId) => (harnessId === undefined ? 1.0 : spendByHarness[harnessId] ?? 0),
259
259
  blendedRate: () => null,
260
+ latency: () => null,
260
261
  trust: () => null,
261
262
  allTrust: () => [],
262
263
  tokenRatio: () => null,
@@ -283,6 +284,7 @@ describe("per-harness trust scoping", () => {
283
284
  conversationSpend: () => 0,
284
285
  spendSince: () => 0,
285
286
  blendedRate: () => null,
287
+ latency: () => null,
286
288
  trust: (_slug, harnessId) => {
287
289
  // Harness A has burned the model; harness B has never tried it.
288
290
  if (harnessId === "harness-a") {
@@ -418,6 +420,7 @@ describe("tier rescue under a guardrail-constrained catalog", () => {
418
420
  conversationSpend: () => 0,
419
421
  spendSince: () => 0,
420
422
  blendedRate: () => null,
423
+ latency: () => null,
421
424
  trust: (slug) => ({
422
425
  slug,
423
426
  attempts: 40,
@@ -536,3 +539,46 @@ describe("task-type routing", () => {
536
539
  }
537
540
  });
538
541
  });
542
+
543
+ describe("latency scoring", () => {
544
+ function ledgerWithLatency(ttftBySlug: Record<string, { ttftMs: number; samples: number }>): Ledger {
545
+ return {
546
+ record: () => {},
547
+ conversationSpend: () => 0,
548
+ spendSince: () => 0,
549
+ blendedRate: () => null,
550
+ trust: () => null,
551
+ allTrust: () => [],
552
+ latency: (slug) => {
553
+ const v = ttftBySlug[slug];
554
+ return v === undefined ? null : { slug, samples: v.samples, ttftMs: v.ttftMs };
555
+ },
556
+ tokenRatio: () => null,
557
+ recentEntries: () => [],
558
+ };
559
+ }
560
+
561
+ const withWeight = (latencyWeight: number): RouterConfig => ({
562
+ ...BASE,
563
+ filters: { ...BASE.filters, latencyWeight, latencyReferenceMs: 5000, latencyMinSamples: 20 },
564
+ });
565
+
566
+ test("penalises a chronically slow model out of the top slot", () => {
567
+ const slow = run({ tier: "simple" }).slug;
568
+ const ledger = ledgerWithLatency({ [slow]: { ttftMs: 60_000, samples: 50 } });
569
+ const d = run({ tier: "simple", cfg: withWeight(2), ledger });
570
+ expect(d.slug).not.toBe(slow);
571
+ });
572
+
573
+ test("latencyWeight 0 disables the penalty", () => {
574
+ const slow = run({ tier: "simple" }).slug;
575
+ const ledger = ledgerWithLatency({ [slow]: { ttftMs: 60_000, samples: 50 } });
576
+ expect(run({ tier: "simple", cfg: withWeight(0), ledger }).slug).toBe(slow);
577
+ });
578
+
579
+ test("a model with too few samples is not penalised", () => {
580
+ const slow = run({ tier: "simple" }).slug;
581
+ const ledger = ledgerWithLatency({ [slow]: { ttftMs: 60_000, samples: 5 } });
582
+ expect(run({ tier: "simple", cfg: withWeight(2), ledger }).slug).toBe(slow);
583
+ });
584
+ });
@@ -145,6 +145,39 @@ describe("trust attribution", () => {
145
145
  });
146
146
  });
147
147
 
148
+ describe("latency signal", () => {
149
+ function latencyOf(rows: Array<Partial<LedgerEntry>>): { samples: number; ttftMs: number } | null {
150
+ const db = openDb(":memory:");
151
+ try {
152
+ const ledger = createLedger(db, cfg);
153
+ for (const r of rows) ledger.record(entry(r));
154
+ const l = ledger.latency("vendor/model");
155
+ return l === null ? null : { samples: l.samples, ttftMs: l.ttftMs };
156
+ } finally {
157
+ db.close();
158
+ }
159
+ }
160
+
161
+ test("averages TTFT over streamed, non-errored turns", () => {
162
+ expect(latencyOf([{ ttftMs: 50 }, { ttftMs: 100 }, { ttftMs: 150 }])).toEqual({ samples: 3, ttftMs: 100 });
163
+ });
164
+
165
+ test("excludes errored, aborted, and non-streamed (null TTFT) rows", () => {
166
+ expect(
167
+ latencyOf([
168
+ { ttftMs: 100 },
169
+ { ttftMs: 9999, error: "upstream_error: boom" },
170
+ { ttftMs: 9999, error: "request aborted" },
171
+ { ttftMs: null },
172
+ ]),
173
+ ).toEqual({ samples: 1, ttftMs: 100 });
174
+ });
175
+
176
+ test("null when no streamed sample exists", () => {
177
+ expect(latencyOf([{ ttftMs: null }, { ttftMs: 0 }])).toBeNull();
178
+ });
179
+ });
180
+
148
181
  describe("v4 migration", () => {
149
182
  test("backfills error_kind from stored error text", () => {
150
183
  const db = openDb(":memory:");
package/test/turn.test.ts CHANGED
@@ -43,7 +43,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
43
43
  data: { axis: "intelligence", minQuality: 0 },
44
44
  chat: { axis: "intelligence", minQuality: 0 },
45
45
  },
46
- filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, minTrustSamples: 5, trustScopedByHarness: false, contextHeadroom: 1.2 },
46
+ filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, minTrustSamples: 5, trustScopedByHarness: false, contextHeadroom: 1.2, latencyWeight: 0, latencyReferenceMs: 5000, latencyMinSamples: 20 },
47
47
  classifier: {
48
48
  ambiguityThreshold: 0,
49
49
  model: "test/adjudicator",
@@ -223,6 +223,7 @@ function mkLedger(): { ledger: Ledger; entries: LedgerEntry[] } {
223
223
  conversationSpend: () => 0,
224
224
  spendSince: () => 0,
225
225
  blendedRate: () => null,
226
+ latency: () => null,
226
227
  trust: () => null,
227
228
  allTrust: () => [],
228
229
  tokenRatio: () => null,