auto-model-router 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.omp-plugin/marketplace.json +2 -2
- package/package.json +1 -1
- package/src/config/defaults.ts +5 -0
- package/src/config/schema.ts +3 -0
- package/src/config/types.ts +11 -0
- package/src/cost/ledger.ts +32 -1
- package/src/cost/types.ts +15 -0
- package/src/router/candidates.ts +30 -5
- package/src/router/classify.ts +16 -3
- package/test/classify.test.ts +59 -0
- package/test/failover.test.ts +2 -1
- package/test/select.test.ts +46 -0
- package/test/trust-attribution.test.ts +33 -0
- package/test/turn.test.ts +2 -1
|
@@ -7,14 +7,14 @@
|
|
|
7
7
|
},
|
|
8
8
|
"metadata": {
|
|
9
9
|
"description": "auto-model-router: a local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
|
|
10
|
-
"version": "0.2.
|
|
10
|
+
"version": "0.2.2",
|
|
11
11
|
"pluginRoot": "."
|
|
12
12
|
},
|
|
13
13
|
"plugins": [
|
|
14
14
|
{
|
|
15
15
|
"name": "auto-model-router",
|
|
16
16
|
"description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter. Runs in-process, routes per turn by price and task complexity, with budget caps, mid-stream escalation, and cache-aware hysteresis.",
|
|
17
|
-
"version": "0.2.
|
|
17
|
+
"version": "0.2.2",
|
|
18
18
|
"author": {
|
|
19
19
|
"name": "drewappling",
|
|
20
20
|
"email": "drewappling@gmail.com"
|
package/package.json
CHANGED
package/src/config/defaults.ts
CHANGED
|
@@ -68,6 +68,11 @@ export const DEFAULT_CONFIG: RouterConfig = {
|
|
|
68
68
|
// even with a tiny guardrail-narrowed catalog.
|
|
69
69
|
trustScopedByHarness: false,
|
|
70
70
|
contextHeadroom: 1.25,
|
|
71
|
+
// Latency scoring is off by default (weight 0): opt in after establishing a
|
|
72
|
+
// baseline. TTFT above the reference inflates a model's effective cost.
|
|
73
|
+
latencyWeight: 0,
|
|
74
|
+
latencyReferenceMs: 5000,
|
|
75
|
+
latencyMinSamples: 20,
|
|
71
76
|
},
|
|
72
77
|
classifier: {
|
|
73
78
|
ambiguityThreshold: 0.6,
|
package/src/config/schema.ts
CHANGED
|
@@ -65,6 +65,9 @@ const filters = z.strictObject({
|
|
|
65
65
|
minTrustSamples: z.number().int().nonnegative().optional(),
|
|
66
66
|
trustScopedByHarness: z.boolean().optional(),
|
|
67
67
|
contextHeadroom: z.number().positive().optional(),
|
|
68
|
+
latencyWeight: z.number().nonnegative().optional(),
|
|
69
|
+
latencyReferenceMs: z.number().positive().optional(),
|
|
70
|
+
latencyMinSamples: z.number().int().nonnegative().optional(),
|
|
68
71
|
});
|
|
69
72
|
|
|
70
73
|
const classifier = z.strictObject({
|
package/src/config/types.ts
CHANGED
|
@@ -138,6 +138,17 @@ export interface FilterConfig {
|
|
|
138
138
|
* model's context window, absorbing token-estimate error and the response.
|
|
139
139
|
*/
|
|
140
140
|
contextHeadroom: number;
|
|
141
|
+
/**
|
|
142
|
+
* How hard to penalise slow models in candidate scoring. A model's mean
|
|
143
|
+
* time-to-first-token above `latencyReferenceMs` inflates its effective
|
|
144
|
+
* cost — the same lever trust uses for flakiness — so a faster model of
|
|
145
|
+
* equal quality and price wins. 0 disables latency scoring entirely.
|
|
146
|
+
*/
|
|
147
|
+
latencyWeight: number;
|
|
148
|
+
/** TTFT (ms) below which a model is considered fully responsive (no penalty). */
|
|
149
|
+
latencyReferenceMs: number;
|
|
150
|
+
/** Streamed samples required before latency is scored against a model. */
|
|
151
|
+
latencyMinSamples: number;
|
|
141
152
|
}
|
|
142
153
|
|
|
143
154
|
export interface ClassifierConfig {
|
package/src/cost/ledger.ts
CHANGED
|
@@ -18,7 +18,7 @@ import type { RouterConfig } from "../config/types.ts";
|
|
|
18
18
|
import { consumePendingEstimate } from "../tokens/estimate.ts";
|
|
19
19
|
import { computeBlendedRate } from "./blended.ts";
|
|
20
20
|
import { computeCost } from "./forecast.ts";
|
|
21
|
-
import type { BlendedRate, Ledger, LedgerEntry, ModelTrust, UsageCounts } from "./types.ts";
|
|
21
|
+
import type { BlendedRate, Ledger, LedgerEntry, ModelLatency, ModelTrust, UsageCounts } from "./types.ts";
|
|
22
22
|
|
|
23
23
|
/** Estimates below this many samples are noise; the default ratio is better. */
|
|
24
24
|
const MIN_CALIBRATION_SAMPLES = 20;
|
|
@@ -67,6 +67,11 @@ interface TrustRow {
|
|
|
67
67
|
mean_cost_error: number | null;
|
|
68
68
|
}
|
|
69
69
|
|
|
70
|
+
interface LatencyRow {
|
|
71
|
+
samples: number;
|
|
72
|
+
ttft_ms: number | null;
|
|
73
|
+
}
|
|
74
|
+
|
|
70
75
|
interface CalibrationRow {
|
|
71
76
|
est_bytes: number;
|
|
72
77
|
actual_tokens: number;
|
|
@@ -98,6 +103,16 @@ const TRUST_SELECT = `COUNT(*) AS attempts,
|
|
|
98
103
|
AVG(CASE WHEN reported_usd IS NOT NULL AND reported_usd > 0
|
|
99
104
|
THEN ABS(reported_usd - predicted_usd) / reported_usd END) AS mean_cost_error`;
|
|
100
105
|
|
|
106
|
+
/**
|
|
107
|
+
* Time-to-first-token, averaged over streamed, non-errored turns. TTFT (not
|
|
108
|
+
* total latency) isolates model+provider responsiveness from answer length: a
|
|
109
|
+
* model is "slow" when it takes a long time to START, not when it was asked for
|
|
110
|
+
* a long answer. Errored/aborted rows and non-streaming rows (null ttft) are
|
|
111
|
+
* excluded — they carry no responsiveness signal.
|
|
112
|
+
*/
|
|
113
|
+
const LATENCY_SELECT = `COUNT(CASE WHEN ttft_ms IS NOT NULL AND ttft_ms > 0 AND error IS NULL THEN 1 END) AS samples,
|
|
114
|
+
AVG(CASE WHEN ttft_ms IS NOT NULL AND ttft_ms > 0 AND error IS NULL THEN ttft_ms END) AS ttft_ms`;
|
|
115
|
+
|
|
101
116
|
/**
|
|
102
117
|
* Recovers the `UpstreamErrorKind` from the text turn.ts stored.
|
|
103
118
|
*
|
|
@@ -127,6 +142,11 @@ function toTrust(slug: string, row: TrustRow): ModelTrust {
|
|
|
127
142
|
};
|
|
128
143
|
}
|
|
129
144
|
|
|
145
|
+
function toLatency(slug: string, row: LatencyRow): ModelLatency | null {
|
|
146
|
+
if (row.samples <= 0 || row.ttft_ms === null) return null;
|
|
147
|
+
return { slug, samples: row.samples, ttftMs: row.ttft_ms };
|
|
148
|
+
}
|
|
149
|
+
|
|
130
150
|
function toEntry(row: LedgerRow): LedgerEntry {
|
|
131
151
|
return {
|
|
132
152
|
id: row.id,
|
|
@@ -192,6 +212,8 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
|
|
|
192
212
|
const trustStmt = db.query(`SELECT ${TRUST_SELECT} FROM ledger WHERE slug = ?`);
|
|
193
213
|
const trustHarnessStmt = db.query(`SELECT ${TRUST_SELECT} FROM ledger WHERE slug = ? AND harness_id = ?`);
|
|
194
214
|
const allTrustStmt = db.query(`SELECT slug, ${TRUST_SELECT} FROM ledger GROUP BY slug`);
|
|
215
|
+
const latencyStmt = db.query(`SELECT ${LATENCY_SELECT} FROM ledger WHERE slug = ?`);
|
|
216
|
+
const latencyHarnessStmt = db.query(`SELECT ${LATENCY_SELECT} FROM ledger WHERE slug = ? AND harness_id = ?`);
|
|
195
217
|
const ratioStmt = db.query("SELECT est_bytes, actual_tokens, samples FROM token_calibration WHERE tokenizer = ?");
|
|
196
218
|
const recentStmt = db.query("SELECT * FROM ledger ORDER BY created_at_ms DESC LIMIT ?");
|
|
197
219
|
const cacheMetaStmt = db.query("SELECT fetched_at_ms FROM catalog_cache WHERE id = 1");
|
|
@@ -301,6 +323,15 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
|
|
|
301
323
|
return rows.map((row) => toTrust(row.slug, row));
|
|
302
324
|
},
|
|
303
325
|
|
|
326
|
+
latency(slug: string, harnessId?: string): ModelLatency | null {
|
|
327
|
+
const row =
|
|
328
|
+
harnessId !== undefined && harnessId !== ""
|
|
329
|
+
? (latencyHarnessStmt.get(slug, harnessId) as LatencyRow | null)
|
|
330
|
+
: (latencyStmt.get(slug) as LatencyRow | null);
|
|
331
|
+
if (row === null) return null;
|
|
332
|
+
return toLatency(slug, row);
|
|
333
|
+
},
|
|
334
|
+
|
|
304
335
|
tokenRatio(tokenizer: string): number | null {
|
|
305
336
|
const row = ratioStmt.get(tokenizer.trim().toLowerCase()) as CalibrationRow | null;
|
|
306
337
|
if (row === null || row.samples < MIN_CALIBRATION_SAMPLES || row.actual_tokens <= 0) return null;
|
package/src/cost/types.ts
CHANGED
|
@@ -159,6 +159,14 @@ export interface ModelTrust {
|
|
|
159
159
|
meanCostError: number;
|
|
160
160
|
}
|
|
161
161
|
|
|
162
|
+
/** Per-model responsiveness learned from our own traffic. Feeds candidate scoring. */
|
|
163
|
+
export interface ModelLatency {
|
|
164
|
+
slug: string;
|
|
165
|
+
samples: number;
|
|
166
|
+
/** Mean time-to-first-token, ms, over streamed non-errored turns. */
|
|
167
|
+
ttftMs: number;
|
|
168
|
+
}
|
|
169
|
+
|
|
162
170
|
export interface Ledger {
|
|
163
171
|
record(entry: LedgerEntry): void;
|
|
164
172
|
/** Total reported (or predicted, when reported is null) spend for a conversation. */
|
|
@@ -172,6 +180,13 @@ export interface Ledger {
|
|
|
172
180
|
/** Per-model reliability over the ledger, optionally scoped to a harness. */
|
|
173
181
|
trust(slug: string, harnessId?: string): ModelTrust | null;
|
|
174
182
|
allTrust(): ModelTrust[];
|
|
183
|
+
/**
|
|
184
|
+
* Per-model mean time-to-first-token (ms), optionally scoped to a harness.
|
|
185
|
+
* Null until `filters.latencyMinSamples` streamed samples exist. TTFT, not
|
|
186
|
+
* total latency: it measures model+provider responsiveness independent of
|
|
187
|
+
* how many tokens the answer happened to need.
|
|
188
|
+
*/
|
|
189
|
+
latency(slug: string, harnessId?: string): ModelLatency | null;
|
|
175
190
|
/** Observed chars-per-token ratio for a tokenizer family; null until calibrated. */
|
|
176
191
|
tokenRatio(tokenizer: string): number | null;
|
|
177
192
|
recentEntries(limit: number): LedgerEntry[];
|
package/src/router/candidates.ts
CHANGED
|
@@ -5,9 +5,9 @@
|
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
7
|
import type { CatalogModel, CatalogSnapshot } from "../catalog/types.ts";
|
|
8
|
-
import type { QualityAxis, RouterConfig } from "../config/types.ts";
|
|
8
|
+
import type { FilterConfig, QualityAxis, RouterConfig } from "../config/types.ts";
|
|
9
9
|
import { forecast, priceAt } from "../cost/forecast.ts";
|
|
10
|
-
import type { Ledger } from "../cost/types.ts";
|
|
10
|
+
import type { Ledger, ModelLatency } from "../cost/types.ts";
|
|
11
11
|
import type { NormRequest } from "../wire/types.ts";
|
|
12
12
|
import { effectivePriceCeiling, effectiveQualityFloor, tierPlanFor } from "./tier-plan.ts";
|
|
13
13
|
import type { Candidate, Features, Rejection, TaskType, Tier } from "./types.ts";
|
|
@@ -68,6 +68,22 @@ function resolveQuality(model: CatalogModel, axis: QualityAxis): { score: number
|
|
|
68
68
|
/** Neutral trust prior for models our ledger has never observed. */
|
|
69
69
|
const UNMEASURED_TRUST = 0.9;
|
|
70
70
|
|
|
71
|
+
/** Excess-ratio cap so one very slow model cannot be penalised into oblivion. */
|
|
72
|
+
const LATENCY_EXCESS_CAP = 3;
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Latency penalty as a multiplier on effective cost (>= 1; 1 = no penalty).
|
|
76
|
+
* Mean TTFT above `latencyReferenceMs` inflates the model's effective cost, the
|
|
77
|
+
* same lever trust uses for flakiness, so a faster model of equal quality and
|
|
78
|
+
* price outranks a sluggish one. Inert when the weight is 0 or the model has
|
|
79
|
+
* too few streamed samples to judge.
|
|
80
|
+
*/
|
|
81
|
+
function latencyMultiplier(latency: ModelLatency | null, filters: FilterConfig): number {
|
|
82
|
+
if (latency === null || filters.latencyWeight <= 0 || latency.samples < filters.latencyMinSamples) return 1;
|
|
83
|
+
const excess = Math.max(0, (latency.ttftMs - filters.latencyReferenceMs) / filters.latencyReferenceMs);
|
|
84
|
+
return 1 + filters.latencyWeight * Math.min(excess, LATENCY_EXCESS_CAP);
|
|
85
|
+
}
|
|
86
|
+
|
|
71
87
|
export function buildCandidates(args: BuildCandidatesArgs): { candidates: Candidate[]; rejected: Rejection[] } {
|
|
72
88
|
const { req, features, tier, task, snapshot, ledger, cfg, expectedCompletionTokens, warmSlug, relaxLevel = 0 } = args;
|
|
73
89
|
// A Set only when non-empty: the common path allocates nothing.
|
|
@@ -215,9 +231,15 @@ export function buildCandidates(args: BuildCandidatesArgs): { candidates: Candid
|
|
|
215
231
|
const trustScore = trust !== null && trust.attempts > 0 ? trust.successRate : UNMEASURED_TRUST;
|
|
216
232
|
const qualityScore = quality?.score ?? 0;
|
|
217
233
|
// Shared scoring: trust converts flakiness into money — a model failing
|
|
218
|
-
// 20% of the time really costs ~25% more in retries.
|
|
219
|
-
//
|
|
220
|
-
|
|
234
|
+
// 20% of the time really costs ~25% more in retries. Latency does the same
|
|
235
|
+
// for slowness (TTFT over the reference). qualityExponent 0 makes this
|
|
236
|
+
// "cheapest above the floor"; the floor does the quality work.
|
|
237
|
+
const latency =
|
|
238
|
+
filters.latencyWeight > 0
|
|
239
|
+
? (ledger?.latency(slug, filters.trustScopedByHarness ? req.harnessId : undefined) ?? null)
|
|
240
|
+
: null;
|
|
241
|
+
const latencyMult = latencyMultiplier(latency, filters);
|
|
242
|
+
const effectiveUsd = (fc.expectedUsd / Math.max(trustScore, 0.5)) * latencyMult;
|
|
221
243
|
const score = Math.pow(qualityScore / 100, tierCfg.qualityExponent) / Math.max(effectiveUsd, 1e-9);
|
|
222
244
|
|
|
223
245
|
const reasons: string[] = [
|
|
@@ -229,6 +251,9 @@ export function buildCandidates(args: BuildCandidatesArgs): { candidates: Candid
|
|
|
229
251
|
: `trust ${trustScore.toFixed(2)} over ${trust.attempts} attempts`,
|
|
230
252
|
`expected $${fc.expectedUsd.toFixed(6)}`,
|
|
231
253
|
];
|
|
254
|
+
if (latencyMult > 1 && latency !== null) {
|
|
255
|
+
reasons.push(`latency penalty ×${latencyMult.toFixed(2)} (ttft ${Math.round(latency.ttftMs)}ms over ${latency.samples} samples)`);
|
|
256
|
+
}
|
|
232
257
|
if (pinned) reasons.push("pinned into tier");
|
|
233
258
|
candidates.push({ model, forecast: fc, qualityScore, trustScore, score, reasons });
|
|
234
259
|
}
|
package/src/router/classify.ts
CHANGED
|
@@ -43,9 +43,18 @@ const W_LARGE_CONTENT = 0.05;
|
|
|
43
43
|
const W_HUGE_CONTENT = 0.1;
|
|
44
44
|
const W_TURN_DEPTH = 0.004; // long conversations accumulate entangled context
|
|
45
45
|
const CAP_TURN_DEPTH = 0.08;
|
|
46
|
-
const W_LOOP_DEPTH = -0.008; // deep
|
|
46
|
+
const W_LOOP_DEPTH = -0.008; // a single deep step is mechanical
|
|
47
47
|
const CAP_LOOP_DEPTH = -0.06;
|
|
48
|
-
|
|
48
|
+
// A sustained autonomous loop is the signal the underlying task is substantial:
|
|
49
|
+
// the agent keeps grinding without human input. Unlike the mechanical-step
|
|
50
|
+
// penalty above, this term ACCUMULATES with depth past the agentic threshold so
|
|
51
|
+
// long/hard loops climb out of trivial instead of scoring like a one-line edit.
|
|
52
|
+
// Capped so pure depth tops out in `simple` (a competent-but-cheap model);
|
|
53
|
+
// reaching `moderate`/`hard` still requires real complexity signals (tool
|
|
54
|
+
// failure, repeated call, keywords) to stack on top.
|
|
55
|
+
const W_AUTONOMOUS_LOOP = 0.06; // base bonus at the threshold
|
|
56
|
+
const W_AUTONOMOUS_LOOP_PER_DEPTH = 0.018; // added per loop step beyond the threshold
|
|
57
|
+
const CAP_AUTONOMOUS_LOOP = 0.34;
|
|
49
58
|
const W_IMAGES = 0.04;
|
|
50
59
|
const W_TOOLS_OFFERED = 0.03;
|
|
51
60
|
|
|
@@ -100,7 +109,11 @@ export function scoreHeuristic(f: Features, cfg: RouterConfig): Classification {
|
|
|
100
109
|
add(Math.min(f.turnDepth * W_TURN_DEPTH, CAP_TURN_DEPTH), `conversation depth ${f.turnDepth}`);
|
|
101
110
|
add(Math.max(f.toolLoopDepth * W_LOOP_DEPTH, CAP_LOOP_DEPTH), `tool loop depth ${f.toolLoopDepth}`);
|
|
102
111
|
if (f.toolLoopDepth >= cfg.classifier.agenticLoopDepth && f.toolLoopDepth > 0) {
|
|
103
|
-
|
|
112
|
+
const excessDepth = f.toolLoopDepth - cfg.classifier.agenticLoopDepth;
|
|
113
|
+
add(
|
|
114
|
+
Math.min(W_AUTONOMOUS_LOOP + excessDepth * W_AUTONOMOUS_LOOP_PER_DEPTH, CAP_AUTONOMOUS_LOOP),
|
|
115
|
+
`autonomous loop depth ${f.toolLoopDepth} (sustained task)`,
|
|
116
|
+
);
|
|
104
117
|
}
|
|
105
118
|
if (f.hasImages) add(W_IMAGES, "image input");
|
|
106
119
|
if (f.toolCount > 0) add(W_TOOLS_OFFERED, `${f.toolCount} tools offered`);
|
package/test/classify.test.ts
CHANGED
|
@@ -37,6 +37,32 @@ function featuresFor(messages: unknown[], tools?: unknown[] | undefined): Featur
|
|
|
37
37
|
return extractFeatures(r, 5000);
|
|
38
38
|
}
|
|
39
39
|
|
|
40
|
+
/** A tool-result continuation at a given autonomous-loop depth, no other signals. */
|
|
41
|
+
function contFeatures(toolLoopDepth: number, over: Partial<Features> = {}): Features {
|
|
42
|
+
return {
|
|
43
|
+
promptTokens: 30000,
|
|
44
|
+
newContentTokens: 200,
|
|
45
|
+
turnDepth: toolLoopDepth,
|
|
46
|
+
toolCount: 12,
|
|
47
|
+
toolSchemaBytes: 9783,
|
|
48
|
+
isToolResultContinuation: true,
|
|
49
|
+
toolLoopDepth,
|
|
50
|
+
distinctToolsUsed: 3,
|
|
51
|
+
lastToolFailed: false,
|
|
52
|
+
repeatedToolCall: false,
|
|
53
|
+
hasImages: false,
|
|
54
|
+
codeBlocks: 0,
|
|
55
|
+
codeBytes: 0,
|
|
56
|
+
looksLikeDiff: false,
|
|
57
|
+
complexityKeywords: [],
|
|
58
|
+
trivialityKeywords: [],
|
|
59
|
+
requestedReasoning: undefined,
|
|
60
|
+
questionCount: 0,
|
|
61
|
+
isTerseInstruction: false,
|
|
62
|
+
...over,
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
|
|
40
66
|
const SYSTEM = { role: "system", content: "You are a coding agent." };
|
|
41
67
|
|
|
42
68
|
/** Upstream double that fails loudly if the adjudicator is consulted. */
|
|
@@ -160,6 +186,39 @@ describe("scoreHeuristic", () => {
|
|
|
160
186
|
expect(c.confidence).toBeGreaterThanOrEqual(0);
|
|
161
187
|
expect(c.confidence).toBeLessThanOrEqual(1);
|
|
162
188
|
});
|
|
189
|
+
|
|
190
|
+
test("a shallow tool-result continuation stays trivial", () => {
|
|
191
|
+
const shallow = scoreHeuristic(contFeatures(2), BASE);
|
|
192
|
+
expect(shallow.tier).toBe("trivial");
|
|
193
|
+
});
|
|
194
|
+
|
|
195
|
+
test("a sustained autonomous loop climbs out of trivial", () => {
|
|
196
|
+
// The failure mode this fixes: a long coding loop pinned to the cheapest
|
|
197
|
+
// tier for dozens of turns because agentic complexity never accumulated.
|
|
198
|
+
const shallow = scoreHeuristic(contFeatures(2), BASE);
|
|
199
|
+
const deep = scoreHeuristic(contFeatures(20), BASE);
|
|
200
|
+
expect(tierIdx(deep.tier)).toBeGreaterThan(tierIdx(shallow.tier));
|
|
201
|
+
expect(deep.tier).not.toBe("trivial");
|
|
202
|
+
});
|
|
203
|
+
|
|
204
|
+
test("score increases monotonically with loop depth past the agentic threshold", () => {
|
|
205
|
+
const depths = [4, 6, 8, 10, 15, 20, 30];
|
|
206
|
+
let prev = -1;
|
|
207
|
+
for (const d of depths) {
|
|
208
|
+
const s = scoreHeuristic(contFeatures(d), BASE).score;
|
|
209
|
+
expect(s).toBeGreaterThanOrEqual(prev);
|
|
210
|
+
prev = s;
|
|
211
|
+
}
|
|
212
|
+
});
|
|
213
|
+
|
|
214
|
+
test("pure loop depth tops out at simple; complexity signals stack to moderate", () => {
|
|
215
|
+
// Depth alone means competent-but-cheap (simple), not a frontier model.
|
|
216
|
+
const veryDeep = scoreHeuristic(contFeatures(90), BASE);
|
|
217
|
+
expect(tierIdx(veryDeep.tier)).toBeLessThanOrEqual(tierIdx("simple"));
|
|
218
|
+
// A failing tool result on top of a deep loop is genuinely hard.
|
|
219
|
+
const deepAndFailing = scoreHeuristic(contFeatures(20, { lastToolFailed: true }), BASE);
|
|
220
|
+
expect(tierIdx(deepAndFailing.tier)).toBeGreaterThanOrEqual(tierIdx("moderate"));
|
|
221
|
+
});
|
|
163
222
|
});
|
|
164
223
|
|
|
165
224
|
describe("pickQualityAxis", () => {
|
package/test/failover.test.ts
CHANGED
|
@@ -43,7 +43,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
|
|
|
43
43
|
data: { axis: "intelligence", minQuality: 0 },
|
|
44
44
|
chat: { axis: "intelligence", minQuality: 0 },
|
|
45
45
|
},
|
|
46
|
-
filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, minTrustSamples: 5, trustScopedByHarness: false, contextHeadroom: 1.2 },
|
|
46
|
+
filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, minTrustSamples: 5, trustScopedByHarness: false, contextHeadroom: 1.2, latencyWeight: 0, latencyReferenceMs: 5000, latencyMinSamples: 20 },
|
|
47
47
|
classifier: {
|
|
48
48
|
ambiguityThreshold: 0,
|
|
49
49
|
model: "test/adjudicator",
|
|
@@ -229,6 +229,7 @@ function mkLedger(): { ledger: Ledger; entries: LedgerEntry[] } {
|
|
|
229
229
|
conversationSpend: () => 0,
|
|
230
230
|
spendSince: () => 0,
|
|
231
231
|
blendedRate: () => null,
|
|
232
|
+
latency: () => null,
|
|
232
233
|
trust: () => null,
|
|
233
234
|
allTrust: () => [],
|
|
234
235
|
tokenRatio: () => null,
|
package/test/select.test.ts
CHANGED
|
@@ -257,6 +257,7 @@ describe("budget guard", () => {
|
|
|
257
257
|
conversationSpend: () => 0,
|
|
258
258
|
spendSince: (_sinceMs, harnessId) => (harnessId === undefined ? 1.0 : spendByHarness[harnessId] ?? 0),
|
|
259
259
|
blendedRate: () => null,
|
|
260
|
+
latency: () => null,
|
|
260
261
|
trust: () => null,
|
|
261
262
|
allTrust: () => [],
|
|
262
263
|
tokenRatio: () => null,
|
|
@@ -283,6 +284,7 @@ describe("per-harness trust scoping", () => {
|
|
|
283
284
|
conversationSpend: () => 0,
|
|
284
285
|
spendSince: () => 0,
|
|
285
286
|
blendedRate: () => null,
|
|
287
|
+
latency: () => null,
|
|
286
288
|
trust: (_slug, harnessId) => {
|
|
287
289
|
// Harness A has burned the model; harness B has never tried it.
|
|
288
290
|
if (harnessId === "harness-a") {
|
|
@@ -418,6 +420,7 @@ describe("tier rescue under a guardrail-constrained catalog", () => {
|
|
|
418
420
|
conversationSpend: () => 0,
|
|
419
421
|
spendSince: () => 0,
|
|
420
422
|
blendedRate: () => null,
|
|
423
|
+
latency: () => null,
|
|
421
424
|
trust: (slug) => ({
|
|
422
425
|
slug,
|
|
423
426
|
attempts: 40,
|
|
@@ -536,3 +539,46 @@ describe("task-type routing", () => {
|
|
|
536
539
|
}
|
|
537
540
|
});
|
|
538
541
|
});
|
|
542
|
+
|
|
543
|
+
describe("latency scoring", () => {
|
|
544
|
+
function ledgerWithLatency(ttftBySlug: Record<string, { ttftMs: number; samples: number }>): Ledger {
|
|
545
|
+
return {
|
|
546
|
+
record: () => {},
|
|
547
|
+
conversationSpend: () => 0,
|
|
548
|
+
spendSince: () => 0,
|
|
549
|
+
blendedRate: () => null,
|
|
550
|
+
trust: () => null,
|
|
551
|
+
allTrust: () => [],
|
|
552
|
+
latency: (slug) => {
|
|
553
|
+
const v = ttftBySlug[slug];
|
|
554
|
+
return v === undefined ? null : { slug, samples: v.samples, ttftMs: v.ttftMs };
|
|
555
|
+
},
|
|
556
|
+
tokenRatio: () => null,
|
|
557
|
+
recentEntries: () => [],
|
|
558
|
+
};
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
const withWeight = (latencyWeight: number): RouterConfig => ({
|
|
562
|
+
...BASE,
|
|
563
|
+
filters: { ...BASE.filters, latencyWeight, latencyReferenceMs: 5000, latencyMinSamples: 20 },
|
|
564
|
+
});
|
|
565
|
+
|
|
566
|
+
test("penalises a chronically slow model out of the top slot", () => {
|
|
567
|
+
const slow = run({ tier: "simple" }).slug;
|
|
568
|
+
const ledger = ledgerWithLatency({ [slow]: { ttftMs: 60_000, samples: 50 } });
|
|
569
|
+
const d = run({ tier: "simple", cfg: withWeight(2), ledger });
|
|
570
|
+
expect(d.slug).not.toBe(slow);
|
|
571
|
+
});
|
|
572
|
+
|
|
573
|
+
test("latencyWeight 0 disables the penalty", () => {
|
|
574
|
+
const slow = run({ tier: "simple" }).slug;
|
|
575
|
+
const ledger = ledgerWithLatency({ [slow]: { ttftMs: 60_000, samples: 50 } });
|
|
576
|
+
expect(run({ tier: "simple", cfg: withWeight(0), ledger }).slug).toBe(slow);
|
|
577
|
+
});
|
|
578
|
+
|
|
579
|
+
test("a model with too few samples is not penalised", () => {
|
|
580
|
+
const slow = run({ tier: "simple" }).slug;
|
|
581
|
+
const ledger = ledgerWithLatency({ [slow]: { ttftMs: 60_000, samples: 5 } });
|
|
582
|
+
expect(run({ tier: "simple", cfg: withWeight(2), ledger }).slug).toBe(slow);
|
|
583
|
+
});
|
|
584
|
+
});
|
|
@@ -145,6 +145,39 @@ describe("trust attribution", () => {
|
|
|
145
145
|
});
|
|
146
146
|
});
|
|
147
147
|
|
|
148
|
+
describe("latency signal", () => {
|
|
149
|
+
function latencyOf(rows: Array<Partial<LedgerEntry>>): { samples: number; ttftMs: number } | null {
|
|
150
|
+
const db = openDb(":memory:");
|
|
151
|
+
try {
|
|
152
|
+
const ledger = createLedger(db, cfg);
|
|
153
|
+
for (const r of rows) ledger.record(entry(r));
|
|
154
|
+
const l = ledger.latency("vendor/model");
|
|
155
|
+
return l === null ? null : { samples: l.samples, ttftMs: l.ttftMs };
|
|
156
|
+
} finally {
|
|
157
|
+
db.close();
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
test("averages TTFT over streamed, non-errored turns", () => {
|
|
162
|
+
expect(latencyOf([{ ttftMs: 50 }, { ttftMs: 100 }, { ttftMs: 150 }])).toEqual({ samples: 3, ttftMs: 100 });
|
|
163
|
+
});
|
|
164
|
+
|
|
165
|
+
test("excludes errored, aborted, and non-streamed (null TTFT) rows", () => {
|
|
166
|
+
expect(
|
|
167
|
+
latencyOf([
|
|
168
|
+
{ ttftMs: 100 },
|
|
169
|
+
{ ttftMs: 9999, error: "upstream_error: boom" },
|
|
170
|
+
{ ttftMs: 9999, error: "request aborted" },
|
|
171
|
+
{ ttftMs: null },
|
|
172
|
+
]),
|
|
173
|
+
).toEqual({ samples: 1, ttftMs: 100 });
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
test("null when no streamed sample exists", () => {
|
|
177
|
+
expect(latencyOf([{ ttftMs: null }, { ttftMs: 0 }])).toBeNull();
|
|
178
|
+
});
|
|
179
|
+
});
|
|
180
|
+
|
|
148
181
|
describe("v4 migration", () => {
|
|
149
182
|
test("backfills error_kind from stored error text", () => {
|
|
150
183
|
const db = openDb(":memory:");
|
package/test/turn.test.ts
CHANGED
|
@@ -43,7 +43,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
|
|
|
43
43
|
data: { axis: "intelligence", minQuality: 0 },
|
|
44
44
|
chat: { axis: "intelligence", minQuality: 0 },
|
|
45
45
|
},
|
|
46
|
-
filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, minTrustSamples: 5, trustScopedByHarness: false, contextHeadroom: 1.2 },
|
|
46
|
+
filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, minTrustSamples: 5, trustScopedByHarness: false, contextHeadroom: 1.2, latencyWeight: 0, latencyReferenceMs: 5000, latencyMinSamples: 20 },
|
|
47
47
|
classifier: {
|
|
48
48
|
ambiguityThreshold: 0,
|
|
49
49
|
model: "test/adjudicator",
|
|
@@ -223,6 +223,7 @@ function mkLedger(): { ledger: Ledger; entries: LedgerEntry[] } {
|
|
|
223
223
|
conversationSpend: () => 0,
|
|
224
224
|
spendSince: () => 0,
|
|
225
225
|
blendedRate: () => null,
|
|
226
|
+
latency: () => null,
|
|
226
227
|
trust: () => null,
|
|
227
228
|
allTrust: () => [],
|
|
228
229
|
tokenRatio: () => null,
|