auto-model-router 0.3.4 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.omp-plugin/marketplace.json +2 -2
- package/README.md +29 -4
- package/omp-extension/report-logic.ts +93 -0
- package/omp-extension/router-configure.ts +128 -2
- package/omp-extension/router-embed.ts +9 -6
- package/package.json +1 -1
- package/src/catalog/ollama-catalog.ts +30 -2
- package/src/cli/config-wizard.ts +11 -0
- package/src/cli/report.ts +7 -2
- package/src/config/defaults.ts +17 -0
- package/src/config/schema.ts +8 -0
- package/src/config/types.ts +65 -0
- package/src/cost/feedback.ts +81 -0
- package/src/cost/ledger.ts +110 -5
- package/src/cost/report.ts +118 -2
- package/src/cost/types.ts +32 -1
- package/src/router/candidates.ts +13 -4
- package/src/router/classify.ts +13 -0
- package/src/router/features.ts +59 -1
- package/src/router/index.ts +23 -5
- package/src/router/learned.ts +202 -0
- package/src/router/select.ts +64 -8
- package/src/router/types.ts +39 -1
- package/src/server/http.ts +82 -5
- package/src/server/overrides.ts +83 -0
- package/src/server/providers.ts +10 -2
- package/src/server/turn.ts +16 -1
- package/src/upstream/ollama-usage.ts +79 -2
- package/src/util/sqlite.ts +30 -0
- package/src/wire/openai/request.ts +4 -0
- package/src/wire/types.ts +2 -0
- package/test/classify.test.ts +13 -0
- package/test/config-wizard.test.ts +8 -6
- package/test/controls.test.ts +238 -0
- package/test/escalate.test.ts +1 -0
- package/test/failover.test.ts +6 -4
- package/test/features.test.ts +63 -0
- package/test/http-resilience.test.ts +1 -1
- package/test/learned.test.ts +61 -0
- package/test/ollama.test.ts +74 -2
- package/test/report-hub.test.ts +6 -2
- package/test/report-logic.test.ts +3 -0
- package/test/report.test.ts +57 -0
- package/test/select.test.ts +126 -1
- package/test/trust-attribution.test.ts +95 -0
- package/test/turn.test.ts +36 -4
- package/tools/replay.ts +267 -156
- package/tools/train-classifier.ts +111 -0
package/src/cost/report.ts
CHANGED
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
11
|
import type { Database } from "bun:sqlite";
|
|
12
|
+
import { createFeedbackStore, type FeedbackCounts } from "./feedback.ts";
|
|
12
13
|
|
|
13
14
|
export interface ReportTotals {
|
|
14
15
|
dispatches: number;
|
|
@@ -26,6 +27,9 @@ export interface ReportTotals {
|
|
|
26
27
|
modelSwitches: number;
|
|
27
28
|
/** Any row in the window carries an estimated cache count. */
|
|
28
29
|
cacheEstimated: boolean;
|
|
30
|
+
/** Turns from omp subagents (`features.isSubagent`), and their spend. */
|
|
31
|
+
subagentDispatches: number;
|
|
32
|
+
subagentSpendUsd: number;
|
|
29
33
|
}
|
|
30
34
|
|
|
31
35
|
export interface ReportRow {
|
|
@@ -50,6 +54,8 @@ export interface ModelRow extends ReportRow {
|
|
|
50
54
|
provider: string;
|
|
51
55
|
/** Dispatch counts per tier, e.g. `{ trivial: 12, moderate: 3 }`. */
|
|
52
56
|
tiers: Record<string, number>;
|
|
57
|
+
/** User verdicts from /router good|bad on turns this model served, in the window. */
|
|
58
|
+
feedback: FeedbackCounts;
|
|
53
59
|
}
|
|
54
60
|
|
|
55
61
|
export interface DayRow {
|
|
@@ -71,6 +77,50 @@ export interface UsageReport {
|
|
|
71
77
|
models: ModelRow[];
|
|
72
78
|
tiers: ReportRow[];
|
|
73
79
|
days: DayRow[];
|
|
80
|
+
/** Mean prompt composition over rows that recorded it; null when none did. */
|
|
81
|
+
anatomy: AnatomyShare | null;
|
|
82
|
+
/** What the window would have cost on one model throughout, per configured baseline. */
|
|
83
|
+
baselines: BaselineRow[];
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export interface BaselineRow {
|
|
87
|
+
slug: string;
|
|
88
|
+
usd: number;
|
|
89
|
+
/** 1 − routed spend ÷ baseline spend; negative when the router cost more. */
|
|
90
|
+
savedShare: number;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** Resolves configured baseline slugs against a catalog lookup; unknown slugs are skipped. */
|
|
94
|
+
export function baselinePrices(slugs: readonly string[], find: (slug: string) => { price: { prompt: number; completion: number; cacheRead?: number } } | undefined): BaselinePrice[] {
|
|
95
|
+
const out: BaselinePrice[] = [];
|
|
96
|
+
for (const slug of slugs) {
|
|
97
|
+
const m = find(slug);
|
|
98
|
+
if (m === undefined) continue;
|
|
99
|
+
out.push({ slug, prompt: m.price.prompt, completion: m.price.completion, ...(m.price.cacheRead === undefined ? {} : { cacheRead: m.price.cacheRead }) });
|
|
100
|
+
}
|
|
101
|
+
return out;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** A baseline's prices per token (the catalog's `Price`, or a subset of it). */
|
|
105
|
+
export interface BaselinePrice {
|
|
106
|
+
slug: string;
|
|
107
|
+
prompt: number;
|
|
108
|
+
completion: number;
|
|
109
|
+
cacheRead?: number;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/** Shares of prompt bytes, 0-1, averaged over the window's dispatches. */
|
|
113
|
+
export interface AnatomyShare {
|
|
114
|
+
rows: number;
|
|
115
|
+
avgMessages: number;
|
|
116
|
+
system: number;
|
|
117
|
+
user: number;
|
|
118
|
+
assistant: number;
|
|
119
|
+
tool: number;
|
|
120
|
+
/** Tool schemas relative to prompt bytes (they ride in the tools param, not the messages). */
|
|
121
|
+
schemas: number;
|
|
122
|
+
olderHalf: number;
|
|
123
|
+
staleTool: number;
|
|
74
124
|
}
|
|
75
125
|
|
|
76
126
|
const USD = "COALESCE(reported_usd, predicted_usd)";
|
|
@@ -129,7 +179,10 @@ function toRow(r: RawRow, windowSpend: number): ReportRow {
|
|
|
129
179
|
* Builds the report for the last `windowDays`. `harnessId` narrows to one
|
|
130
180
|
* harness (the `X-Omp-Harness` header); empty means everything.
|
|
131
181
|
*/
|
|
132
|
-
export function buildUsageReport(
|
|
182
|
+
export function buildUsageReport(
|
|
183
|
+
db: Database,
|
|
184
|
+
opts: { windowDays: number; harnessId?: string; nowMs?: number; baselines?: readonly BaselinePrice[] },
|
|
185
|
+
): UsageReport {
|
|
133
186
|
const nowMs = opts.nowMs ?? Date.now();
|
|
134
187
|
const windowDays = Math.max(1, opts.windowDays);
|
|
135
188
|
const sinceMs = nowMs - windowDays * 86_400_000;
|
|
@@ -146,6 +199,8 @@ export function buildUsageReport(db: Database, opts: { windowDays: number; harne
|
|
|
146
199
|
COALESCE(SUM(${CT}), 0) AS cached_tokens,
|
|
147
200
|
COALESCE(SUM(${COMP}), 0) AS completion_tokens,
|
|
148
201
|
SUM(CASE WHEN ${EST} THEN 1 ELSE 0 END) AS estimated_rows,
|
|
202
|
+
SUM(CASE WHEN json_extract(features, '$.isSubagent') = 1 THEN 1 ELSE 0 END) AS subagent_rows,
|
|
203
|
+
COALESCE(SUM(CASE WHEN json_extract(features, '$.isSubagent') = 1 THEN ${USD} ELSE 0 END), 0) AS subagent_spend,
|
|
149
204
|
SUM(CASE WHEN escalation_signal IS NOT NULL THEN 1 ELSE 0 END) AS escalations,
|
|
150
205
|
SUM(CASE WHEN instr(reasons, 'failover:') > 0 THEN 1 ELSE 0 END) AS failovers,
|
|
151
206
|
SUM(CASE WHEN error IS NOT NULL THEN 1 ELSE 0 END) AS errors,
|
|
@@ -160,6 +215,8 @@ export function buildUsageReport(db: Database, opts: { windowDays: number; harne
|
|
|
160
215
|
cached_tokens: number;
|
|
161
216
|
completion_tokens: number;
|
|
162
217
|
estimated_rows: number | null;
|
|
218
|
+
subagent_rows: number | null;
|
|
219
|
+
subagent_spend: number;
|
|
163
220
|
escalations: number | null;
|
|
164
221
|
failovers: number | null;
|
|
165
222
|
errors: number | null;
|
|
@@ -196,10 +253,12 @@ export function buildUsageReport(db: Database, opts: { windowDays: number; harne
|
|
|
196
253
|
rec[m.tier] = m.n;
|
|
197
254
|
mixByModel.set(m.key, rec);
|
|
198
255
|
}
|
|
256
|
+
const feedbackBySlug = createFeedbackStore(db).countsBySlug(sinceMs, harnessId);
|
|
199
257
|
const models: ModelRow[] = modelRows.map((r) => ({
|
|
200
258
|
...toRow(r, windowSpend),
|
|
201
259
|
provider: r.key.startsWith("ollama/") ? "ollama" : "openrouter",
|
|
202
260
|
tiers: mixByModel.get(r.key) ?? {},
|
|
261
|
+
feedback: feedbackBySlug.get(r.key) ?? { good: 0, bad: 0 },
|
|
203
262
|
}));
|
|
204
263
|
|
|
205
264
|
const tiers = (
|
|
@@ -221,6 +280,42 @@ export function buildUsageReport(db: Database, opts: { windowDays: number; harne
|
|
|
221
280
|
cacheHitRate: d.prompt_tokens > 0 ? d.cached_tokens / d.prompt_tokens : 0,
|
|
222
281
|
}));
|
|
223
282
|
|
|
283
|
+
const an = db
|
|
284
|
+
.query(
|
|
285
|
+
`SELECT COUNT(*) AS rows, AVG(json_extract(features, '$.anatomy.messages')) AS msgs,
|
|
286
|
+
AVG(json_extract(features, '$.anatomy.systemBytes')) AS sys, AVG(json_extract(features, '$.anatomy.userBytes')) AS usr,
|
|
287
|
+
AVG(json_extract(features, '$.anatomy.assistantBytes')) AS asst, AVG(json_extract(features, '$.anatomy.toolBytes')) AS tool,
|
|
288
|
+
AVG(json_extract(features, '$.toolSchemaBytes')) AS schemas,
|
|
289
|
+
AVG(json_extract(features, '$.anatomy.olderHalfBytes')) AS older, AVG(json_extract(features, '$.anatomy.staleToolBytes')) AS stale
|
|
290
|
+
FROM ledger WHERE ${where} AND json_extract(features, '$.anatomy.messages') IS NOT NULL`,
|
|
291
|
+
)
|
|
292
|
+
.get(bind) as { rows: number; msgs: number | null; sys: number | null; usr: number | null; asst: number | null; tool: number | null; schemas: number | null; older: number | null; stale: number | null };
|
|
293
|
+
let anatomy: AnatomyShare | null = null;
|
|
294
|
+
if (an.rows > 0) {
|
|
295
|
+
const total = (an.sys ?? 0) + (an.usr ?? 0) + (an.asst ?? 0) + (an.tool ?? 0);
|
|
296
|
+
const share = (v: number | null): number => (total > 0 ? (v ?? 0) / total : 0);
|
|
297
|
+
anatomy = {
|
|
298
|
+
rows: an.rows,
|
|
299
|
+
avgMessages: Math.round(an.msgs ?? 0),
|
|
300
|
+
system: share(an.sys),
|
|
301
|
+
user: share(an.usr),
|
|
302
|
+
assistant: share(an.asst),
|
|
303
|
+
tool: share(an.tool),
|
|
304
|
+
schemas: share(an.schemas),
|
|
305
|
+
olderHalf: share(an.older),
|
|
306
|
+
staleTool: share(an.stale),
|
|
307
|
+
};
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
// Counterfactual: the window's tokens on one model throughout, at list
|
|
311
|
+
// price with the window's own cache hit rate (cached tokens read at the
|
|
312
|
+
// baseline's cache rate, or full price when it publishes none).
|
|
313
|
+
const baselines: BaselineRow[] = (opts.baselines ?? []).map((b) => {
|
|
314
|
+
const fresh = Math.max(0, t.prompt_tokens - t.cached_tokens);
|
|
315
|
+
const usd = fresh * b.prompt + t.cached_tokens * (b.cacheRead ?? b.prompt) + t.completion_tokens * b.completion;
|
|
316
|
+
return { slug: b.slug, usd, savedShare: usd > 0 ? 1 - t.spend / usd : 0 };
|
|
317
|
+
});
|
|
318
|
+
|
|
224
319
|
return {
|
|
225
320
|
generatedAtMs: nowMs,
|
|
226
321
|
windowDays,
|
|
@@ -239,11 +334,15 @@ export function buildUsageReport(db: Database, opts: { windowDays: number; harne
|
|
|
239
334
|
aborted: t.aborted ?? 0,
|
|
240
335
|
modelSwitches: switches,
|
|
241
336
|
cacheEstimated: (t.estimated_rows ?? 0) > 0,
|
|
337
|
+
subagentDispatches: t.subagent_rows ?? 0,
|
|
338
|
+
subagentSpendUsd: t.subagent_spend,
|
|
242
339
|
},
|
|
243
340
|
providers,
|
|
244
341
|
models,
|
|
245
342
|
tiers,
|
|
246
343
|
days,
|
|
344
|
+
anatomy,
|
|
345
|
+
baselines,
|
|
247
346
|
};
|
|
248
347
|
}
|
|
249
348
|
|
|
@@ -294,6 +393,22 @@ export function reportView(r: UsageReport, opts: { maxModels?: number } = {}): R
|
|
|
294
393
|
`spend ${usd(t.spendUsd)} over ${num(t.dispatches)} dispatches in ${num(t.conversations)} conversations · ${usd(t.dispatches > 0 ? t.spendUsd / t.dispatches : 0)}/dispatch`,
|
|
295
394
|
`prompt ${num(t.promptTokens)} tok (cache hit ${pct(t.cacheHitRate, t.cacheEstimated)}) · completion ${num(t.completionTokens)} tok · switches ${num(t.modelSwitches)} · escalations ${num(t.escalations)} · failovers ${num(t.failovers)} · errors ${num(t.errors)} (${num(t.aborted)} aborted)`,
|
|
296
395
|
];
|
|
396
|
+
if (r.baselines.length > 0 && t.dispatches > 0) {
|
|
397
|
+
summary.push(
|
|
398
|
+
`same traffic on one model: ${r.baselines
|
|
399
|
+
.map((b) => `${b.slug} ${usd(b.usd)} (router ${b.savedShare >= 0 ? "saved" : "cost extra"} ${pct(Math.abs(b.savedShare))})`)
|
|
400
|
+
.join(" · ")}`,
|
|
401
|
+
);
|
|
402
|
+
}
|
|
403
|
+
if (t.subagentDispatches > 0) {
|
|
404
|
+
summary.push(`subagents: ${num(t.subagentDispatches)} dispatches, ${usd(t.subagentSpendUsd)} (${pct(t.spendUsd > 0 ? t.subagentSpendUsd / t.spendUsd : 0)} of spend)`);
|
|
405
|
+
}
|
|
406
|
+
const a = r.anatomy;
|
|
407
|
+
if (a !== null) {
|
|
408
|
+
summary.push(
|
|
409
|
+
`prompt anatomy (mean of ${num(a.rows)}): tool results ${pct(a.tool)} · assistant ${pct(a.assistant)} · user ${pct(a.user)} · system ${pct(a.system)} · tool schemas +${pct(a.schemas)} · older half ${pct(a.olderHalf)} · stale tool results ${pct(a.staleTool)} · ${num(a.avgMessages)} messages`,
|
|
410
|
+
);
|
|
411
|
+
}
|
|
297
412
|
const tables: ReportTable[] = [];
|
|
298
413
|
if (r.providers.length > 0) {
|
|
299
414
|
tables.push({
|
|
@@ -307,7 +422,7 @@ export function reportView(r: UsageReport, opts: { maxModels?: number } = {}): R
|
|
|
307
422
|
tables.push({
|
|
308
423
|
id: "models",
|
|
309
424
|
title: `models (top ${Math.min(maxModels, r.models.length)} of ${r.models.length} by spend)`,
|
|
310
|
-
headers: ["model", "dispatches", "spend", "share", "cache", "ttft", "speed", "tiers"],
|
|
425
|
+
headers: ["model", "dispatches", "spend", "share", "cache", "ttft", "speed", "feedback", "tiers"],
|
|
311
426
|
rows: r.models.slice(0, maxModels).map((m) => [
|
|
312
427
|
m.key,
|
|
313
428
|
num(m.dispatches),
|
|
@@ -316,6 +431,7 @@ export function reportView(r: UsageReport, opts: { maxModels?: number } = {}): R
|
|
|
316
431
|
pct(m.cacheHitRate, m.cacheEstimated),
|
|
317
432
|
ms(m.avgTtftMs),
|
|
318
433
|
tps(m.tokensPerSec),
|
|
434
|
+
m.feedback.good + m.feedback.bad === 0 ? "" : `+${m.feedback.good}/-${m.feedback.bad}`,
|
|
319
435
|
Object.entries(m.tiers)
|
|
320
436
|
.sort((a, b) => b[1] - a[1])
|
|
321
437
|
.map(([k, v]) => `${k}:${v}`)
|
package/src/cost/types.ts
CHANGED
|
@@ -177,8 +177,11 @@ export interface ModelTrust {
|
|
|
177
177
|
escalations: number;
|
|
178
178
|
/** Attempts that ended in an upstream error. */
|
|
179
179
|
errors: number;
|
|
180
|
-
/** Laplace-smoothed success rate, 0-1. */
|
|
180
|
+
/** Laplace-smoothed success rate, 0-1; user verdicts weigh in at filters.feedbackWeight. */
|
|
181
181
|
successRate: number;
|
|
182
|
+
/** User verdicts in the window (/router good|bad). */
|
|
183
|
+
feedbackGood?: number;
|
|
184
|
+
feedbackBad?: number;
|
|
182
185
|
/** Mean absolute relative prediction error, for forecast calibration. */
|
|
183
186
|
meanCostError: number;
|
|
184
187
|
}
|
|
@@ -200,9 +203,24 @@ export interface ModelLatency {
|
|
|
200
203
|
tokensPerSec: number;
|
|
201
204
|
}
|
|
202
205
|
|
|
206
|
+
/**
|
|
207
|
+
* How often a model's prompt cache actually hit when the router expected it
|
|
208
|
+
* warm: the previous kept turn of the conversation was on the same model
|
|
209
|
+
* within `hysteresis.cacheWarmTtlMs`. Provider-side misses (a model whose
|
|
210
|
+
* cache is flaky, or absent) show up here as a low rate. Router-estimated
|
|
211
|
+
* cache counts (Ollama) are excluded: they are constructed, not observed.
|
|
212
|
+
*/
|
|
213
|
+
export interface ModelCacheReliability {
|
|
214
|
+
slug: string;
|
|
215
|
+
samples: number;
|
|
216
|
+
/** Mean cached / expected-cached over those samples, 0-1. */
|
|
217
|
+
hitRate: number;
|
|
218
|
+
}
|
|
219
|
+
|
|
203
220
|
export interface LedgerSignals {
|
|
204
221
|
trust: ModelTrust | null;
|
|
205
222
|
latency: ModelLatency | null;
|
|
223
|
+
cache?: ModelCacheReliability | null;
|
|
206
224
|
}
|
|
207
225
|
|
|
208
226
|
/** Measured price of a probe escalation: what the retry billed per prompt token of the failed turn. */
|
|
@@ -241,7 +259,20 @@ export interface Ledger {
|
|
|
241
259
|
* escalation-cost term in candidate scoring is inert without it.
|
|
242
260
|
*/
|
|
243
261
|
escalationCost?(windowDays: number): EscalationCost | null;
|
|
262
|
+
/**
|
|
263
|
+
* Observed cache hit rate when a warm cache was expected (see
|
|
264
|
+
* ModelCacheReliability). Null until any sample exists. Optional so fakes
|
|
265
|
+
* need not implement it; the stay/switch comparison assumes a reliable
|
|
266
|
+
* cache without it.
|
|
267
|
+
*/
|
|
268
|
+
cacheReliability?(slug: string): ModelCacheReliability | null;
|
|
244
269
|
/** Observed chars-per-token ratio for a tokenizer family; null until calibrated. */
|
|
245
270
|
tokenRatio(tokenizer: string): number | null;
|
|
246
271
|
recentEntries(limit: number): LedgerEntry[];
|
|
272
|
+
/** Spend since an instant on slugs with a prefix (`ollama/`), for provider-level reconciliation. Optional. */
|
|
273
|
+
providerSpendSince?(slugPrefix: string, sinceMs: number): number;
|
|
274
|
+
/** Newest kept (non-wasted) entry for an omp session, for /router why and feedback. Optional so fakes need not implement it. */
|
|
275
|
+
latestForSession?(ompSessionId: string): LedgerEntry | null;
|
|
276
|
+
/** Newest entries for an omp session, newest first. Optional. */
|
|
277
|
+
entriesForSession?(ompSessionId: string, limit: number): LedgerEntry[];
|
|
247
278
|
}
|
package/src/router/candidates.ts
CHANGED
|
@@ -111,12 +111,21 @@ function expectedWaitMs(latency: ModelLatency, expectedCompletionTokens: number)
|
|
|
111
111
|
* tiny cost and capped at LATENCY_EXCESS_CAP, so the model stays cheapest. That is
|
|
112
112
|
* `filters.maxExpectedWaitMs`'s job — a hard drop, applied in buildCandidates.
|
|
113
113
|
*/
|
|
114
|
-
|
|
115
|
-
|
|
114
|
+
/**
|
|
115
|
+
* The latency weight in force for a turn: the continuation weight on a
|
|
116
|
+
* tool-result continuation when one is configured, else the general one.
|
|
117
|
+
* A person waits on first token only when the turn is theirs.
|
|
118
|
+
*/
|
|
119
|
+
export function latencyWeightFor(filters: FilterConfig, isToolResultContinuation: boolean): number {
|
|
120
|
+
return isToolResultContinuation && filters.latencyWeightContinuation !== undefined ? filters.latencyWeightContinuation : filters.latencyWeight;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function latencyMultiplier(latency: ModelLatency | null, filters: FilterConfig, expectedCompletionTokens: number, weight = filters.latencyWeight): number {
|
|
124
|
+
if (latency === null || weight <= 0 || latency.samples < filters.latencyMinSamples) return 1;
|
|
116
125
|
const waitMs = expectedWaitMs(latency, expectedCompletionTokens);
|
|
117
126
|
const refWaitMs = filters.latencyReferenceMs + (expectedCompletionTokens / filters.latencyReferenceTokensPerSec) * 1000;
|
|
118
127
|
const excess = refWaitMs > 0 ? Math.max(0, (waitMs - refWaitMs) / refWaitMs) : 0;
|
|
119
|
-
return 1 +
|
|
128
|
+
return 1 + weight * Math.min(excess, LATENCY_EXCESS_CAP);
|
|
120
129
|
}
|
|
121
130
|
|
|
122
131
|
export function buildCandidates(args: BuildCandidatesArgs): { candidates: Candidate[]; rejected: Rejection[] } {
|
|
@@ -320,7 +329,7 @@ export function buildCandidates(args: BuildCandidatesArgs): { candidates: Candid
|
|
|
320
329
|
// 20% of the time really costs ~25% more in retries. Latency does the same
|
|
321
330
|
// for slowness (TTFT over the reference). qualityExponent 0 makes this
|
|
322
331
|
// "cheapest above the floor"; the floor does the quality work.
|
|
323
|
-
const latencyMult = latencyMultiplier(latency, filters, expectedCompletionTokens);
|
|
332
|
+
const latencyMult = latencyMultiplier(latency, filters, expectedCompletionTokens, latencyWeightFor(filters, features.isToolResultContinuation));
|
|
324
333
|
// Escalation-cost term: the trust divisor prices a failure as a retry of
|
|
325
334
|
// THIS model, but a probe escalation re-dispatches the whole prompt on
|
|
326
335
|
// the next tier's model — measured at ~700x a cheap model's own turn cost.
|
package/src/router/classify.ts
CHANGED
|
@@ -10,6 +10,7 @@ import type { Ledger } from "../cost/types.ts";
|
|
|
10
10
|
import { estimateTokens } from "../tokens/estimate.ts";
|
|
11
11
|
import type { UpstreamClient } from "../upstream/types.ts";
|
|
12
12
|
import { sha256Hex } from "../util/hash.ts";
|
|
13
|
+
import { loadLearnedModel, predictRisk } from "./learned.ts";
|
|
13
14
|
import type { NormRequest, ReasoningLevel } from "../wire/types.ts";
|
|
14
15
|
import type { Classification, Features, TaskType, Tier } from "./types.ts";
|
|
15
16
|
|
|
@@ -165,6 +166,9 @@ export function scoreHeuristic(f: Features, cfg: RouterConfig): Classification {
|
|
|
165
166
|
// the served model must still accept image input — is enforced separately
|
|
166
167
|
// on `req.hasImages` in candidate selection, exactly as `classifyTask` does.
|
|
167
168
|
if (f.hasNewImage) add(W_IMAGES, "new image input");
|
|
169
|
+
if (f.readOnlyToolTail === true && cfg.classifier.readOnlyToolWeight > 0) {
|
|
170
|
+
add(-cfg.classifier.readOnlyToolWeight, "read-only tool loop (the model is looking, not deciding)");
|
|
171
|
+
}
|
|
168
172
|
if (f.toolCount > 0) add(W_TOOLS_OFFERED, `${f.toolCount} tools offered`);
|
|
169
173
|
|
|
170
174
|
score = Math.min(1, Math.max(0, score));
|
|
@@ -303,6 +307,15 @@ export async function classify(
|
|
|
303
307
|
): Promise<Classification> {
|
|
304
308
|
const heuristic = scoreHeuristic(f, cfg);
|
|
305
309
|
const cc = cfg.classifier;
|
|
310
|
+
// Advisory learned risk: recorded beside the decision, never acted on here.
|
|
311
|
+
if (cc.learnedModelPath !== "") {
|
|
312
|
+
const model = await loadLearnedModel(cc.learnedModelPath);
|
|
313
|
+
if (model !== null) {
|
|
314
|
+
const risk = predictRisk(model, f);
|
|
315
|
+
heuristic.learnedRisk = risk;
|
|
316
|
+
heuristic.reasons.push(`learned: p(escalate)=${risk.toFixed(3)}`);
|
|
317
|
+
}
|
|
318
|
+
}
|
|
306
319
|
if (cc.ambiguityThreshold <= 0 || heuristic.confidence >= cc.ambiguityThreshold) return heuristic;
|
|
307
320
|
|
|
308
321
|
const digest = buildDigest(req, f);
|
package/src/router/features.ts
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
10
|
import type { NormMessage, NormRequest } from "../wire/types.ts";
|
|
11
|
-
import type { Features } from "./types.ts";
|
|
11
|
+
import type { Features, PromptAnatomy } from "./types.ts";
|
|
12
12
|
|
|
13
13
|
/**
|
|
14
14
|
* Complexity signals. Deliberately small: each hit pushes the turn toward a
|
|
@@ -224,6 +224,37 @@ export function extractFeatures(req: NormRequest, promptTokens: number): Feature
|
|
|
224
224
|
codeBlocks === 0 &&
|
|
225
225
|
(terminators === null ? 0 : terminators.length) <= 1;
|
|
226
226
|
|
|
227
|
+
// Prompt anatomy: where the bytes sit. Cheap (one pass over textBytes) and
|
|
228
|
+
// content-free, so it is safe to record on every row.
|
|
229
|
+
const anatomy: PromptAnatomy = { messages: 0, systemBytes: 0, userBytes: 0, assistantBytes: 0, toolBytes: 0, olderHalfBytes: 0, staleToolBytes: 0 };
|
|
230
|
+
const nonSystem = messages.filter((m) => m.role !== "system");
|
|
231
|
+
const olderHalfEnd = Math.floor(nonSystem.length / 2);
|
|
232
|
+
const staleEnd = Math.max(0, nonSystem.length - 20);
|
|
233
|
+
nonSystem.forEach((m, i) => {
|
|
234
|
+
if (i < olderHalfEnd) anatomy.olderHalfBytes += m.textBytes;
|
|
235
|
+
if (m.role === "tool" && i < staleEnd) anatomy.staleToolBytes += m.textBytes;
|
|
236
|
+
});
|
|
237
|
+
for (const m of messages) {
|
|
238
|
+
anatomy.messages++;
|
|
239
|
+
if (m.role === "system") anatomy.systemBytes += m.textBytes;
|
|
240
|
+
else if (m.role === "user") anatomy.userBytes += m.textBytes;
|
|
241
|
+
else if (m.role === "assistant") anatomy.assistantBytes += m.textBytes;
|
|
242
|
+
else anatomy.toolBytes += m.textBytes;
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
// Read-only tool loop: the assistant call behind a tool-result tail used
|
|
246
|
+
// only tools that look at things. Tool names are the harness's own; the
|
|
247
|
+
// set covers omp's built-ins and their common aliases.
|
|
248
|
+
let readOnlyToolTail = false;
|
|
249
|
+
if (isToolResultContinuation) {
|
|
250
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
251
|
+
const m = messages[i];
|
|
252
|
+
if (m === undefined || m.role !== "assistant") continue;
|
|
253
|
+
if (m.toolCalls.length > 0) readOnlyToolTail = m.toolCalls.every((tc) => READ_ONLY_TOOLS.has(tc.name.toLowerCase()));
|
|
254
|
+
break;
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
|
|
227
258
|
return {
|
|
228
259
|
promptTokens,
|
|
229
260
|
newContentTokens,
|
|
@@ -246,5 +277,32 @@ export function extractFeatures(req: NormRequest, promptTokens: number): Feature
|
|
|
246
277
|
requestedReasoning: req.reasoning,
|
|
247
278
|
questionCount,
|
|
248
279
|
isTerseInstruction,
|
|
280
|
+
anatomy,
|
|
281
|
+
isSubagent: req.isSubagent,
|
|
282
|
+
readOnlyToolTail,
|
|
249
283
|
};
|
|
250
284
|
}
|
|
285
|
+
|
|
286
|
+
/** Tools that read state without changing it, in omp, Claude Code and Hermes naming. */
|
|
287
|
+
export const READ_ONLY_TOOLS: ReadonlySet<string> = new Set([
|
|
288
|
+
"read",
|
|
289
|
+
"read_file",
|
|
290
|
+
"grep",
|
|
291
|
+
"glob",
|
|
292
|
+
"ls",
|
|
293
|
+
"list",
|
|
294
|
+
"list_dir",
|
|
295
|
+
"find",
|
|
296
|
+
"lsp",
|
|
297
|
+
"ast_grep",
|
|
298
|
+
"search",
|
|
299
|
+
"web_search",
|
|
300
|
+
"web_fetch",
|
|
301
|
+
"webfetch",
|
|
302
|
+
"websearch",
|
|
303
|
+
"fetch",
|
|
304
|
+
"cat",
|
|
305
|
+
"view",
|
|
306
|
+
"inspect_image",
|
|
307
|
+
"todo",
|
|
308
|
+
]);
|
package/src/router/index.ts
CHANGED
|
@@ -38,11 +38,18 @@ export interface RouterDeps {
|
|
|
38
38
|
*/
|
|
39
39
|
const NEUTRAL_TOKENIZER = "gpt";
|
|
40
40
|
|
|
41
|
-
function resolveProfile(cfg: RouterConfig, requestedModel: string): ProfileConfig {
|
|
42
|
-
const exact = cfg.profiles.find((p) => p.id === requestedModel);
|
|
43
|
-
if (exact !== undefined) return exact;
|
|
41
|
+
export function resolveProfile(cfg: RouterConfig, requestedModel: string, isSubagent = false): ProfileConfig {
|
|
44
42
|
const fallback = cfg.profiles[0];
|
|
45
43
|
if (fallback === undefined) throw new Error("no router profiles configured");
|
|
44
|
+
// A subagent asking for the default profile is routed under the subagent
|
|
45
|
+
// profile when one is configured and exists; an explicit other profile
|
|
46
|
+
// (auto-max, auto-cheap) is honoured as asked.
|
|
47
|
+
const exact = cfg.profiles.find((p) => p.id === requestedModel);
|
|
48
|
+
if (isSubagent && cfg.server.subagentProfile !== "" && (exact === undefined || exact.id === fallback.id)) {
|
|
49
|
+
const sub = cfg.profiles.find((p) => p.id === cfg.server.subagentProfile);
|
|
50
|
+
if (sub !== undefined) return sub;
|
|
51
|
+
}
|
|
52
|
+
if (exact !== undefined) return exact;
|
|
46
53
|
return fallback;
|
|
47
54
|
}
|
|
48
55
|
|
|
@@ -52,7 +59,7 @@ export function createRouter(deps: RouterDeps): Router {
|
|
|
52
59
|
return {
|
|
53
60
|
async route(
|
|
54
61
|
req: NormRequest,
|
|
55
|
-
opts: { attempt: number; escalateFrom?: Tier; excludeSlugs?: readonly string[] },
|
|
62
|
+
opts: { attempt: number; escalateFrom?: Tier; excludeSlugs?: readonly string[]; forceTier?: Tier; forceSlug?: string },
|
|
56
63
|
): Promise<Decision> {
|
|
57
64
|
const state = conversations.get(req.conversationKey) ?? conversations.load(req.conversationKey);
|
|
58
65
|
const snapshot = await catalog.get();
|
|
@@ -78,6 +85,16 @@ export function createRouter(deps: RouterDeps): Router {
|
|
|
78
85
|
score: 1,
|
|
79
86
|
reasons: [`escalated from ${opts.escalateFrom} after attempt ${opts.attempt - 1} was rejected`],
|
|
80
87
|
};
|
|
88
|
+
} else if (opts.forceTier !== undefined) {
|
|
89
|
+
// A session override from omp: the user chose the tier for a while.
|
|
90
|
+
classification = {
|
|
91
|
+
tier: opts.forceTier,
|
|
92
|
+
task: classifyTask(features),
|
|
93
|
+
confidence: 1,
|
|
94
|
+
source: "forced",
|
|
95
|
+
score: 1,
|
|
96
|
+
reasons: [`tier ${opts.forceTier} forced by session override (/router tier)`],
|
|
97
|
+
};
|
|
81
98
|
} else {
|
|
82
99
|
classification = await classify(req, features, config, { upstream, ledger, catalog });
|
|
83
100
|
}
|
|
@@ -86,13 +103,14 @@ export function createRouter(deps: RouterDeps): Router {
|
|
|
86
103
|
req,
|
|
87
104
|
features,
|
|
88
105
|
classification,
|
|
89
|
-
profile: resolveProfile(config, req.requestedModel),
|
|
106
|
+
profile: resolveProfile(config, req.requestedModel, req.isSubagent),
|
|
90
107
|
state,
|
|
91
108
|
snapshot,
|
|
92
109
|
ledger,
|
|
93
110
|
cfg: config,
|
|
94
111
|
nowMs: Date.now(),
|
|
95
112
|
...(opts.excludeSlugs === undefined ? {} : { excludeSlugs: opts.excludeSlugs }),
|
|
113
|
+
...(opts.forceSlug === undefined ? {} : { forceSlug: opts.forceSlug }),
|
|
96
114
|
});
|
|
97
115
|
},
|
|
98
116
|
};
|