auto-model-router 0.3.4 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/.omp-plugin/marketplace.json +2 -2
  2. package/README.md +29 -4
  3. package/omp-extension/report-logic.ts +93 -0
  4. package/omp-extension/router-configure.ts +128 -2
  5. package/omp-extension/router-embed.ts +9 -6
  6. package/package.json +1 -1
  7. package/src/catalog/ollama-catalog.ts +30 -2
  8. package/src/cli/config-wizard.ts +11 -0
  9. package/src/cli/report.ts +7 -2
  10. package/src/config/defaults.ts +17 -0
  11. package/src/config/schema.ts +8 -0
  12. package/src/config/types.ts +65 -0
  13. package/src/cost/feedback.ts +81 -0
  14. package/src/cost/ledger.ts +110 -5
  15. package/src/cost/report.ts +118 -2
  16. package/src/cost/types.ts +32 -1
  17. package/src/router/candidates.ts +13 -4
  18. package/src/router/classify.ts +13 -0
  19. package/src/router/features.ts +59 -1
  20. package/src/router/index.ts +23 -5
  21. package/src/router/learned.ts +202 -0
  22. package/src/router/select.ts +64 -8
  23. package/src/router/types.ts +39 -1
  24. package/src/server/http.ts +82 -5
  25. package/src/server/overrides.ts +83 -0
  26. package/src/server/providers.ts +10 -2
  27. package/src/server/turn.ts +16 -1
  28. package/src/upstream/ollama-usage.ts +79 -2
  29. package/src/util/sqlite.ts +30 -0
  30. package/src/wire/openai/request.ts +4 -0
  31. package/src/wire/types.ts +2 -0
  32. package/test/classify.test.ts +13 -0
  33. package/test/config-wizard.test.ts +8 -6
  34. package/test/controls.test.ts +238 -0
  35. package/test/escalate.test.ts +1 -0
  36. package/test/failover.test.ts +6 -4
  37. package/test/features.test.ts +63 -0
  38. package/test/http-resilience.test.ts +1 -1
  39. package/test/learned.test.ts +61 -0
  40. package/test/ollama.test.ts +74 -2
  41. package/test/report-hub.test.ts +6 -2
  42. package/test/report-logic.test.ts +3 -0
  43. package/test/report.test.ts +57 -0
  44. package/test/select.test.ts +126 -1
  45. package/test/trust-attribution.test.ts +95 -0
  46. package/test/turn.test.ts +36 -4
  47. package/tools/replay.ts +267 -156
  48. package/tools/train-classifier.ts +111 -0
@@ -9,6 +9,7 @@
9
9
  */
10
10
 
11
11
  import type { Database } from "bun:sqlite";
12
+ import { createFeedbackStore, type FeedbackCounts } from "./feedback.ts";
12
13
 
13
14
  export interface ReportTotals {
14
15
  dispatches: number;
@@ -26,6 +27,9 @@ export interface ReportTotals {
26
27
  modelSwitches: number;
27
28
  /** Any row in the window carries an estimated cache count. */
28
29
  cacheEstimated: boolean;
30
+ /** Turns from omp subagents (`features.isSubagent`), and their spend. */
31
+ subagentDispatches: number;
32
+ subagentSpendUsd: number;
29
33
  }
30
34
 
31
35
  export interface ReportRow {
@@ -50,6 +54,8 @@ export interface ModelRow extends ReportRow {
50
54
  provider: string;
51
55
  /** Dispatch counts per tier, e.g. `{ trivial: 12, moderate: 3 }`. */
52
56
  tiers: Record<string, number>;
57
+ /** User verdicts from /router good|bad on turns this model served, in the window. */
58
+ feedback: FeedbackCounts;
53
59
  }
54
60
 
55
61
  export interface DayRow {
@@ -71,6 +77,50 @@ export interface UsageReport {
71
77
  models: ModelRow[];
72
78
  tiers: ReportRow[];
73
79
  days: DayRow[];
80
+ /** Mean prompt composition over rows that recorded it; null when none did. */
81
+ anatomy: AnatomyShare | null;
82
+ /** What the window would have cost on one model throughout, per configured baseline. */
83
+ baselines: BaselineRow[];
84
+ }
85
+
86
+ export interface BaselineRow {
87
+ slug: string;
88
+ usd: number;
89
+ /** 1 − routed spend ÷ baseline spend; negative when the router cost more. */
90
+ savedShare: number;
91
+ }
92
+
93
+ /** Resolves configured baseline slugs against a catalog lookup; unknown slugs are skipped. */
94
+ export function baselinePrices(slugs: readonly string[], find: (slug: string) => { price: { prompt: number; completion: number; cacheRead?: number } } | undefined): BaselinePrice[] {
95
+ const out: BaselinePrice[] = [];
96
+ for (const slug of slugs) {
97
+ const m = find(slug);
98
+ if (m === undefined) continue;
99
+ out.push({ slug, prompt: m.price.prompt, completion: m.price.completion, ...(m.price.cacheRead === undefined ? {} : { cacheRead: m.price.cacheRead }) });
100
+ }
101
+ return out;
102
+ }
103
+
104
+ /** A baseline's prices per token (the catalog's `Price`, or a subset of it). */
105
+ export interface BaselinePrice {
106
+ slug: string;
107
+ prompt: number;
108
+ completion: number;
109
+ cacheRead?: number;
110
+ }
111
+
112
+ /** Shares of prompt bytes, 0-1, averaged over the window's dispatches. */
113
+ export interface AnatomyShare {
114
+ rows: number;
115
+ avgMessages: number;
116
+ system: number;
117
+ user: number;
118
+ assistant: number;
119
+ tool: number;
120
+ /** Tool schemas relative to prompt bytes (they ride in the tools param, not the messages). */
121
+ schemas: number;
122
+ olderHalf: number;
123
+ staleTool: number;
74
124
  }
75
125
 
76
126
  const USD = "COALESCE(reported_usd, predicted_usd)";
@@ -129,7 +179,10 @@ function toRow(r: RawRow, windowSpend: number): ReportRow {
129
179
  * Builds the report for the last `windowDays`. `harnessId` narrows to one
130
180
  * harness (the `X-Omp-Harness` header); empty means everything.
131
181
  */
132
- export function buildUsageReport(db: Database, opts: { windowDays: number; harnessId?: string; nowMs?: number }): UsageReport {
182
+ export function buildUsageReport(
183
+ db: Database,
184
+ opts: { windowDays: number; harnessId?: string; nowMs?: number; baselines?: readonly BaselinePrice[] },
185
+ ): UsageReport {
133
186
  const nowMs = opts.nowMs ?? Date.now();
134
187
  const windowDays = Math.max(1, opts.windowDays);
135
188
  const sinceMs = nowMs - windowDays * 86_400_000;
@@ -146,6 +199,8 @@ export function buildUsageReport(db: Database, opts: { windowDays: number; harne
146
199
  COALESCE(SUM(${CT}), 0) AS cached_tokens,
147
200
  COALESCE(SUM(${COMP}), 0) AS completion_tokens,
148
201
  SUM(CASE WHEN ${EST} THEN 1 ELSE 0 END) AS estimated_rows,
202
+ SUM(CASE WHEN json_extract(features, '$.isSubagent') = 1 THEN 1 ELSE 0 END) AS subagent_rows,
203
+ COALESCE(SUM(CASE WHEN json_extract(features, '$.isSubagent') = 1 THEN ${USD} ELSE 0 END), 0) AS subagent_spend,
149
204
  SUM(CASE WHEN escalation_signal IS NOT NULL THEN 1 ELSE 0 END) AS escalations,
150
205
  SUM(CASE WHEN instr(reasons, 'failover:') > 0 THEN 1 ELSE 0 END) AS failovers,
151
206
  SUM(CASE WHEN error IS NOT NULL THEN 1 ELSE 0 END) AS errors,
@@ -160,6 +215,8 @@ export function buildUsageReport(db: Database, opts: { windowDays: number; harne
160
215
  cached_tokens: number;
161
216
  completion_tokens: number;
162
217
  estimated_rows: number | null;
218
+ subagent_rows: number | null;
219
+ subagent_spend: number;
163
220
  escalations: number | null;
164
221
  failovers: number | null;
165
222
  errors: number | null;
@@ -196,10 +253,12 @@ export function buildUsageReport(db: Database, opts: { windowDays: number; harne
196
253
  rec[m.tier] = m.n;
197
254
  mixByModel.set(m.key, rec);
198
255
  }
256
+ const feedbackBySlug = createFeedbackStore(db).countsBySlug(sinceMs, harnessId);
199
257
  const models: ModelRow[] = modelRows.map((r) => ({
200
258
  ...toRow(r, windowSpend),
201
259
  provider: r.key.startsWith("ollama/") ? "ollama" : "openrouter",
202
260
  tiers: mixByModel.get(r.key) ?? {},
261
+ feedback: feedbackBySlug.get(r.key) ?? { good: 0, bad: 0 },
203
262
  }));
204
263
 
205
264
  const tiers = (
@@ -221,6 +280,42 @@ export function buildUsageReport(db: Database, opts: { windowDays: number; harne
221
280
  cacheHitRate: d.prompt_tokens > 0 ? d.cached_tokens / d.prompt_tokens : 0,
222
281
  }));
223
282
 
283
+ const an = db
284
+ .query(
285
+ `SELECT COUNT(*) AS rows, AVG(json_extract(features, '$.anatomy.messages')) AS msgs,
286
+ AVG(json_extract(features, '$.anatomy.systemBytes')) AS sys, AVG(json_extract(features, '$.anatomy.userBytes')) AS usr,
287
+ AVG(json_extract(features, '$.anatomy.assistantBytes')) AS asst, AVG(json_extract(features, '$.anatomy.toolBytes')) AS tool,
288
+ AVG(json_extract(features, '$.toolSchemaBytes')) AS schemas,
289
+ AVG(json_extract(features, '$.anatomy.olderHalfBytes')) AS older, AVG(json_extract(features, '$.anatomy.staleToolBytes')) AS stale
290
+ FROM ledger WHERE ${where} AND json_extract(features, '$.anatomy.messages') IS NOT NULL`,
291
+ )
292
+ .get(bind) as { rows: number; msgs: number | null; sys: number | null; usr: number | null; asst: number | null; tool: number | null; schemas: number | null; older: number | null; stale: number | null };
293
+ let anatomy: AnatomyShare | null = null;
294
+ if (an.rows > 0) {
295
+ const total = (an.sys ?? 0) + (an.usr ?? 0) + (an.asst ?? 0) + (an.tool ?? 0);
296
+ const share = (v: number | null): number => (total > 0 ? (v ?? 0) / total : 0);
297
+ anatomy = {
298
+ rows: an.rows,
299
+ avgMessages: Math.round(an.msgs ?? 0),
300
+ system: share(an.sys),
301
+ user: share(an.usr),
302
+ assistant: share(an.asst),
303
+ tool: share(an.tool),
304
+ schemas: share(an.schemas),
305
+ olderHalf: share(an.older),
306
+ staleTool: share(an.stale),
307
+ };
308
+ }
309
+
310
+ // Counterfactual: the window's tokens on one model throughout, at list
311
+ // price with the window's own cache hit rate (cached tokens read at the
312
+ // baseline's cache rate, or full price when it publishes none).
313
+ const baselines: BaselineRow[] = (opts.baselines ?? []).map((b) => {
314
+ const fresh = Math.max(0, t.prompt_tokens - t.cached_tokens);
315
+ const usd = fresh * b.prompt + t.cached_tokens * (b.cacheRead ?? b.prompt) + t.completion_tokens * b.completion;
316
+ return { slug: b.slug, usd, savedShare: usd > 0 ? 1 - t.spend / usd : 0 };
317
+ });
318
+
224
319
  return {
225
320
  generatedAtMs: nowMs,
226
321
  windowDays,
@@ -239,11 +334,15 @@ export function buildUsageReport(db: Database, opts: { windowDays: number; harne
239
334
  aborted: t.aborted ?? 0,
240
335
  modelSwitches: switches,
241
336
  cacheEstimated: (t.estimated_rows ?? 0) > 0,
337
+ subagentDispatches: t.subagent_rows ?? 0,
338
+ subagentSpendUsd: t.subagent_spend,
242
339
  },
243
340
  providers,
244
341
  models,
245
342
  tiers,
246
343
  days,
344
+ anatomy,
345
+ baselines,
247
346
  };
248
347
  }
249
348
 
@@ -294,6 +393,22 @@ export function reportView(r: UsageReport, opts: { maxModels?: number } = {}): R
294
393
  `spend ${usd(t.spendUsd)} over ${num(t.dispatches)} dispatches in ${num(t.conversations)} conversations · ${usd(t.dispatches > 0 ? t.spendUsd / t.dispatches : 0)}/dispatch`,
295
394
  `prompt ${num(t.promptTokens)} tok (cache hit ${pct(t.cacheHitRate, t.cacheEstimated)}) · completion ${num(t.completionTokens)} tok · switches ${num(t.modelSwitches)} · escalations ${num(t.escalations)} · failovers ${num(t.failovers)} · errors ${num(t.errors)} (${num(t.aborted)} aborted)`,
296
395
  ];
396
+ if (r.baselines.length > 0 && t.dispatches > 0) {
397
+ summary.push(
398
+ `same traffic on one model: ${r.baselines
399
+ .map((b) => `${b.slug} ${usd(b.usd)} (router ${b.savedShare >= 0 ? "saved" : "cost extra"} ${pct(Math.abs(b.savedShare))})`)
400
+ .join(" · ")}`,
401
+ );
402
+ }
403
+ if (t.subagentDispatches > 0) {
404
+ summary.push(`subagents: ${num(t.subagentDispatches)} dispatches, ${usd(t.subagentSpendUsd)} (${pct(t.spendUsd > 0 ? t.subagentSpendUsd / t.spendUsd : 0)} of spend)`);
405
+ }
406
+ const a = r.anatomy;
407
+ if (a !== null) {
408
+ summary.push(
409
+ `prompt anatomy (mean of ${num(a.rows)}): tool results ${pct(a.tool)} · assistant ${pct(a.assistant)} · user ${pct(a.user)} · system ${pct(a.system)} · tool schemas +${pct(a.schemas)} · older half ${pct(a.olderHalf)} · stale tool results ${pct(a.staleTool)} · ${num(a.avgMessages)} messages`,
410
+ );
411
+ }
297
412
  const tables: ReportTable[] = [];
298
413
  if (r.providers.length > 0) {
299
414
  tables.push({
@@ -307,7 +422,7 @@ export function reportView(r: UsageReport, opts: { maxModels?: number } = {}): R
307
422
  tables.push({
308
423
  id: "models",
309
424
  title: `models (top ${Math.min(maxModels, r.models.length)} of ${r.models.length} by spend)`,
310
- headers: ["model", "dispatches", "spend", "share", "cache", "ttft", "speed", "tiers"],
425
+ headers: ["model", "dispatches", "spend", "share", "cache", "ttft", "speed", "feedback", "tiers"],
311
426
  rows: r.models.slice(0, maxModels).map((m) => [
312
427
  m.key,
313
428
  num(m.dispatches),
@@ -316,6 +431,7 @@ export function reportView(r: UsageReport, opts: { maxModels?: number } = {}): R
316
431
  pct(m.cacheHitRate, m.cacheEstimated),
317
432
  ms(m.avgTtftMs),
318
433
  tps(m.tokensPerSec),
434
+ m.feedback.good + m.feedback.bad === 0 ? "" : `+${m.feedback.good}/-${m.feedback.bad}`,
319
435
  Object.entries(m.tiers)
320
436
  .sort((a, b) => b[1] - a[1])
321
437
  .map(([k, v]) => `${k}:${v}`)
package/src/cost/types.ts CHANGED
@@ -177,8 +177,11 @@ export interface ModelTrust {
177
177
  escalations: number;
178
178
  /** Attempts that ended in an upstream error. */
179
179
  errors: number;
180
- /** Laplace-smoothed success rate, 0-1. */
180
+ /** Laplace-smoothed success rate, 0-1; user verdicts weigh in at filters.feedbackWeight. */
181
181
  successRate: number;
182
+ /** User verdicts in the window (/router good|bad). */
183
+ feedbackGood?: number;
184
+ feedbackBad?: number;
182
185
  /** Mean absolute relative prediction error, for forecast calibration. */
183
186
  meanCostError: number;
184
187
  }
@@ -200,9 +203,24 @@ export interface ModelLatency {
200
203
  tokensPerSec: number;
201
204
  }
202
205
 
206
+ /**
207
+ * How often a model's prompt cache actually hit when the router expected it
208
+ * warm: the previous kept turn of the conversation was on the same model
209
+ * within `hysteresis.cacheWarmTtlMs`. Provider-side misses (a model whose
210
+ * cache is flaky, or absent) show up here as a low rate. Router-estimated
211
+ * cache counts (Ollama) are excluded: they are constructed, not observed.
212
+ */
213
+ export interface ModelCacheReliability {
214
+ slug: string;
215
+ samples: number;
216
+ /** Mean cached / expected-cached over those samples, 0-1. */
217
+ hitRate: number;
218
+ }
219
+
203
220
  export interface LedgerSignals {
204
221
  trust: ModelTrust | null;
205
222
  latency: ModelLatency | null;
223
+ cache?: ModelCacheReliability | null;
206
224
  }
207
225
 
208
226
  /** Measured price of a probe escalation: what the retry billed per prompt token of the failed turn. */
@@ -241,7 +259,20 @@ export interface Ledger {
241
259
  * escalation-cost term in candidate scoring is inert without it.
242
260
  */
243
261
  escalationCost?(windowDays: number): EscalationCost | null;
262
+ /**
263
+ * Observed cache hit rate when a warm cache was expected (see
264
+ * ModelCacheReliability). Null until any sample exists. Optional so fakes
265
+ * need not implement it; the stay/switch comparison assumes a reliable
266
+ * cache without it.
267
+ */
268
+ cacheReliability?(slug: string): ModelCacheReliability | null;
244
269
  /** Observed chars-per-token ratio for a tokenizer family; null until calibrated. */
245
270
  tokenRatio(tokenizer: string): number | null;
246
271
  recentEntries(limit: number): LedgerEntry[];
272
+ /** Spend since an instant on slugs with a prefix (`ollama/`), for provider-level reconciliation. Optional. */
273
+ providerSpendSince?(slugPrefix: string, sinceMs: number): number;
274
+ /** Newest kept (non-wasted) entry for an omp session, for /router why and feedback. Optional so fakes need not implement it. */
275
+ latestForSession?(ompSessionId: string): LedgerEntry | null;
276
+ /** Newest entries for an omp session, newest first. Optional. */
277
+ entriesForSession?(ompSessionId: string, limit: number): LedgerEntry[];
247
278
  }
@@ -111,12 +111,21 @@ function expectedWaitMs(latency: ModelLatency, expectedCompletionTokens: number)
111
111
  * tiny cost and capped at LATENCY_EXCESS_CAP, so the model stays cheapest. That is
112
112
  * `filters.maxExpectedWaitMs`'s job — a hard drop, applied in buildCandidates.
113
113
  */
114
- function latencyMultiplier(latency: ModelLatency | null, filters: FilterConfig, expectedCompletionTokens: number): number {
115
- if (latency === null || filters.latencyWeight <= 0 || latency.samples < filters.latencyMinSamples) return 1;
114
+ /**
115
+ * The latency weight in force for a turn: the continuation weight on a
116
+ * tool-result continuation when one is configured, else the general one.
117
+ * A person waits on first token only when the turn is theirs.
118
+ */
119
+ export function latencyWeightFor(filters: FilterConfig, isToolResultContinuation: boolean): number {
120
+ return isToolResultContinuation && filters.latencyWeightContinuation !== undefined ? filters.latencyWeightContinuation : filters.latencyWeight;
121
+ }
122
+
123
+ function latencyMultiplier(latency: ModelLatency | null, filters: FilterConfig, expectedCompletionTokens: number, weight = filters.latencyWeight): number {
124
+ if (latency === null || weight <= 0 || latency.samples < filters.latencyMinSamples) return 1;
116
125
  const waitMs = expectedWaitMs(latency, expectedCompletionTokens);
117
126
  const refWaitMs = filters.latencyReferenceMs + (expectedCompletionTokens / filters.latencyReferenceTokensPerSec) * 1000;
118
127
  const excess = refWaitMs > 0 ? Math.max(0, (waitMs - refWaitMs) / refWaitMs) : 0;
119
- return 1 + filters.latencyWeight * Math.min(excess, LATENCY_EXCESS_CAP);
128
+ return 1 + weight * Math.min(excess, LATENCY_EXCESS_CAP);
120
129
  }
121
130
 
122
131
  export function buildCandidates(args: BuildCandidatesArgs): { candidates: Candidate[]; rejected: Rejection[] } {
@@ -320,7 +329,7 @@ export function buildCandidates(args: BuildCandidatesArgs): { candidates: Candid
320
329
  // 20% of the time really costs ~25% more in retries. Latency does the same
321
330
  // for slowness (TTFT over the reference). qualityExponent 0 makes this
322
331
  // "cheapest above the floor"; the floor does the quality work.
323
- const latencyMult = latencyMultiplier(latency, filters, expectedCompletionTokens);
332
+ const latencyMult = latencyMultiplier(latency, filters, expectedCompletionTokens, latencyWeightFor(filters, features.isToolResultContinuation));
324
333
  // Escalation-cost term: the trust divisor prices a failure as a retry of
325
334
  // THIS model, but a probe escalation re-dispatches the whole prompt on
326
335
  // the next tier's model — measured at ~700x a cheap model's own turn cost.
@@ -10,6 +10,7 @@ import type { Ledger } from "../cost/types.ts";
10
10
  import { estimateTokens } from "../tokens/estimate.ts";
11
11
  import type { UpstreamClient } from "../upstream/types.ts";
12
12
  import { sha256Hex } from "../util/hash.ts";
13
+ import { loadLearnedModel, predictRisk } from "./learned.ts";
13
14
  import type { NormRequest, ReasoningLevel } from "../wire/types.ts";
14
15
  import type { Classification, Features, TaskType, Tier } from "./types.ts";
15
16
 
@@ -165,6 +166,9 @@ export function scoreHeuristic(f: Features, cfg: RouterConfig): Classification {
165
166
  // the served model must still accept image input — is enforced separately
166
167
  // on `req.hasImages` in candidate selection, exactly as `classifyTask` does.
167
168
  if (f.hasNewImage) add(W_IMAGES, "new image input");
169
+ if (f.readOnlyToolTail === true && cfg.classifier.readOnlyToolWeight > 0) {
170
+ add(-cfg.classifier.readOnlyToolWeight, "read-only tool loop (the model is looking, not deciding)");
171
+ }
168
172
  if (f.toolCount > 0) add(W_TOOLS_OFFERED, `${f.toolCount} tools offered`);
169
173
 
170
174
  score = Math.min(1, Math.max(0, score));
@@ -303,6 +307,15 @@ export async function classify(
303
307
  ): Promise<Classification> {
304
308
  const heuristic = scoreHeuristic(f, cfg);
305
309
  const cc = cfg.classifier;
310
+ // Advisory learned risk: recorded beside the decision, never acted on here.
311
+ if (cc.learnedModelPath !== "") {
312
+ const model = await loadLearnedModel(cc.learnedModelPath);
313
+ if (model !== null) {
314
+ const risk = predictRisk(model, f);
315
+ heuristic.learnedRisk = risk;
316
+ heuristic.reasons.push(`learned: p(escalate)=${risk.toFixed(3)}`);
317
+ }
318
+ }
306
319
  if (cc.ambiguityThreshold <= 0 || heuristic.confidence >= cc.ambiguityThreshold) return heuristic;
307
320
 
308
321
  const digest = buildDigest(req, f);
@@ -8,7 +8,7 @@
8
8
  */
9
9
 
10
10
  import type { NormMessage, NormRequest } from "../wire/types.ts";
11
- import type { Features } from "./types.ts";
11
+ import type { Features, PromptAnatomy } from "./types.ts";
12
12
 
13
13
  /**
14
14
  * Complexity signals. Deliberately small: each hit pushes the turn toward a
@@ -224,6 +224,37 @@ export function extractFeatures(req: NormRequest, promptTokens: number): Feature
224
224
  codeBlocks === 0 &&
225
225
  (terminators === null ? 0 : terminators.length) <= 1;
226
226
 
227
+ // Prompt anatomy: where the bytes sit. Cheap (one pass over textBytes) and
228
+ // content-free, so it is safe to record on every row.
229
+ const anatomy: PromptAnatomy = { messages: 0, systemBytes: 0, userBytes: 0, assistantBytes: 0, toolBytes: 0, olderHalfBytes: 0, staleToolBytes: 0 };
230
+ const nonSystem = messages.filter((m) => m.role !== "system");
231
+ const olderHalfEnd = Math.floor(nonSystem.length / 2);
232
+ const staleEnd = Math.max(0, nonSystem.length - 20);
233
+ nonSystem.forEach((m, i) => {
234
+ if (i < olderHalfEnd) anatomy.olderHalfBytes += m.textBytes;
235
+ if (m.role === "tool" && i < staleEnd) anatomy.staleToolBytes += m.textBytes;
236
+ });
237
+ for (const m of messages) {
238
+ anatomy.messages++;
239
+ if (m.role === "system") anatomy.systemBytes += m.textBytes;
240
+ else if (m.role === "user") anatomy.userBytes += m.textBytes;
241
+ else if (m.role === "assistant") anatomy.assistantBytes += m.textBytes;
242
+ else anatomy.toolBytes += m.textBytes;
243
+ }
244
+
245
+ // Read-only tool loop: the assistant call behind a tool-result tail used
246
+ // only tools that look at things. Tool names are the harness's own; the
247
+ // set covers omp's built-ins and their common aliases.
248
+ let readOnlyToolTail = false;
249
+ if (isToolResultContinuation) {
250
+ for (let i = messages.length - 1; i >= 0; i--) {
251
+ const m = messages[i];
252
+ if (m === undefined || m.role !== "assistant") continue;
253
+ if (m.toolCalls.length > 0) readOnlyToolTail = m.toolCalls.every((tc) => READ_ONLY_TOOLS.has(tc.name.toLowerCase()));
254
+ break;
255
+ }
256
+ }
257
+
227
258
  return {
228
259
  promptTokens,
229
260
  newContentTokens,
@@ -246,5 +277,32 @@ export function extractFeatures(req: NormRequest, promptTokens: number): Feature
246
277
  requestedReasoning: req.reasoning,
247
278
  questionCount,
248
279
  isTerseInstruction,
280
+ anatomy,
281
+ isSubagent: req.isSubagent,
282
+ readOnlyToolTail,
249
283
  };
250
284
  }
285
+
286
+ /** Tools that read state without changing it, in omp, Claude Code and Hermes naming. */
287
+ export const READ_ONLY_TOOLS: ReadonlySet<string> = new Set([
288
+ "read",
289
+ "read_file",
290
+ "grep",
291
+ "glob",
292
+ "ls",
293
+ "list",
294
+ "list_dir",
295
+ "find",
296
+ "lsp",
297
+ "ast_grep",
298
+ "search",
299
+ "web_search",
300
+ "web_fetch",
301
+ "webfetch",
302
+ "websearch",
303
+ "fetch",
304
+ "cat",
305
+ "view",
306
+ "inspect_image",
307
+ "todo",
308
+ ]);
@@ -38,11 +38,18 @@ export interface RouterDeps {
38
38
  */
39
39
  const NEUTRAL_TOKENIZER = "gpt";
40
40
 
41
- function resolveProfile(cfg: RouterConfig, requestedModel: string): ProfileConfig {
42
- const exact = cfg.profiles.find((p) => p.id === requestedModel);
43
- if (exact !== undefined) return exact;
41
+ export function resolveProfile(cfg: RouterConfig, requestedModel: string, isSubagent = false): ProfileConfig {
44
42
  const fallback = cfg.profiles[0];
45
43
  if (fallback === undefined) throw new Error("no router profiles configured");
44
+ // A subagent asking for the default profile is routed under the subagent
45
+ // profile when one is configured and exists; an explicit other profile
46
+ // (auto-max, auto-cheap) is honoured as asked.
47
+ const exact = cfg.profiles.find((p) => p.id === requestedModel);
48
+ if (isSubagent && cfg.server.subagentProfile !== "" && (exact === undefined || exact.id === fallback.id)) {
49
+ const sub = cfg.profiles.find((p) => p.id === cfg.server.subagentProfile);
50
+ if (sub !== undefined) return sub;
51
+ }
52
+ if (exact !== undefined) return exact;
46
53
  return fallback;
47
54
  }
48
55
 
@@ -52,7 +59,7 @@ export function createRouter(deps: RouterDeps): Router {
52
59
  return {
53
60
  async route(
54
61
  req: NormRequest,
55
- opts: { attempt: number; escalateFrom?: Tier; excludeSlugs?: readonly string[] },
62
+ opts: { attempt: number; escalateFrom?: Tier; excludeSlugs?: readonly string[]; forceTier?: Tier; forceSlug?: string },
56
63
  ): Promise<Decision> {
57
64
  const state = conversations.get(req.conversationKey) ?? conversations.load(req.conversationKey);
58
65
  const snapshot = await catalog.get();
@@ -78,6 +85,16 @@ export function createRouter(deps: RouterDeps): Router {
78
85
  score: 1,
79
86
  reasons: [`escalated from ${opts.escalateFrom} after attempt ${opts.attempt - 1} was rejected`],
80
87
  };
88
+ } else if (opts.forceTier !== undefined) {
89
+ // A session override from omp: the user chose the tier for a while.
90
+ classification = {
91
+ tier: opts.forceTier,
92
+ task: classifyTask(features),
93
+ confidence: 1,
94
+ source: "forced",
95
+ score: 1,
96
+ reasons: [`tier ${opts.forceTier} forced by session override (/router tier)`],
97
+ };
81
98
  } else {
82
99
  classification = await classify(req, features, config, { upstream, ledger, catalog });
83
100
  }
@@ -86,13 +103,14 @@ export function createRouter(deps: RouterDeps): Router {
86
103
  req,
87
104
  features,
88
105
  classification,
89
- profile: resolveProfile(config, req.requestedModel),
106
+ profile: resolveProfile(config, req.requestedModel, req.isSubagent),
90
107
  state,
91
108
  snapshot,
92
109
  ledger,
93
110
  cfg: config,
94
111
  nowMs: Date.now(),
95
112
  ...(opts.excludeSlugs === undefined ? {} : { excludeSlugs: opts.excludeSlugs }),
113
+ ...(opts.forceSlug === undefined ? {} : { forceSlug: opts.forceSlug }),
96
114
  });
97
115
  },
98
116
  };