auto-model-router 0.7.1 → 0.7.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.omp-plugin/marketplace.json +2 -2
- package/README.md +24 -6
- package/omp-extension/router-toast.ts +4 -1
- package/omp-extension/toast-logic.ts +98 -4
- package/package.json +1 -1
- package/src/cli/config-wizard.ts +1 -0
- package/src/config/defaults.ts +4 -0
- package/src/config/schema.ts +1 -0
- package/src/config/types.ts +15 -0
- package/src/router/select.ts +12 -0
- package/test/failover.test.ts +1 -1
- package/test/select.test.ts +28 -1
- package/test/toast-logic.test.ts +61 -0
- package/test/turn.test.ts +1 -1
|
@@ -7,14 +7,14 @@
|
|
|
7
7
|
},
|
|
8
8
|
"metadata": {
|
|
9
9
|
"description": "auto-model-router: a local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
|
|
10
|
-
"version": "0.7.
|
|
10
|
+
"version": "0.7.3",
|
|
11
11
|
"pluginRoot": "."
|
|
12
12
|
},
|
|
13
13
|
"plugins": [
|
|
14
14
|
{
|
|
15
15
|
"name": "auto-model-router",
|
|
16
16
|
"description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter. Runs in-process, routes per turn by price and task complexity, with budget caps, mid-stream escalation, and cache-aware hysteresis.",
|
|
17
|
-
"version": "0.7.
|
|
17
|
+
"version": "0.7.3",
|
|
18
18
|
"author": {
|
|
19
19
|
"name": "drewappling",
|
|
20
20
|
"email": "drewappling@gmail.com"
|
package/README.md
CHANGED
|
@@ -235,7 +235,7 @@ bun tools/install.ts --no-toast --no-configure # only the required embed exten
|
|
|
235
235
|
The installer adds:
|
|
236
236
|
|
|
237
237
|
- `router-embed.ts` — **required**; runs the router in-process.
|
|
238
|
-
- `router-toast.ts` — optional;
|
|
238
|
+
- `router-toast.ts` — optional; per-turn toasts naming the model and why it was chosen.
|
|
239
239
|
- `router-configure.ts` — optional; the `/router` command (configure, usage reports, status).
|
|
240
240
|
- `router-digest.ts` — optional; condenses large tool results with a cheap model before an expensive one reads them (needs `digest.enabled`).
|
|
241
241
|
|
|
@@ -814,6 +814,7 @@ disk and back up the previous file to a timestamped `.bak`.
|
|
|
814
814
|
| `AUTO_MODEL_ROUTER_URL` | Toast/base URL override (the toast reads the shared port file first). | — |
|
|
815
815
|
| `AUTO_MODEL_ROUTER_API_KEY` | Client bearer for the toast poll when `server.apiKey` is set. | — |
|
|
816
816
|
| `OMP_HARNESS_ID` | Per-harness toast scoping. | — |
|
|
817
|
+
| `AUTO_MODEL_ROUTER_TOAST` | `compact` for the one-line toast; anything else keeps the decision trail. | verbose |
|
|
817
818
|
|
|
818
819
|
---
|
|
819
820
|
|
|
@@ -907,6 +908,7 @@ Each task (`coding`, `vision`, `documentation`, `data`, `chat`) is a
|
|
|
907
908
|
| `minTrustSamples` | `12` | Attempts before trust is enforced. |
|
|
908
909
|
| `trustScopedByHarness` | `false` | `true` = each harness reads only its own trust rows. |
|
|
909
910
|
| `contextHeadroom` | `1.25` | Fraction of context kept free (a model must fit prompt × this). |
|
|
911
|
+
| `reasoningCompletionFloor` | `512` | Smallest completion budget a **reasoning** model is dispatched with. A reasoning model spends the budget thinking before it answers, so a caller's tight cap (omp asks ~12 tokens for a conversation title) returns nothing: measured on `ollama/gpt-oss:20b`, which hit the cap having produced no content and cost a dead dispatch plus a failover. Raised only for models that reason, never above the model's own ceiling, and a cap is an upper bound so a direct answer still stops early. 0 disables. |
|
|
910
912
|
| `latencyWeight` | `0` | How hard to penalise slow models in scoring (soft multiplier on effective cost). `0` disables it. |
|
|
911
913
|
| `latencyMinSamples` | `20` | Streamed samples before latency is judged against a model. |
|
|
912
914
|
| `cacheReliabilityMinSamples` | `10` | Warm-expected samples before a model's observed cache hit rate discounts its "stay warm" price in the stay/switch comparison. A model whose cache misses when it should be warm (measured: 5-6% on glm/gemini, 11% on ling, 50% on nex) is kept less eagerly. `0` assumes every cache is reliable. |
|
|
@@ -1468,11 +1470,27 @@ come from a small omp extension that polls the router's in-process ledger:
|
|
|
1468
1470
|
// omp-extension/router-toast.ts (shipped in this repo)
|
|
1469
1471
|
```
|
|
1470
1472
|
|
|
1471
|
-
It raises a TUI toast (`ctx.ui.notify`)
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1473
|
+
It raises a TUI toast (`ctx.ui.notify`) whenever a model is chosen, explaining
|
|
1474
|
+
the decision rather than just naming it — provider first, so a mixed catalog is
|
|
1475
|
+
legible at a glance:
|
|
1476
|
+
|
|
1477
|
+
```
|
|
1478
|
+
openrouter · openai/gpt-5.2 [hard] · $0.01820
|
|
1479
|
+
why: failover: z-ai/glm-5.3-flash empty_completion (hit the length cap having…
|
|
1480
|
+
escalated from moderate
|
|
1481
|
+
48.2k prompt · 12.8k compacted · 11 tools · attempt 2 · 2.1s to first token
|
|
1482
|
+
```
|
|
1483
|
+
|
|
1484
|
+
The **why** lines come from the router's own decision trail, picked by how much
|
|
1485
|
+
they change what you would do: a failover or a policy pin explains a surprising
|
|
1486
|
+
model outright, a hold or a tier rescue explains why the obvious cheaper pick was
|
|
1487
|
+
skipped, and when nothing surprising happened the ranking rationale itself is
|
|
1488
|
+
shown. The last line is what the model was actually handed. An unreported cost
|
|
1489
|
+
falls back to the forecast (`~$…`).
|
|
1490
|
+
|
|
1491
|
+
`AUTO_MODEL_ROUTER_TOAST=compact` restores the old one-liner
|
|
1492
|
+
(`openrouter · meta/muse-glimmer-30b [trivial] · $0.00001`). Install the
|
|
1493
|
+
extension by adding the file's absolute path to omp's `extensions:` list.
|
|
1476
1494
|
|
|
1477
1495
|
Because the embedded router binds a random port, the toast resolves the router
|
|
1478
1496
|
base URL on every poll in this order: the embedded router's port file
|
|
@@ -38,6 +38,9 @@ import { newestId, selectToasts, type ToastDecision } from "./toast-logic.ts";
|
|
|
38
38
|
// Empty ⇒ toast every harness (single-harness default).
|
|
39
39
|
const HARNESS_ID = process.env.OMP_HARNESS_ID ?? "";
|
|
40
40
|
const POLL_MS = 2_000;
|
|
41
|
+
// The toast explains the choice by default: model, tier, cost, why it was picked
|
|
42
|
+
// and what it was handed. `AUTO_MODEL_ROUTER_TOAST=compact` restores the one-liner.
|
|
43
|
+
const VERBOSE = (process.env.AUTO_MODEL_ROUTER_TOAST ?? "").toLowerCase() !== "compact";
|
|
41
44
|
|
|
42
45
|
export default function (pi: ExtensionAPI): void {
|
|
43
46
|
pi.setLabel("auto-model-router toast");
|
|
@@ -91,7 +94,7 @@ export default function (pi: ExtensionAPI): void {
|
|
|
91
94
|
const entries = body.entries;
|
|
92
95
|
if (!Array.isArray(entries) || entries.length === 0) return;
|
|
93
96
|
|
|
94
|
-
for (const t of selectToasts(entries, lastSeenId, HARNESS_ID, sessionId)) {
|
|
97
|
+
for (const t of selectToasts(entries, lastSeenId, HARNESS_ID, sessionId, VERBOSE)) {
|
|
95
98
|
ctx.ui.notify(t.text, "info");
|
|
96
99
|
}
|
|
97
100
|
lastSeenId = newestId(entries) ?? lastSeenId;
|
|
@@ -88,6 +88,20 @@ export interface ToastDecision {
|
|
|
88
88
|
* default. Lets the toast scope to a single interactive session.
|
|
89
89
|
*/
|
|
90
90
|
ompSessionId?: string;
|
|
91
|
+
/** The router's own decision trail, already written for people. */
|
|
92
|
+
reasons?: string[];
|
|
93
|
+
/** Classifier inputs; only a few are worth surfacing. */
|
|
94
|
+
features?: { promptTokens?: number; toolCount?: number; turnDepth?: number; isToolResultContinuation?: boolean } | null;
|
|
95
|
+
/** Attempt index within the turn; >0 means this served after an escalation. */
|
|
96
|
+
attempt?: number;
|
|
97
|
+
/** Prompt tokens compaction removed before dispatch. */
|
|
98
|
+
promptTokensSaved?: number;
|
|
99
|
+
/** What the router expected this to cost, before the upstream reported. */
|
|
100
|
+
predictedUsd?: number;
|
|
101
|
+
latencyMs?: number;
|
|
102
|
+
ttftMs?: number | null;
|
|
103
|
+
task?: string | null;
|
|
104
|
+
classificationSource?: string;
|
|
91
105
|
}
|
|
92
106
|
|
|
93
107
|
export interface ToastMessage {
|
|
@@ -110,10 +124,89 @@ export function providerOf(slug: string): { provider: string; model: string } {
|
|
|
110
124
|
return { provider: "openrouter", model: slug };
|
|
111
125
|
}
|
|
112
126
|
|
|
113
|
-
|
|
127
|
+
/**
|
|
128
|
+
* Which parts of the decision trail earn a line in a toast.
|
|
129
|
+
*
|
|
130
|
+
* The router writes many reasons per turn; a toast has room for two. These
|
|
131
|
+
* patterns are ordered by how much they change what the reader would do:
|
|
132
|
+
* a failover or a policy pin explains a surprising model outright, a hold or a
|
|
133
|
+
* rescue explains why the obvious cheaper pick was skipped, and the rest is
|
|
134
|
+
* ordinary ranking that the tier already conveys.
|
|
135
|
+
*/
|
|
136
|
+
const REASON_PRIORITY: readonly RegExp[] = [
|
|
137
|
+
/failover|escalat/i,
|
|
138
|
+
/policy|pin(ned)?|allow|deny/i,
|
|
139
|
+
/held|sticky|hysteresis|switch margin/i,
|
|
140
|
+
/budget raised|reasons before it answers/i,
|
|
141
|
+
// The RESCUE wording only: "cheapest above the quality floor" is ordinary
|
|
142
|
+
// ranking, and matching a bare "floor" would push it above real surprises.
|
|
143
|
+
/tier rescue|relaxed|adaptive (floor|ceiling)/i,
|
|
144
|
+
/cache|warm/i,
|
|
145
|
+
/compact/i,
|
|
146
|
+
/explor/i,
|
|
147
|
+
];
|
|
148
|
+
|
|
149
|
+
const clip = (text: string, max: number): string => (text.length > max ? `${text.slice(0, max - 1)}…` : text);
|
|
150
|
+
|
|
151
|
+
/** Up to `limit` reasons, most explanatory first, each trimmed for one line. */
|
|
152
|
+
export function whyReasons(reasons: readonly string[] | undefined, limit = 2): string[] {
|
|
153
|
+
if (reasons === undefined || reasons.length === 0) return [];
|
|
154
|
+
const picked: string[] = [];
|
|
155
|
+
const seen = new Set<number>();
|
|
156
|
+
for (const re of REASON_PRIORITY) {
|
|
157
|
+
for (let i = 0; i < reasons.length && picked.length < limit; i++) {
|
|
158
|
+
const r = reasons[i];
|
|
159
|
+
if (r === undefined || seen.has(i) || !re.test(r)) continue;
|
|
160
|
+
seen.add(i);
|
|
161
|
+
picked.push(clip(r.replace(/\s+/g, " ").trim(), 90));
|
|
162
|
+
}
|
|
163
|
+
if (picked.length >= limit) break;
|
|
164
|
+
}
|
|
165
|
+
// Nothing matched a pattern: the first reason is the ranking rationale itself.
|
|
166
|
+
if (picked.length === 0 && reasons[0] !== undefined) picked.push(clip(reasons[0].replace(/\s+/g, " ").trim(), 90));
|
|
167
|
+
return picked;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
const tokens = (n: number): string => (n >= 1000 ? `${Math.round(n / 100) / 10}k` : String(n));
|
|
171
|
+
|
|
172
|
+
/** The turn's shape in a few words: what the model was actually handed. */
|
|
173
|
+
export function factsOf(d: ToastDecision): string[] {
|
|
174
|
+
const out: string[] = [];
|
|
175
|
+
const f = d.features ?? undefined;
|
|
176
|
+
if (f?.promptTokens !== undefined && f.promptTokens > 0) out.push(`${tokens(f.promptTokens)} prompt`);
|
|
177
|
+
if (d.promptTokensSaved !== undefined && d.promptTokensSaved > 0) out.push(`${tokens(d.promptTokensSaved)} compacted`);
|
|
178
|
+
if (f?.toolCount !== undefined && f.toolCount > 0) out.push(`${f.toolCount} tools`);
|
|
179
|
+
if (f?.isToolResultContinuation === true) out.push("tool continuation");
|
|
180
|
+
if (d.attempt !== undefined && d.attempt > 0) out.push(`attempt ${d.attempt + 1}`);
|
|
181
|
+
if (d.task !== undefined && d.task !== null && d.task !== "" && d.task !== "coding") out.push(d.task);
|
|
182
|
+
if (d.ttftMs !== undefined && d.ttftMs !== null && d.ttftMs > 0) out.push(`${(d.ttftMs / 1000).toFixed(1)}s to first token`);
|
|
183
|
+
else if (d.latencyMs !== undefined && d.latencyMs > 0) out.push(`${(d.latencyMs / 1000).toFixed(1)}s`);
|
|
184
|
+
return out;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/** `$0.00042`, or the prediction when the upstream reported nothing. */
|
|
188
|
+
function costOf(d: ToastDecision): string {
|
|
189
|
+
if (d.reportedUsd !== null && d.reportedUsd !== undefined) return `$${d.reportedUsd.toFixed(5)}`;
|
|
190
|
+
if (d.predictedUsd !== undefined && d.predictedUsd > 0) return `~$${d.predictedUsd.toFixed(5)}`;
|
|
191
|
+
return "";
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* The toast body. `verbose` (the default) adds the decision trail and the
|
|
196
|
+
* turn's shape under the headline; `compact` is the original single line, for
|
|
197
|
+
* anyone who wants the model name and nothing else.
|
|
198
|
+
*/
|
|
199
|
+
export function toToastText(d: ToastDecision, verbose = true): string {
|
|
114
200
|
const { provider, model } = providerOf(d.servedSlug ?? d.slug);
|
|
115
|
-
const cost = d
|
|
116
|
-
|
|
201
|
+
const cost = costOf(d);
|
|
202
|
+
const head = `${provider} \u00b7 ${model} [${d.tier}]${cost === "" ? "" : ` \u00b7 ${cost}`}`;
|
|
203
|
+
if (!verbose) return head;
|
|
204
|
+
const lines = [head];
|
|
205
|
+
const why = whyReasons(d.reasons);
|
|
206
|
+
for (const [i, r] of why.entries()) lines.push(`${i === 0 ? "why: " : " "}${r}`);
|
|
207
|
+
const facts = factsOf(d);
|
|
208
|
+
if (facts.length > 0) lines.push(facts.join(" \u00b7 "));
|
|
209
|
+
return lines.join("\n");
|
|
117
210
|
}
|
|
118
211
|
|
|
119
212
|
/**
|
|
@@ -129,6 +222,7 @@ export function selectToasts(
|
|
|
129
222
|
lastSeenId: string | null,
|
|
130
223
|
harnessId = "",
|
|
131
224
|
ompSessionId = "",
|
|
225
|
+
verbose = true,
|
|
132
226
|
): ToastMessage[] {
|
|
133
227
|
if (lastSeenId === null) return [];
|
|
134
228
|
// `entries` is newest-first. Entries strictly newer than lastSeenId are the
|
|
@@ -143,7 +237,7 @@ export function selectToasts(
|
|
|
143
237
|
if (d.wasted) continue;
|
|
144
238
|
if (harnessId !== "" && d.harnessId !== harnessId) continue;
|
|
145
239
|
if (ompSessionId !== "" && d.ompSessionId !== ompSessionId) continue;
|
|
146
|
-
out.push({ model: d.servedSlug ?? d.slug, tier: d.tier, costUsd: d.reportedUsd, text: toToastText(d) });
|
|
240
|
+
out.push({ model: d.servedSlug ?? d.slug, tier: d.tier, costUsd: d.reportedUsd, text: toToastText(d, verbose) });
|
|
147
241
|
}
|
|
148
242
|
return out;
|
|
149
243
|
}
|
package/package.json
CHANGED
package/src/cli/config-wizard.ts
CHANGED
|
@@ -189,6 +189,7 @@ export const WIZARD_SECTIONS: readonly SectionSpec[] = [
|
|
|
189
189
|
{ path: "filters.trustScopedByHarness", label: "Scope trust per harness", kind: "boolean" },
|
|
190
190
|
{ path: "filters.trustWindowDays", label: "Trust window", kind: "number", min: 0, hint: "days, 0=all time" },
|
|
191
191
|
{ path: "filters.contextHeadroom", label: "Context headroom", kind: "number", min: 1 },
|
|
192
|
+
{ path: "filters.reasoningCompletionFloor", label: "Reasoning completion floor", kind: "number", min: 0 },
|
|
192
193
|
{ path: "filters.latencyWeight", label: "Latency weight", kind: "number", min: 0, hint: "0=ignore speed" },
|
|
193
194
|
{ path: "filters.latencyReferenceMs", label: "Latency reference TTFT", kind: "number", min: 1, hint: "ms" },
|
|
194
195
|
{ path: "filters.latencyReferenceTokensPerSec", label: "Latency reference speed", kind: "number", min: 1, hint: "tok/s" },
|
package/src/config/defaults.ts
CHANGED
|
@@ -114,6 +114,10 @@ export const DEFAULT_CONFIG: RouterConfig = {
|
|
|
114
114
|
// opt-in after a replay run prices it.
|
|
115
115
|
trustWindowDays: 0,
|
|
116
116
|
contextHeadroom: 1.25,
|
|
117
|
+
// Enough for a short chain of thought plus a brief answer. Measured against
|
|
118
|
+
// the failure it fixes: gpt-oss:20b and nemotron-3-nano returned empty at a
|
|
119
|
+
// 12-token cap, then failed over.
|
|
120
|
+
reasoningCompletionFloor: 512,
|
|
117
121
|
// Latency scoring is off by default (weight 0): opt in after establishing a
|
|
118
122
|
// baseline. Expected total wait (TTFT + expected completion / throughput)
|
|
119
123
|
// above the references inflates a model's effective cost.
|
package/src/config/schema.ts
CHANGED
|
@@ -95,6 +95,7 @@ const filters = z.strictObject({
|
|
|
95
95
|
trustScopedByHarness: z.boolean().optional(),
|
|
96
96
|
trustWindowDays: z.number().nonnegative().optional(),
|
|
97
97
|
contextHeadroom: z.number().positive().optional(),
|
|
98
|
+
reasoningCompletionFloor: z.number().int().nonnegative().optional(),
|
|
98
99
|
latencyWeight: z.number().nonnegative().optional(),
|
|
99
100
|
latencyReferenceMs: z.number().positive().optional(),
|
|
100
101
|
latencyReferenceTokensPerSec: z.number().positive().optional(),
|
package/src/config/types.ts
CHANGED
|
@@ -221,6 +221,21 @@ export interface FilterConfig {
|
|
|
221
221
|
includeFree: boolean;
|
|
222
222
|
/** Require `supported_parameters` to include `tools` whenever the request offers tools. */
|
|
223
223
|
requireToolSupport: boolean;
|
|
224
|
+
/**
|
|
225
|
+
* Smallest completion budget a REASONING model is dispatched with, tokens.
|
|
226
|
+
*
|
|
227
|
+
* A reasoning model spends the budget thinking before it answers, so a
|
|
228
|
+
* caller's tight cap returns nothing at all: omp asks for ~12 tokens for a
|
|
229
|
+
* conversation title, and `ollama/gpt-oss:20b` hit the cap having produced
|
|
230
|
+
* no content, which cost a dead dispatch and a failover to another model.
|
|
231
|
+
* When the chosen model reasons and the caller asked for less than this, the
|
|
232
|
+
* dispatch is raised to this floor (never above the model's own completion
|
|
233
|
+
* ceiling). Models that answer directly keep the caller's cap.
|
|
234
|
+
*
|
|
235
|
+
* A cap is an upper bound, not a target — a model that answers in ten tokens
|
|
236
|
+
* still stops at ten. 0 disables the floor.
|
|
237
|
+
*/
|
|
238
|
+
reasoningCompletionFloor: number;
|
|
224
239
|
/** Drop models whose ledger success rate is below this, once `minTrustSamples` is met. */
|
|
225
240
|
minTrust: number;
|
|
226
241
|
/**
|
package/src/router/select.ts
CHANGED
|
@@ -608,6 +608,18 @@ export function select(args: SelectArgs): Decision {
|
|
|
608
608
|
// The ceiling is a hard limit anyway; passing it explicitly also caps runaway completions.
|
|
609
609
|
maxTokens = maxTokens === undefined ? ceiling : Math.min(maxTokens, ceiling);
|
|
610
610
|
}
|
|
611
|
+
// A reasoning model spends the budget thinking before it answers, so a caller's
|
|
612
|
+
// tight cap returns nothing and the turn fails over having paid for the dispatch.
|
|
613
|
+
// Raise it to the floor for those models only; the ceiling still wins.
|
|
614
|
+
const floor = cfg.filters.reasoningCompletionFloor;
|
|
615
|
+
const thinksBeforeAnswering = chosen.model.reasoningMandatory || (chosen.model.supportsReasoning && reasoning !== "off");
|
|
616
|
+
if (floor > 0 && thinksBeforeAnswering && maxTokens !== undefined && maxTokens < floor) {
|
|
617
|
+
const raised = ceiling === undefined ? floor : Math.min(floor, ceiling);
|
|
618
|
+
if (raised > maxTokens) {
|
|
619
|
+
reasons.push(`completion budget raised ${maxTokens} → ${raised}: ${chosen.model.slug} reasons before it answers`);
|
|
620
|
+
maxTokens = raised;
|
|
621
|
+
}
|
|
622
|
+
}
|
|
611
623
|
const stripAssistantReasoning = !(chosen.model.supportsReasoning && REASONING_REPLAY_AUTHORS[chosen.model.author] === true);
|
|
612
624
|
|
|
613
625
|
// The recorded forecast is the EXPECTED price of this dispatch, not the
|
package/test/failover.test.ts
CHANGED
|
@@ -46,7 +46,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
|
|
|
46
46
|
data: { axis: "intelligence", minQuality: 0 },
|
|
47
47
|
chat: { axis: "intelligence", minQuality: 0 },
|
|
48
48
|
},
|
|
49
|
-
filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, feedbackWeight: 0, feedbackByTask: false, minTrustSamples: 5, trustScopedByHarness: false, trustWindowDays: 0, contextHeadroom: 1.2, latencyWeight: 0, latencyReferenceMs: 5000, latencyReferenceTokensPerSec: 30, cacheReliabilityMinSamples: 10, latencyMinSamples: 20, escalationCostWeight: 0 },
|
|
49
|
+
filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, feedbackWeight: 0, feedbackByTask: false, minTrustSamples: 5, trustScopedByHarness: false, trustWindowDays: 0, contextHeadroom: 1.2, reasoningCompletionFloor: 0, latencyWeight: 0, latencyReferenceMs: 5000, latencyReferenceTokensPerSec: 30, cacheReliabilityMinSamples: 10, latencyMinSamples: 20, escalationCostWeight: 0 },
|
|
50
50
|
classifier: {
|
|
51
51
|
ambiguityThreshold: 0,
|
|
52
52
|
model: "test/adjudicator", learnedModelPath: "",
|
package/test/select.test.ts
CHANGED
|
@@ -82,9 +82,11 @@ function run(opts: {
|
|
|
82
82
|
tier?: Tier;
|
|
83
83
|
ledger?: Ledger | null;
|
|
84
84
|
harnessId?: string;
|
|
85
|
+
maxTokens?: number;
|
|
85
86
|
}) {
|
|
86
87
|
const cfg = opts.cfg ?? BASE;
|
|
87
|
-
const
|
|
88
|
+
const base = request(opts.userText ?? "tidy the retry helper");
|
|
89
|
+
const req = opts.maxTokens === undefined ? base : { ...base, maxTokens: opts.maxTokens };
|
|
88
90
|
const features = extractFeatures(req, opts.promptTokens ?? 4000);
|
|
89
91
|
const heuristic = scoreHeuristic(features, cfg);
|
|
90
92
|
const classification = opts.tier === undefined ? heuristic : { ...heuristic, tier: opts.tier };
|
|
@@ -353,6 +355,31 @@ describe("decision shape", () => {
|
|
|
353
355
|
}
|
|
354
356
|
});
|
|
355
357
|
|
|
358
|
+
test("a reasoning model gets the completion floor; a direct one keeps the caller's cap", () => {
|
|
359
|
+
const usable = (m: (typeof MODELS)[number]): boolean => m.supportsTools && m.contextLength >= 32_000 && (m.maxCompletionTokens ?? 100_000) >= 4096;
|
|
360
|
+
const thinker = MODELS.find((m) => usable(m) && m.supportsReasoning);
|
|
361
|
+
const direct = MODELS.find((m) => usable(m) && !m.supportsReasoning && !m.reasoningMandatory);
|
|
362
|
+
expect(thinker).toBeDefined();
|
|
363
|
+
expect(direct).toBeDefined();
|
|
364
|
+
const withFloor = (slug: string, floor: number): RouterConfig => ({ ...BASE, filters: { ...BASE.filters, allow: [slug], reasoningCompletionFloor: floor } });
|
|
365
|
+
|
|
366
|
+
// omp asks for a dozen tokens for a title; a reasoning model would spend them thinking
|
|
367
|
+
// and return nothing, so the dispatch is raised.
|
|
368
|
+
const raised = run({ tier: "trivial", cfg: withFloor(thinker!.slug, 512), maxTokens: 12 });
|
|
369
|
+
expect(raised.slug).toBe(thinker!.slug);
|
|
370
|
+
expect(raised.maxTokens).toBe(512);
|
|
371
|
+
expect(raised.reasons.some((r) => r.includes("reasons before it answers"))).toBe(true);
|
|
372
|
+
|
|
373
|
+
// A model that answers directly is untouched: its cap is the caller's.
|
|
374
|
+
const kept = run({ tier: "trivial", cfg: withFloor(direct!.slug, 512), maxTokens: 12 });
|
|
375
|
+
expect(kept.slug).toBe(direct!.slug);
|
|
376
|
+
expect(kept.maxTokens).toBe(12);
|
|
377
|
+
|
|
378
|
+
// The floor never raises past what the caller already asked for, and 0 disables it.
|
|
379
|
+
expect(run({ tier: "trivial", cfg: withFloor(thinker!.slug, 512), maxTokens: 4000 }).maxTokens).toBe(4000);
|
|
380
|
+
expect(run({ tier: "trivial", cfg: withFloor(thinker!.slug, 0), maxTokens: 12 }).maxTokens).toBe(12);
|
|
381
|
+
});
|
|
382
|
+
|
|
356
383
|
test("plans a probe for cheap tiers and leaves the top tier unprobed", () => {
|
|
357
384
|
expect(run({ tier: "trivial" }).probe.enabled).toBe(true);
|
|
358
385
|
// Nothing above `hard` to escalate into, so probing it would only add latency.
|
package/test/toast-logic.test.ts
CHANGED
|
@@ -7,8 +7,10 @@ import {
|
|
|
7
7
|
newestId,
|
|
8
8
|
providerOf,
|
|
9
9
|
resolveRouterUrl,
|
|
10
|
+
factsOf,
|
|
10
11
|
selectToasts,
|
|
11
12
|
toToastText,
|
|
13
|
+
whyReasons,
|
|
12
14
|
type ToastDecision,
|
|
13
15
|
} from "../omp-extension/toast-logic.ts";
|
|
14
16
|
|
|
@@ -199,3 +201,62 @@ describe("toToastText", () => {
|
|
|
199
201
|
expect(providerOf("z-ai/glm-5.3-flash")).toEqual({ provider: "openrouter", model: "z-ai/glm-5.3-flash" });
|
|
200
202
|
});
|
|
201
203
|
});
|
|
204
|
+
|
|
205
|
+
describe("verbose toast", () => {
|
|
206
|
+
const FULL = dec({
|
|
207
|
+
slug: "ollama/gpt-oss:20b",
|
|
208
|
+
servedSlug: "ollama/gpt-oss:20b",
|
|
209
|
+
tier: "trivial",
|
|
210
|
+
reportedUsd: 0.000308,
|
|
211
|
+
reasons: [
|
|
212
|
+
"policy: pinned to ollama/gpt-oss:20b",
|
|
213
|
+
"completion budget raised 12 to 512: ollama/gpt-oss:20b reasons before it answers",
|
|
214
|
+
"cheapest above the quality floor",
|
|
215
|
+
],
|
|
216
|
+
features: { promptTokens: 22899, toolCount: 11, isToolResultContinuation: true },
|
|
217
|
+
attempt: 1,
|
|
218
|
+
promptTokensSaved: 12800,
|
|
219
|
+
ttftMs: 2100,
|
|
220
|
+
});
|
|
221
|
+
|
|
222
|
+
test("the headline keeps its shape and the body explains the choice", () => {
|
|
223
|
+
const lines = toToastText(FULL).split(String.fromCharCode(10));
|
|
224
|
+
expect(lines[0]).toBe("ollama · gpt-oss:20b [trivial] · $0.00031");
|
|
225
|
+
expect(lines[1]).toBe("why: policy: pinned to ollama/gpt-oss:20b");
|
|
226
|
+
expect(lines[2]).toContain("completion budget raised");
|
|
227
|
+
expect(lines[3]).toBe("22.9k prompt · 12.8k compacted · 11 tools · tool continuation · attempt 2 · 2.1s to first token");
|
|
228
|
+
});
|
|
229
|
+
|
|
230
|
+
test("compact is the old single line", () => {
|
|
231
|
+
expect(toToastText(FULL, false)).toBe("ollama · gpt-oss:20b [trivial] · $0.00031");
|
|
232
|
+
expect(toToastText(FULL, false).includes(String.fromCharCode(10))).toBe(false);
|
|
233
|
+
});
|
|
234
|
+
|
|
235
|
+
test("a surprise outranks ordinary ranking, and ranking shows when nothing surprised", () => {
|
|
236
|
+
// "cheapest above the quality floor" is ordinary: a failover and a hold win.
|
|
237
|
+
expect(whyReasons(["cheapest above the quality floor", "failover: x/y empty_completion; retrying a/b"])[0]).toContain("failover");
|
|
238
|
+
expect(whyReasons(["cheapest above the quality floor", "held from the previous turn: switch margin not cleared"])[0]).toContain("held");
|
|
239
|
+
// Nothing notable: the ranking rationale itself is the answer.
|
|
240
|
+
expect(whyReasons(["cheapest above the quality floor"])).toEqual(["cheapest above the quality floor"]);
|
|
241
|
+
expect(whyReasons([])).toEqual([]);
|
|
242
|
+
expect(whyReasons(undefined)).toEqual([]);
|
|
243
|
+
// At most two lines, however many reasons the router recorded.
|
|
244
|
+
expect(whyReasons(["failover: a", "policy: b", "held: c", "cache warm"]).length).toBe(2);
|
|
245
|
+
});
|
|
246
|
+
|
|
247
|
+
test("an unreported cost falls back to the prediction, and thin decisions stay short", () => {
|
|
248
|
+
expect(toToastText(dec({ reportedUsd: null, predictedUsd: 0.00042, reasons: [], features: null }))).toBe("openrouter · meta/muse-glimmer-30b [trivial] · ~$0.00042");
|
|
249
|
+
expect(toToastText(dec({ reportedUsd: null, features: null }))).toBe("openrouter · meta/muse-glimmer-30b [trivial]");
|
|
250
|
+
});
|
|
251
|
+
|
|
252
|
+
test("facts skip what a reader does not need", () => {
|
|
253
|
+
expect(factsOf(dec({ features: { promptTokens: 0, toolCount: 0 }, attempt: 0, task: "coding" }))).toEqual([]);
|
|
254
|
+
expect(factsOf(dec({ features: { promptTokens: 900 }, task: "vision" }))).toEqual(["900 prompt", "vision"]);
|
|
255
|
+
});
|
|
256
|
+
|
|
257
|
+
test("selectToasts renders compact when asked", () => {
|
|
258
|
+
const entries = [dec({ id: "d2", slug: "x/b", reasons: ["failover: nope"] }), dec({ id: "d1" })];
|
|
259
|
+
expect(selectToasts(entries, "d1", "", "", false)[0]?.text.includes(String.fromCharCode(10))).toBe(false);
|
|
260
|
+
expect(selectToasts(entries, "d1")[0]?.text.includes(String.fromCharCode(10))).toBe(true);
|
|
261
|
+
});
|
|
262
|
+
});
|
package/test/turn.test.ts
CHANGED
|
@@ -46,7 +46,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
|
|
|
46
46
|
data: { axis: "intelligence", minQuality: 0 },
|
|
47
47
|
chat: { axis: "intelligence", minQuality: 0 },
|
|
48
48
|
},
|
|
49
|
-
filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, feedbackWeight: 0, feedbackByTask: false, minTrustSamples: 5, trustScopedByHarness: false, trustWindowDays: 0, contextHeadroom: 1.2, latencyWeight: 0, latencyReferenceMs: 5000, latencyReferenceTokensPerSec: 30, cacheReliabilityMinSamples: 10, latencyMinSamples: 20, escalationCostWeight: 0 },
|
|
49
|
+
filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, feedbackWeight: 0, feedbackByTask: false, minTrustSamples: 5, trustScopedByHarness: false, trustWindowDays: 0, contextHeadroom: 1.2, reasoningCompletionFloor: 0, latencyWeight: 0, latencyReferenceMs: 5000, latencyReferenceTokensPerSec: 30, cacheReliabilityMinSamples: 10, latencyMinSamples: 20, escalationCostWeight: 0 },
|
|
50
50
|
classifier: {
|
|
51
51
|
ambiguityThreshold: 0,
|
|
52
52
|
model: "test/adjudicator", learnedModelPath: "",
|