auto-model-router 0.4.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.omp-plugin/marketplace.json +2 -2
- package/README.md +5 -2
- package/omp-extension/router-embed.ts +9 -6
- package/package.json +1 -1
- package/src/cli/config-wizard.ts +3 -0
- package/src/config/defaults.ts +8 -0
- package/src/config/schema.ts +3 -0
- package/src/config/types.ts +24 -0
- package/src/cost/ledger.ts +37 -5
- package/src/cost/report.ts +12 -0
- package/src/cost/types.ts +4 -1
- package/src/router/classify.ts +3 -0
- package/src/router/features.ts +39 -0
- package/src/router/index.ts +11 -4
- package/src/router/types.ts +8 -0
- package/src/wire/openai/request.ts +4 -0
- package/src/wire/types.ts +2 -0
- package/test/classify.test.ts +13 -0
- package/test/config-wizard.test.ts +6 -5
- package/test/controls.test.ts +16 -1
- package/test/escalate.test.ts +1 -0
- package/test/failover.test.ts +4 -3
- package/test/features.test.ts +32 -0
- package/test/http-resilience.test.ts +1 -1
- package/test/report-hub.test.ts +2 -0
- package/test/report.test.ts +14 -0
- package/test/trust-attribution.test.ts +53 -0
- package/test/turn.test.ts +4 -3
- package/tools/replay.ts +1 -0
|
@@ -7,14 +7,14 @@
|
|
|
7
7
|
},
|
|
8
8
|
"metadata": {
|
|
9
9
|
"description": "auto-model-router: a local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
|
|
10
|
-
"version": "0.4.
|
|
10
|
+
"version": "0.4.1",
|
|
11
11
|
"pluginRoot": "."
|
|
12
12
|
},
|
|
13
13
|
"plugins": [
|
|
14
14
|
{
|
|
15
15
|
"name": "auto-model-router",
|
|
16
16
|
"description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter. Runs in-process, routes per turn by price and task complexity, with budget caps, mid-stream escalation, and cache-aware hysteresis.",
|
|
17
|
-
"version": "0.4.
|
|
17
|
+
"version": "0.4.1",
|
|
18
18
|
"author": {
|
|
19
19
|
"name": "drewappling",
|
|
20
20
|
"email": "drewappling@gmail.com"
|
package/README.md
CHANGED
|
@@ -499,7 +499,7 @@ What it shows, for the window:
|
|
|
499
499
|
|
|
500
500
|
| Block | Columns |
|
|
501
501
|
| --- | --- |
|
|
502
|
-
| totals | spend, dispatches, conversations, $/dispatch, prompt and completion tokens, cache hit rate, model switches, escalations, failovers, errors (aborted separately) |
|
|
502
|
+
| totals | spend, dispatches, conversations, $/dispatch, prompt and completion tokens, cache hit rate, model switches, escalations, failovers, errors (aborted separately), subagent dispatches and their share of spend |
|
|
503
503
|
| prompt anatomy | mean share of prompt bytes by role (tool results, assistant, user, system), tool schemas beside them, the older half of the conversation, and tool results older than the newest 20 messages — what compaction can reach. Recorded per turn from v0.3.5. |
|
|
504
504
|
| providers | per upstream (`openrouter`, `ollama`): dispatches, spend, share, cache hit, mean TTFT, tokens/s, escalations, errors |
|
|
505
505
|
| models | per served slug (top 12 by spend): the same plus user feedback (`+good/-bad` from `/router good\|bad`) and the tier mix it was routed for |
|
|
@@ -592,6 +592,7 @@ what each one does. All values are optional; omit a key to use its default.
|
|
|
592
592
|
| `host` | `127.0.0.1` | Bind address. `0.0.0.0`/`::` listen on all interfaces (the provider still advertises loopback). |
|
|
593
593
|
| `port` | `0` | Bind port. `0` = let the OS pick a free ephemeral port (the embedded router's default). |
|
|
594
594
|
| `apiKey` | unset | Optional client bearer token. When set, every request must send `Authorization: Bearer <key>`. |
|
|
595
|
+
| `subagentProfile` | `auto-sub` | Profile omp subagents are routed under when they ask for the default one. The embed extension marks sessions without a UI with `X-Omp-Subagent: 1`; delegated work (reads, searches, summaries) never needs the top tier. Empty disables the remap. |
|
|
595
596
|
| `harnessId` | unset | Harness identity sent as `X-Omp-Harness`; scopes per-harness daily budgets and toasts. |
|
|
596
597
|
|
|
597
598
|
### `openrouter`
|
|
@@ -663,6 +664,7 @@ Each task (`coding`, `vision`, `documentation`, `data`, `chat`) is a
|
|
|
663
664
|
| `deny` | `[]` | Glob denylist; matching slugs are excluded. |
|
|
664
665
|
| `includeFree` | `false` | Include free models (rate-limited hard; usually excluded). |
|
|
665
666
|
| `requireToolSupport` | `true` | Only models that support tool calls. |
|
|
667
|
+
| `feedbackWeight` | `0` | How much a `/router good\|bad` verdict weighs in a model's trust rate: a bad verdict counts as this many failures, a good one as this many successes. `0` records verdicts without acting on them. |
|
|
666
668
|
| `minTrust` | `0.7` | Minimum success rate; models below this (after `minTrustSamples`) are demoted. |
|
|
667
669
|
| `minTrustSamples` | `12` | Attempts before trust is enforced. |
|
|
668
670
|
| `trustScopedByHarness` | `false` | `true` = each harness reads only its own trust rows. |
|
|
@@ -688,6 +690,7 @@ Each task (`coding`, `vision`, `documentation`, `data`, `chat`) is a
|
|
|
688
690
|
| `toolAxis` | `coding` | Quality axis for tool-heavy turns. |
|
|
689
691
|
| `chatAxis` | `intelligence` | Quality axis for chat turns. |
|
|
690
692
|
| `agenticLoopDepth` | `3` | Tool-loop depth at which a turn is treated as agentic. |
|
|
693
|
+
| `readOnlyToolWeight` | `0` | Score subtracted when a tool-result continuation follows an assistant turn that used only read-only tools (read, grep, glob, ls, lsp…). Recorded as `features.readOnlyToolTail` either way; enable after `tools/replay.ts` prices it. |
|
|
691
694
|
| `mechanicalRetryFactor` | `0.2` | Fraction of the failed-tool and circular-call weights kept on a tool-result continuation; `1` disables the damping. |
|
|
692
695
|
|
|
693
696
|
### `escalation` — mid-stream retry upward
|
|
@@ -767,7 +770,7 @@ Each profile is a complete entry (arrays replace wholesale):
|
|
|
767
770
|
|
|
768
771
|
| Key | Default | Meaning |
|
|
769
772
|
| --- | --- | --- |
|
|
770
|
-
| `id` | `auto` / `auto-cheap` / `auto-max` | Model id omp selects. |
|
|
773
|
+
| `id` | `auto` / `auto-cheap` / `auto-max` / `auto-sub` | Model id omp selects. `auto-sub` (trivial..moderate) is what subagents get via `server.subagentProfile`. |
|
|
771
774
|
| `name` | `Auto (auto-model-router)` etc. | Display name. |
|
|
772
775
|
| `minTier` / `maxTier` | `trivial`/`hard`, `trivial`/`simple`, `moderate`/`hard` | Tier envelope. |
|
|
773
776
|
| `contextWindow` | `400000` | Advertised context window (drives omp's compaction). |
|
|
@@ -89,7 +89,7 @@ function trackProcessExit(): void {
|
|
|
89
89
|
* Registers the auto-model-router provider (and its virtual models) into omp's model
|
|
90
90
|
* registry at a specific bound port.
|
|
91
91
|
*/
|
|
92
|
-
function registerRouterProvider(pi: ExtensionAPI, port: number, cfg: RouterConfig, sessionId: string): void {
|
|
92
|
+
function registerRouterProvider(pi: ExtensionAPI, port: number, cfg: RouterConfig, sessionId: string, subagent: boolean): void {
|
|
93
93
|
// cwd is omp's workspace, which is what the agentdox scope is derived from
|
|
94
94
|
// when none is configured explicitly.
|
|
95
95
|
const providerConfig = buildProviderConfig(port, cfg, process.cwd());
|
|
@@ -100,6 +100,9 @@ function registerRouterProvider(pi: ExtensionAPI, port: number, cfg: RouterConfi
|
|
|
100
100
|
// Per-session scoping: lets the toast surface only this session's decisions
|
|
101
101
|
// even when several omp sessions share one embedded router's ledger.
|
|
102
102
|
if (sessionId !== "") headers["X-Omp-Session"] = sessionId;
|
|
103
|
+
// A session without a UI is a subagent (or a headless run): the router
|
|
104
|
+
// routes its turns under server.subagentProfile.
|
|
105
|
+
if (subagent) headers["X-Omp-Subagent"] = "1";
|
|
103
106
|
// Which agentdox project's shared context this workspace's turns draw on.
|
|
104
107
|
if (providerConfig.agentdoxScope !== undefined && providerConfig.agentdoxScope !== "") {
|
|
105
108
|
headers["X-Agentdox-Scope"] = providerConfig.agentdoxScope;
|
|
@@ -171,7 +174,7 @@ export default function (pi: ExtensionAPI): void {
|
|
|
171
174
|
// second bind would take a different port and orphan every model handle
|
|
172
175
|
// omp already resolved against the first one.
|
|
173
176
|
if (app !== null && boundPort !== null) {
|
|
174
|
-
registerRouterProvider(pi, boundPort, cfg, sessionId);
|
|
177
|
+
registerRouterProvider(pi, boundPort, cfg, sessionId, !ctx.hasUI);
|
|
175
178
|
return;
|
|
176
179
|
}
|
|
177
180
|
|
|
@@ -181,7 +184,7 @@ export default function (pi: ExtensionAPI): void {
|
|
|
181
184
|
// The main writes the port file before spawning subagents.
|
|
182
185
|
const shared = readEmbedPort(portFile);
|
|
183
186
|
if (shared !== null && (await probeEmbed(shared))) {
|
|
184
|
-
registerRouterProvider(pi, shared, cfg, sessionId);
|
|
187
|
+
registerRouterProvider(pi, shared, cfg, sessionId, !ctx.hasUI);
|
|
185
188
|
return;
|
|
186
189
|
}
|
|
187
190
|
// No live interactive session (headless batch runs, CI, the
|
|
@@ -193,7 +196,7 @@ export default function (pi: ExtensionAPI): void {
|
|
|
193
196
|
if (started.server.port === undefined) return;
|
|
194
197
|
app = started;
|
|
195
198
|
boundPort = started.server.port;
|
|
196
|
-
registerRouterProvider(pi, boundPort, cfg, sessionId);
|
|
199
|
+
registerRouterProvider(pi, boundPort, cfg, sessionId, !ctx.hasUI);
|
|
197
200
|
return;
|
|
198
201
|
}
|
|
199
202
|
|
|
@@ -207,7 +210,7 @@ export default function (pi: ExtensionAPI): void {
|
|
|
207
210
|
// default — never take this path, so sessions stay independent.
|
|
208
211
|
if (requestedPort !== 0 && (await probeEmbed(requestedPort))) {
|
|
209
212
|
writeEmbedPort(portFile, requestedPort);
|
|
210
|
-
registerRouterProvider(pi, requestedPort, cfg, sessionId);
|
|
213
|
+
registerRouterProvider(pi, requestedPort, cfg, sessionId, !ctx.hasUI);
|
|
211
214
|
pi.setLabel(`auto-model-router embed (shared :${requestedPort})`);
|
|
212
215
|
return;
|
|
213
216
|
}
|
|
@@ -266,7 +269,7 @@ export default function (pi: ExtensionAPI): void {
|
|
|
266
269
|
|
|
267
270
|
// Register BEFORE any await: everything omp resolves after this point
|
|
268
271
|
// picks up the live URL, so the registration must not sit behind I/O.
|
|
269
|
-
registerRouterProvider(pi, actualPort, cfg, sessionId);
|
|
272
|
+
registerRouterProvider(pi, actualPort, cfg, sessionId, !ctx.hasUI);
|
|
270
273
|
pi.setLabel(`auto-model-router embed :${actualPort}`);
|
|
271
274
|
|
|
272
275
|
// NO `session_shutdown` teardown. That event is emitted from session
|
package/package.json
CHANGED
package/src/cli/config-wizard.ts
CHANGED
|
@@ -114,6 +114,7 @@ export const WIZARD_SECTIONS: readonly SectionSpec[] = [
|
|
|
114
114
|
{ path: "server.port", label: "Listen port", kind: "number", min: 1, max: 65535 },
|
|
115
115
|
{ path: "server.apiKey", label: "Client bearer token", kind: "string", optional: true, secret: true },
|
|
116
116
|
{ path: "server.harnessId", label: "Default harness id", kind: "string", optional: true },
|
|
117
|
+
{ path: "server.subagentProfile", label: "Subagent profile", kind: "string", optional: true, hint: "profile id for omp subagents; blank = none" },
|
|
117
118
|
{ path: "server.maxConcurrentTurns", label: "Max concurrent turns", kind: "number", min: 1, hint: "per process, all sessions" },
|
|
118
119
|
],
|
|
119
120
|
},
|
|
@@ -182,6 +183,7 @@ export const WIZARD_SECTIONS: readonly SectionSpec[] = [
|
|
|
182
183
|
{ path: "filters.includeFree", label: "Include free models", kind: "boolean" },
|
|
183
184
|
{ path: "filters.requireToolSupport", label: "Require tool support", kind: "boolean" },
|
|
184
185
|
{ path: "filters.minTrust", label: "Min trust", kind: "number", min: 0, max: 1 },
|
|
186
|
+
{ path: "filters.feedbackWeight", label: "Feedback weight in trust", kind: "number", min: 0, hint: "0=record only; a bad verdict = this many failures" },
|
|
185
187
|
{ path: "filters.minTrustSamples", label: "Min trust samples", kind: "number", min: 0 },
|
|
186
188
|
{ path: "filters.trustScopedByHarness", label: "Scope trust per harness", kind: "boolean" },
|
|
187
189
|
{ path: "filters.trustWindowDays", label: "Trust window", kind: "number", min: 0, hint: "days, 0=all time" },
|
|
@@ -210,6 +212,7 @@ export const WIZARD_SECTIONS: readonly SectionSpec[] = [
|
|
|
210
212
|
{ path: "classifier.chatAxis", label: "Chat axis", kind: "enum", options: AXES },
|
|
211
213
|
{ path: "classifier.agenticLoopDepth", label: "Agentic loop depth", kind: "number", min: 0, hint: "tool rounds before damping" },
|
|
212
214
|
{ path: "classifier.mechanicalRetryFactor", label: "Mechanical retry factor", kind: "number", min: 0, max: 1 },
|
|
215
|
+
{ path: "classifier.readOnlyToolWeight", label: "Read-only tool loop weight", kind: "number", min: 0, hint: "0=record only" },
|
|
213
216
|
{ path: "classifier.reasoningWeights.medium", label: "Reasoning weight: medium", kind: "number", min: 0 },
|
|
214
217
|
{ path: "classifier.reasoningWeights.high", label: "Reasoning weight: high", kind: "number", min: 0 },
|
|
215
218
|
{ path: "classifier.reasoningWeights.xhigh", label: "Reasoning weight: xhigh", kind: "number", min: 0 },
|
package/src/config/defaults.ts
CHANGED
|
@@ -15,6 +15,8 @@ export const DEFAULT_CONFIG: RouterConfig = {
|
|
|
15
15
|
// is deterministic, so peers reuse it), so this covers N sessions plus
|
|
16
16
|
// their subagents. Was effectively 8 per session when each bound its own.
|
|
17
17
|
maxConcurrentTurns: 24,
|
|
18
|
+
// omp subagents (no UI) route under this profile: delegated work, capped at moderate.
|
|
19
|
+
subagentProfile: "auto-sub",
|
|
18
20
|
},
|
|
19
21
|
openrouter: {
|
|
20
22
|
baseUrl: "https://openrouter.ai/api/v1",
|
|
@@ -99,6 +101,8 @@ export const DEFAULT_CONFIG: RouterConfig = {
|
|
|
99
101
|
includeFree: false,
|
|
100
102
|
requireToolSupport: true,
|
|
101
103
|
minTrust: 0.7,
|
|
104
|
+
// Verdicts are recorded and reported first; weigh them once there are some.
|
|
105
|
+
feedbackWeight: 0,
|
|
102
106
|
minTrustSamples: 12,
|
|
103
107
|
// Shared trust by default: more samples, demotion guard stays effective
|
|
104
108
|
// even with a tiny guardrail-narrowed catalog.
|
|
@@ -142,6 +146,8 @@ export const DEFAULT_CONFIG: RouterConfig = {
|
|
|
142
146
|
// A mechanical retry (failed tool call + tool-result continuation) keeps
|
|
143
147
|
// only a fifth of the +0.26; a user-visible failure keeps the full weight.
|
|
144
148
|
mechanicalRetryFactor: 0.2,
|
|
149
|
+
// Recorded, not acted on, until replay prices it. See ClassifierConfig.readOnlyToolWeight.
|
|
150
|
+
readOnlyToolWeight: 0,
|
|
145
151
|
// Shipped reasoning values, unchanged. See ClassifierConfig.reasoningWeights:
|
|
146
152
|
// a harness that pins the level for a whole session turns these into a
|
|
147
153
|
// constant tier offset, in which case `medium` belongs near 0.
|
|
@@ -298,6 +304,8 @@ export const DEFAULT_CONFIG: RouterConfig = {
|
|
|
298
304
|
{ id: "auto", name: "Auto (auto-model-router)", minTier: "trivial", maxTier: "hard", contextWindow: 400_000, maxTokens: 32_000 },
|
|
299
305
|
{ id: "auto-cheap", name: "Auto Cheap (auto-model-router)", minTier: "trivial", maxTier: "simple", contextWindow: 400_000, maxTokens: 32_000 },
|
|
300
306
|
{ id: "auto-max", name: "Auto Max (auto-model-router)", minTier: "moderate", maxTier: "hard", contextWindow: 400_000, maxTokens: 32_000 },
|
|
307
|
+
// Subagent envelope (server.subagentProfile): never the top tier for delegated work.
|
|
308
|
+
{ id: "auto-sub", name: "Auto Subagent (auto-model-router)", minTier: "trivial", maxTier: "moderate", contextWindow: 400_000, maxTokens: 32_000 },
|
|
301
309
|
],
|
|
302
310
|
ledger: {
|
|
303
311
|
// Resolved by loadConfig: empty ⇒ `$AUTO_MODEL_ROUTER_HOME/router.db`.
|
package/src/config/schema.ts
CHANGED
|
@@ -20,6 +20,7 @@ const server = z.strictObject({
|
|
|
20
20
|
port: z.number().int().min(0).max(65_535).optional(),
|
|
21
21
|
apiKey: z.string().optional(),
|
|
22
22
|
harnessId: z.string().optional(),
|
|
23
|
+
subagentProfile: z.string().optional(),
|
|
23
24
|
maxConcurrentTurns: z.number().int().positive().max(1_000).optional(),
|
|
24
25
|
});
|
|
25
26
|
|
|
@@ -88,6 +89,7 @@ const filters = z.strictObject({
|
|
|
88
89
|
includeFree: z.boolean().optional(),
|
|
89
90
|
requireToolSupport: z.boolean().optional(),
|
|
90
91
|
minTrust: z.number().min(0).max(1).optional(),
|
|
92
|
+
feedbackWeight: z.number().nonnegative().optional(),
|
|
91
93
|
minTrustSamples: z.number().int().nonnegative().optional(),
|
|
92
94
|
trustScopedByHarness: z.boolean().optional(),
|
|
93
95
|
trustWindowDays: z.number().nonnegative().optional(),
|
|
@@ -114,6 +116,7 @@ const classifier = z.strictObject({
|
|
|
114
116
|
chatAxis: qualityAxis.optional(),
|
|
115
117
|
agenticLoopDepth: z.number().int().nonnegative().optional(),
|
|
116
118
|
mechanicalRetryFactor: z.number().min(0).max(1).optional(),
|
|
119
|
+
readOnlyToolWeight: z.number().nonnegative().optional(),
|
|
117
120
|
reasoningWeights: z
|
|
118
121
|
.strictObject({
|
|
119
122
|
medium: z.number().nonnegative().optional(),
|
package/src/config/types.ts
CHANGED
|
@@ -38,6 +38,14 @@ export interface ServerConfig {
|
|
|
38
38
|
* ⇒ no header (single-harness default).
|
|
39
39
|
*/
|
|
40
40
|
harnessId?: string;
|
|
41
|
+
/**
|
|
42
|
+
* Profile that requests from omp subagents (`X-Omp-Subagent: 1`, set by
|
|
43
|
+
* the embed extension for sessions without a UI) are routed under when they
|
|
44
|
+
* ask for the default profile. Subagents do delegated, bounded work — file
|
|
45
|
+
* reads, searches, summaries — that rarely needs the top tier. Empty
|
|
46
|
+
* disables the remap; a name with no matching profile is ignored.
|
|
47
|
+
*/
|
|
48
|
+
subagentProfile: string;
|
|
41
49
|
/**
|
|
42
50
|
* Concurrent in-flight turns this router process will accept; excess gets a
|
|
43
51
|
* 429 rather than being queued, so a local flood cannot pile up unbounded
|
|
@@ -215,6 +223,15 @@ export interface FilterConfig {
|
|
|
215
223
|
requireToolSupport: boolean;
|
|
216
224
|
/** Drop models whose ledger success rate is below this, once `minTrustSamples` is met. */
|
|
217
225
|
minTrust: number;
|
|
226
|
+
/**
|
|
227
|
+
* How much a user verdict (/router good|bad) weighs in a model's trust
|
|
228
|
+
* rate: each bad verdict counts as this many failures and each good one as
|
|
229
|
+
* this many successes, beside escalations and errors. 0 (default) records
|
|
230
|
+
* verdicts without acting on them. A person judging an answer wrong is a
|
|
231
|
+
* stronger signal than a probe rejection, so values of 2-5 are sensible
|
|
232
|
+
* once a week of verdicts is in the report.
|
|
233
|
+
*/
|
|
234
|
+
feedbackWeight: number;
|
|
218
235
|
/** Attempts required before `minTrust` is enforced against a model. */
|
|
219
236
|
minTrustSamples: number;
|
|
220
237
|
/**
|
|
@@ -347,6 +364,13 @@ export interface ClassifierConfig {
|
|
|
347
364
|
* loops buy the hard tier. 1 preserves the shipped behaviour.
|
|
348
365
|
*/
|
|
349
366
|
mechanicalRetryFactor: number;
|
|
367
|
+
/**
|
|
368
|
+
* Score subtracted when the newest assistant turn issued only read-only
|
|
369
|
+
* tools (read, grep, glob, ls, lsp…) and this is the tool-result
|
|
370
|
+
* continuation: the model is looking, not deciding. 0 (default) records
|
|
371
|
+
* the feature without acting on it — enable after a replay prices it.
|
|
372
|
+
*/
|
|
373
|
+
readOnlyToolWeight: number;
|
|
350
374
|
/**
|
|
351
375
|
* Score added when the CLIENT asks for a reasoning effort, per level. The
|
|
352
376
|
* premise is that asking for reasoning states expected difficulty directly.
|
package/src/cost/ledger.ts
CHANGED
|
@@ -94,6 +94,15 @@ interface TrustRow {
|
|
|
94
94
|
mean_cost_error: number | null;
|
|
95
95
|
}
|
|
96
96
|
|
|
97
|
+
interface FeedbackRow {
|
|
98
|
+
good: number | null;
|
|
99
|
+
bad: number | null;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** Verdict counts per served slug since a cutoff (optionally one harness). */
|
|
103
|
+
const FEEDBACK_SELECT = `COALESCE(SUM(CASE WHEN f.verdict = 'good' THEN 1 ELSE 0 END), 0) AS good,
|
|
104
|
+
COALESCE(SUM(CASE WHEN f.verdict = 'bad' THEN 1 ELSE 0 END), 0) AS bad`;
|
|
105
|
+
|
|
97
106
|
interface LatencyRow {
|
|
98
107
|
samples: number;
|
|
99
108
|
ttft_ms: number | null;
|
|
@@ -186,15 +195,23 @@ function errorKindOf(error: string | null): string | null {
|
|
|
186
195
|
return error.slice(0, sep);
|
|
187
196
|
}
|
|
188
197
|
|
|
189
|
-
function toTrust(slug: string, row: TrustRow): ModelTrust {
|
|
198
|
+
function toTrust(slug: string, row: TrustRow, fb: FeedbackRow | null = null, feedbackWeight = 0): ModelTrust {
|
|
190
199
|
// Laplace smoothing: an untried model scores a neutral 1/2, and a failure
|
|
191
200
|
// is an attempt superseded by an escalation or ended in an upstream error.
|
|
201
|
+
// A user verdict counts as feedbackWeight extra attempts of that outcome.
|
|
202
|
+
const good = fb?.good ?? 0;
|
|
203
|
+
const bad = fb?.bad ?? 0;
|
|
204
|
+
const w = feedbackWeight > 0 ? feedbackWeight : 0;
|
|
205
|
+
const attempts = row.attempts + w * (good + bad);
|
|
206
|
+
const failures = row.failures + w * bad;
|
|
192
207
|
return {
|
|
193
208
|
slug,
|
|
194
209
|
attempts: row.attempts,
|
|
195
210
|
escalations: row.escalations,
|
|
196
211
|
errors: row.errors,
|
|
197
|
-
|
|
212
|
+
feedbackGood: good,
|
|
213
|
+
feedbackBad: bad,
|
|
214
|
+
successRate: (attempts - failures + 1) / (attempts + 2),
|
|
198
215
|
meanCostError: row.mean_cost_error ?? 0,
|
|
199
216
|
};
|
|
200
217
|
}
|
|
@@ -312,6 +329,17 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
|
|
|
312
329
|
const trustStmt = db.query(`SELECT ${TRUST_SELECT} FROM ledger WHERE slug = ? AND created_at_ms > ?`);
|
|
313
330
|
const trustHarnessStmt = db.query(`SELECT ${TRUST_SELECT} FROM ledger WHERE slug = ? AND harness_id = ? AND created_at_ms > ?`);
|
|
314
331
|
const allTrustStmt = db.query(`SELECT slug, ${TRUST_SELECT} FROM ledger WHERE created_at_ms > ? GROUP BY slug`);
|
|
332
|
+
const feedbackStmt = db.query(`SELECT ${FEEDBACK_SELECT} FROM feedback f WHERE f.slug = ? AND f.created_at_ms > ?`);
|
|
333
|
+
const feedbackHarnessStmt = db.query(
|
|
334
|
+
`SELECT ${FEEDBACK_SELECT} FROM feedback f JOIN ledger l ON l.id = f.ledger_id WHERE f.slug = ? AND l.harness_id = ? AND f.created_at_ms > ?`,
|
|
335
|
+
);
|
|
336
|
+
const allFeedbackStmt = db.query(`SELECT f.slug, ${FEEDBACK_SELECT} FROM feedback f WHERE f.created_at_ms > ? GROUP BY f.slug`);
|
|
337
|
+
const feedbackFor = (slug: string, harnessId: string | undefined, cutoff: number): FeedbackRow | null => {
|
|
338
|
+
if (cfg.filters.feedbackWeight <= 0) return null;
|
|
339
|
+
return harnessId !== undefined && harnessId !== ""
|
|
340
|
+
? (feedbackHarnessStmt.get(slug, harnessId, cutoff) as FeedbackRow | null)
|
|
341
|
+
: (feedbackStmt.get(slug, cutoff) as FeedbackRow | null);
|
|
342
|
+
};
|
|
315
343
|
const latencyStmt = db.query(
|
|
316
344
|
`SELECT ${LATENCY_SELECT} FROM (SELECT * FROM ledger WHERE slug = ? ORDER BY created_at_ms DESC LIMIT ${LATENCY_WINDOW_ROWS})`,
|
|
317
345
|
);
|
|
@@ -447,13 +475,17 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
|
|
|
447
475
|
? (trustHarnessStmt.get(slug, harnessId, cutoff) as TrustRow | null)
|
|
448
476
|
: (trustStmt.get(slug, cutoff) as TrustRow | null);
|
|
449
477
|
if (row === null || row.attempts === 0) return null;
|
|
450
|
-
return toTrust(slug, row);
|
|
478
|
+
return toTrust(slug, row, feedbackFor(slug, harnessId, cutoff), cfg.filters.feedbackWeight);
|
|
451
479
|
},
|
|
452
480
|
|
|
453
481
|
allTrust(): ModelTrust[] {
|
|
454
482
|
const cutoff = cfg.filters.trustWindowDays > 0 ? Date.now() - cfg.filters.trustWindowDays * DAY_MS : 0;
|
|
455
483
|
const rows = allTrustStmt.all(cutoff) as (TrustRow & { slug: string })[];
|
|
456
|
-
|
|
484
|
+
const fb = new Map<string, FeedbackRow>();
|
|
485
|
+
if (cfg.filters.feedbackWeight > 0) {
|
|
486
|
+
for (const r of allFeedbackStmt.all(cutoff) as (FeedbackRow & { slug: string })[]) fb.set(r.slug, r);
|
|
487
|
+
}
|
|
488
|
+
return rows.map((row) => toTrust(row.slug, row, fb.get(row.slug) ?? null, cfg.filters.feedbackWeight));
|
|
457
489
|
},
|
|
458
490
|
|
|
459
491
|
latency(slug: string, harnessId?: string): ModelLatency | null {
|
|
@@ -476,7 +508,7 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
|
|
|
476
508
|
? (latencyHarnessStmt.get(slug, harnessId) as LatencyRow | null)
|
|
477
509
|
: (latencyStmt.get(slug) as LatencyRow | null);
|
|
478
510
|
out.set(slug, {
|
|
479
|
-
trust: trustRow === null || trustRow.attempts === 0 ? null : toTrust(slug, trustRow),
|
|
511
|
+
trust: trustRow === null || trustRow.attempts === 0 ? null : toTrust(slug, trustRow, feedbackFor(slug, harnessId, cutoff), cfg.filters.feedbackWeight),
|
|
480
512
|
latency: latencyRow === null ? null : toLatency(slug, latencyRow),
|
|
481
513
|
});
|
|
482
514
|
}
|
package/src/cost/report.ts
CHANGED
|
@@ -27,6 +27,9 @@ export interface ReportTotals {
|
|
|
27
27
|
modelSwitches: number;
|
|
28
28
|
/** Any row in the window carries an estimated cache count. */
|
|
29
29
|
cacheEstimated: boolean;
|
|
30
|
+
/** Turns from omp subagents (`features.isSubagent`), and their spend. */
|
|
31
|
+
subagentDispatches: number;
|
|
32
|
+
subagentSpendUsd: number;
|
|
30
33
|
}
|
|
31
34
|
|
|
32
35
|
export interface ReportRow {
|
|
@@ -196,6 +199,8 @@ export function buildUsageReport(
|
|
|
196
199
|
COALESCE(SUM(${CT}), 0) AS cached_tokens,
|
|
197
200
|
COALESCE(SUM(${COMP}), 0) AS completion_tokens,
|
|
198
201
|
SUM(CASE WHEN ${EST} THEN 1 ELSE 0 END) AS estimated_rows,
|
|
202
|
+
SUM(CASE WHEN json_extract(features, '$.isSubagent') = 1 THEN 1 ELSE 0 END) AS subagent_rows,
|
|
203
|
+
COALESCE(SUM(CASE WHEN json_extract(features, '$.isSubagent') = 1 THEN ${USD} ELSE 0 END), 0) AS subagent_spend,
|
|
199
204
|
SUM(CASE WHEN escalation_signal IS NOT NULL THEN 1 ELSE 0 END) AS escalations,
|
|
200
205
|
SUM(CASE WHEN instr(reasons, 'failover:') > 0 THEN 1 ELSE 0 END) AS failovers,
|
|
201
206
|
SUM(CASE WHEN error IS NOT NULL THEN 1 ELSE 0 END) AS errors,
|
|
@@ -210,6 +215,8 @@ export function buildUsageReport(
|
|
|
210
215
|
cached_tokens: number;
|
|
211
216
|
completion_tokens: number;
|
|
212
217
|
estimated_rows: number | null;
|
|
218
|
+
subagent_rows: number | null;
|
|
219
|
+
subagent_spend: number;
|
|
213
220
|
escalations: number | null;
|
|
214
221
|
failovers: number | null;
|
|
215
222
|
errors: number | null;
|
|
@@ -327,6 +334,8 @@ export function buildUsageReport(
|
|
|
327
334
|
aborted: t.aborted ?? 0,
|
|
328
335
|
modelSwitches: switches,
|
|
329
336
|
cacheEstimated: (t.estimated_rows ?? 0) > 0,
|
|
337
|
+
subagentDispatches: t.subagent_rows ?? 0,
|
|
338
|
+
subagentSpendUsd: t.subagent_spend,
|
|
330
339
|
},
|
|
331
340
|
providers,
|
|
332
341
|
models,
|
|
@@ -391,6 +400,9 @@ export function reportView(r: UsageReport, opts: { maxModels?: number } = {}): R
|
|
|
391
400
|
.join(" · ")}`,
|
|
392
401
|
);
|
|
393
402
|
}
|
|
403
|
+
if (t.subagentDispatches > 0) {
|
|
404
|
+
summary.push(`subagents: ${num(t.subagentDispatches)} dispatches, ${usd(t.subagentSpendUsd)} (${pct(t.spendUsd > 0 ? t.subagentSpendUsd / t.spendUsd : 0)} of spend)`);
|
|
405
|
+
}
|
|
394
406
|
const a = r.anatomy;
|
|
395
407
|
if (a !== null) {
|
|
396
408
|
summary.push(
|
package/src/cost/types.ts
CHANGED
|
@@ -177,8 +177,11 @@ export interface ModelTrust {
|
|
|
177
177
|
escalations: number;
|
|
178
178
|
/** Attempts that ended in an upstream error. */
|
|
179
179
|
errors: number;
|
|
180
|
-
/** Laplace-smoothed success rate, 0-1. */
|
|
180
|
+
/** Laplace-smoothed success rate, 0-1; user verdicts weigh in at filters.feedbackWeight. */
|
|
181
181
|
successRate: number;
|
|
182
|
+
/** User verdicts in the window (/router good|bad). */
|
|
183
|
+
feedbackGood?: number;
|
|
184
|
+
feedbackBad?: number;
|
|
182
185
|
/** Mean absolute relative prediction error, for forecast calibration. */
|
|
183
186
|
meanCostError: number;
|
|
184
187
|
}
|
package/src/router/classify.ts
CHANGED
|
@@ -166,6 +166,9 @@ export function scoreHeuristic(f: Features, cfg: RouterConfig): Classification {
|
|
|
166
166
|
// the served model must still accept image input — is enforced separately
|
|
167
167
|
// on `req.hasImages` in candidate selection, exactly as `classifyTask` does.
|
|
168
168
|
if (f.hasNewImage) add(W_IMAGES, "new image input");
|
|
169
|
+
if (f.readOnlyToolTail === true && cfg.classifier.readOnlyToolWeight > 0) {
|
|
170
|
+
add(-cfg.classifier.readOnlyToolWeight, "read-only tool loop (the model is looking, not deciding)");
|
|
171
|
+
}
|
|
169
172
|
if (f.toolCount > 0) add(W_TOOLS_OFFERED, `${f.toolCount} tools offered`);
|
|
170
173
|
|
|
171
174
|
score = Math.min(1, Math.max(0, score));
|
package/src/router/features.ts
CHANGED
|
@@ -242,6 +242,19 @@ export function extractFeatures(req: NormRequest, promptTokens: number): Feature
|
|
|
242
242
|
else anatomy.toolBytes += m.textBytes;
|
|
243
243
|
}
|
|
244
244
|
|
|
245
|
+
// Read-only tool loop: the assistant call behind a tool-result tail used
|
|
246
|
+
// only tools that look at things. Tool names are the harness's own; the
|
|
247
|
+
// set covers omp's built-ins and their common aliases.
|
|
248
|
+
let readOnlyToolTail = false;
|
|
249
|
+
if (isToolResultContinuation) {
|
|
250
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
251
|
+
const m = messages[i];
|
|
252
|
+
if (m === undefined || m.role !== "assistant") continue;
|
|
253
|
+
if (m.toolCalls.length > 0) readOnlyToolTail = m.toolCalls.every((tc) => READ_ONLY_TOOLS.has(tc.name.toLowerCase()));
|
|
254
|
+
break;
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
|
|
245
258
|
return {
|
|
246
259
|
promptTokens,
|
|
247
260
|
newContentTokens,
|
|
@@ -265,5 +278,31 @@ export function extractFeatures(req: NormRequest, promptTokens: number): Feature
|
|
|
265
278
|
questionCount,
|
|
266
279
|
isTerseInstruction,
|
|
267
280
|
anatomy,
|
|
281
|
+
isSubagent: req.isSubagent,
|
|
282
|
+
readOnlyToolTail,
|
|
268
283
|
};
|
|
269
284
|
}
|
|
285
|
+
|
|
286
|
+
/** Tools that read state without changing it, in omp, Claude Code and Hermes naming. */
|
|
287
|
+
export const READ_ONLY_TOOLS: ReadonlySet<string> = new Set([
|
|
288
|
+
"read",
|
|
289
|
+
"read_file",
|
|
290
|
+
"grep",
|
|
291
|
+
"glob",
|
|
292
|
+
"ls",
|
|
293
|
+
"list",
|
|
294
|
+
"list_dir",
|
|
295
|
+
"find",
|
|
296
|
+
"lsp",
|
|
297
|
+
"ast_grep",
|
|
298
|
+
"search",
|
|
299
|
+
"web_search",
|
|
300
|
+
"web_fetch",
|
|
301
|
+
"webfetch",
|
|
302
|
+
"websearch",
|
|
303
|
+
"fetch",
|
|
304
|
+
"cat",
|
|
305
|
+
"view",
|
|
306
|
+
"inspect_image",
|
|
307
|
+
"todo",
|
|
308
|
+
]);
|
package/src/router/index.ts
CHANGED
|
@@ -38,11 +38,18 @@ export interface RouterDeps {
|
|
|
38
38
|
*/
|
|
39
39
|
const NEUTRAL_TOKENIZER = "gpt";
|
|
40
40
|
|
|
41
|
-
function resolveProfile(cfg: RouterConfig, requestedModel: string): ProfileConfig {
|
|
42
|
-
const exact = cfg.profiles.find((p) => p.id === requestedModel);
|
|
43
|
-
if (exact !== undefined) return exact;
|
|
41
|
+
export function resolveProfile(cfg: RouterConfig, requestedModel: string, isSubagent = false): ProfileConfig {
|
|
44
42
|
const fallback = cfg.profiles[0];
|
|
45
43
|
if (fallback === undefined) throw new Error("no router profiles configured");
|
|
44
|
+
// A subagent asking for the default profile is routed under the subagent
|
|
45
|
+
// profile when one is configured and exists; an explicit other profile
|
|
46
|
+
// (auto-max, auto-cheap) is honoured as asked.
|
|
47
|
+
const exact = cfg.profiles.find((p) => p.id === requestedModel);
|
|
48
|
+
if (isSubagent && cfg.server.subagentProfile !== "" && (exact === undefined || exact.id === fallback.id)) {
|
|
49
|
+
const sub = cfg.profiles.find((p) => p.id === cfg.server.subagentProfile);
|
|
50
|
+
if (sub !== undefined) return sub;
|
|
51
|
+
}
|
|
52
|
+
if (exact !== undefined) return exact;
|
|
46
53
|
return fallback;
|
|
47
54
|
}
|
|
48
55
|
|
|
@@ -96,7 +103,7 @@ export function createRouter(deps: RouterDeps): Router {
|
|
|
96
103
|
req,
|
|
97
104
|
features,
|
|
98
105
|
classification,
|
|
99
|
-
profile: resolveProfile(config, req.requestedModel),
|
|
106
|
+
profile: resolveProfile(config, req.requestedModel, req.isSubagent),
|
|
100
107
|
state,
|
|
101
108
|
snapshot,
|
|
102
109
|
ledger,
|
package/src/router/types.ts
CHANGED
|
@@ -97,6 +97,14 @@ export interface Features {
|
|
|
97
97
|
* recorded before it existed.
|
|
98
98
|
*/
|
|
99
99
|
anatomy?: PromptAnatomy;
|
|
100
|
+
/** The request came from an omp subagent (`X-Omp-Subagent`). */
|
|
101
|
+
isSubagent?: boolean;
|
|
102
|
+
/**
|
|
103
|
+
* This is a tool-result continuation and the newest assistant turn issued
|
|
104
|
+
* only read-only tools (read, grep, glob, ls, lsp…): the model is looking,
|
|
105
|
+
* not deciding. Recorded for replay; scored at classifier.readOnlyToolWeight.
|
|
106
|
+
*/
|
|
107
|
+
readOnlyToolTail?: boolean;
|
|
100
108
|
}
|
|
101
109
|
|
|
102
110
|
/** Prompt bytes by message role and by age, plus the tool-schema bytes beside them. */
|
|
@@ -292,6 +292,9 @@ export function parseChatRequest(body: unknown, headers: Headers): NormRequest {
|
|
|
292
292
|
// ⇒ the server falls back to its configured default scope.
|
|
293
293
|
const agentdoxScope = (headers.get("x-agentdox-scope") ?? "").trim();
|
|
294
294
|
|
|
295
|
+
// Subagent marker from the embed extension (sessions without a UI).
|
|
296
|
+
const isSubagent = (headers.get("x-omp-subagent") ?? "").trim() === "1";
|
|
297
|
+
|
|
295
298
|
if (typeof b.model !== "string" || b.model.length === 0) {
|
|
296
299
|
throw invalidRequest("model must be a non-empty string");
|
|
297
300
|
}
|
|
@@ -352,6 +355,7 @@ export function parseChatRequest(body: unknown, headers: Headers): NormRequest {
|
|
|
352
355
|
harnessId,
|
|
353
356
|
ompSessionId,
|
|
354
357
|
agentdoxScope,
|
|
358
|
+
isSubagent,
|
|
355
359
|
requestedModel,
|
|
356
360
|
messages,
|
|
357
361
|
tools,
|
package/src/wire/types.ts
CHANGED
|
@@ -82,6 +82,8 @@ export interface NormRequest {
|
|
|
82
82
|
* and if that is empty too the bridge stays inert for this request.
|
|
83
83
|
*/
|
|
84
84
|
agentdoxScope: string;
|
|
85
|
+
/** `X-Omp-Subagent: 1`: the caller is an omp subagent (a session without a UI). */
|
|
86
|
+
isSubagent: boolean;
|
|
85
87
|
/** Virtual model the client selected, e.g. `auto`, `auto-cheap`, `auto-max`. */
|
|
86
88
|
requestedModel: string;
|
|
87
89
|
messages: NormMessage[];
|
package/test/classify.test.ts
CHANGED
|
@@ -458,3 +458,16 @@ describe("classifyTask", () => {
|
|
|
458
458
|
expect(classifyTask(featuresFor([SYSTEM, { role: "user", content: "explain the architecture of the system" }], []))).toBe("documentation");
|
|
459
459
|
});
|
|
460
460
|
});
|
|
461
|
+
|
|
462
|
+
describe("classifier.readOnlyToolWeight", () => {
|
|
463
|
+
test("subtracts only when enabled and the tail is a read-only loop", () => {
|
|
464
|
+
const base = { ...featuresFor([{ role: "user", content: "look" }]), isToolResultContinuation: true, readOnlyToolTail: true };
|
|
465
|
+
const off = scoreHeuristic(base, DEFAULT_CONFIG);
|
|
466
|
+
const cfg = structuredClone(DEFAULT_CONFIG);
|
|
467
|
+
cfg.classifier.readOnlyToolWeight = 0.1;
|
|
468
|
+
const on = scoreHeuristic(base, cfg);
|
|
469
|
+
expect(on.score).toBeCloseTo(Math.max(0, off.score - 0.1), 6);
|
|
470
|
+
expect(on.reasons.some((r) => r.includes("read-only tool loop"))).toBe(true);
|
|
471
|
+
expect(scoreHeuristic({ ...base, readOnlyToolTail: false }, cfg).score).toBeCloseTo(off.score, 6);
|
|
472
|
+
});
|
|
473
|
+
});
|
|
@@ -280,6 +280,7 @@ describe("runWizard", () => {
|
|
|
280
280
|
"", // keep apiKey
|
|
281
281
|
"", // keep harnessId
|
|
282
282
|
"", // keep maxConcurrentTurns
|
|
283
|
+
"", // keep subagentProfile
|
|
283
284
|
"s",
|
|
284
285
|
]);
|
|
285
286
|
expect(partial).toEqual({ server: { port: 9000 } });
|
|
@@ -358,7 +359,7 @@ describe("runWizard: profiles", () => {
|
|
|
358
359
|
const profiles = (partial ?? {})["profiles"];
|
|
359
360
|
expect(Array.isArray(profiles)).toBe(true);
|
|
360
361
|
if (!Array.isArray(profiles)) return;
|
|
361
|
-
expect(profiles).toHaveLength(
|
|
362
|
+
expect(profiles).toHaveLength(4);
|
|
362
363
|
expect(profiles[0]).toMatchObject({ id: "auto", contextWindow: 500000 });
|
|
363
364
|
expect(profiles[1]).toMatchObject({ id: "auto-cheap", contextWindow: 400000 });
|
|
364
365
|
});
|
|
@@ -379,8 +380,8 @@ describe("runWizard: profiles", () => {
|
|
|
379
380
|
const profiles = (partial ?? {})["profiles"];
|
|
380
381
|
expect(Array.isArray(profiles)).toBe(true);
|
|
381
382
|
if (!Array.isArray(profiles)) return;
|
|
382
|
-
expect(profiles).toHaveLength(
|
|
383
|
-
expect(profiles[
|
|
383
|
+
expect(profiles).toHaveLength(5);
|
|
384
|
+
expect(profiles[4]).toEqual({
|
|
384
385
|
id: "auto-fast",
|
|
385
386
|
name: "Auto Fast",
|
|
386
387
|
minTier: "trivial",
|
|
@@ -400,11 +401,11 @@ describe("runWizard: profiles", () => {
|
|
|
400
401
|
const profiles = (partial ?? {})["profiles"];
|
|
401
402
|
expect(Array.isArray(profiles)).toBe(true);
|
|
402
403
|
if (!Array.isArray(profiles)) return;
|
|
403
|
-
expect(profiles.map((p) => (p as Record<string, unknown>)["id"])).toEqual(["auto", "auto-max"]);
|
|
404
|
+
expect(profiles.map((p) => (p as Record<string, unknown>)["id"])).toEqual(["auto", "auto-max", "auto-sub"]);
|
|
404
405
|
});
|
|
405
406
|
|
|
406
407
|
test("refuses to delete the last remaining profile", async () => {
|
|
407
|
-
const { out } = await drive(["p", "x3", "x2", "x1", "b", "q"]);
|
|
408
|
+
const { out } = await drive(["p", "x4", "x3", "x2", "x1", "b", "q"]);
|
|
408
409
|
expect(out).toContain("cannot delete the last profile");
|
|
409
410
|
});
|
|
410
411
|
|
package/test/controls.test.ts
CHANGED
|
@@ -6,6 +6,7 @@ import { createFeedbackStore } from "../src/cost/feedback.ts";
|
|
|
6
6
|
import { createLedger } from "../src/cost/ledger.ts";
|
|
7
7
|
import { EMPTY_USAGE, type LedgerEntry } from "../src/cost/types.ts";
|
|
8
8
|
import { startServer, type StartedServer } from "../src/server/http.ts";
|
|
9
|
+
import { resolveProfile } from "../src/router/index.ts";
|
|
9
10
|
import { ollamaRunway } from "../src/server/http.ts";
|
|
10
11
|
import { createSessionOverrides, OVERRIDE_TTL_MS } from "../src/server/overrides.ts";
|
|
11
12
|
import { openDb } from "../src/util/sqlite.ts";
|
|
@@ -121,7 +122,7 @@ describe("override and feedback endpoints", () => {
|
|
|
121
122
|
beforeAll(() => {
|
|
122
123
|
const cfg: RouterConfig = {
|
|
123
124
|
...structuredClone(DEFAULT_CONFIG),
|
|
124
|
-
server: { host: "127.0.0.1", port: 0, maxConcurrentTurns: 24 },
|
|
125
|
+
server: { host: "127.0.0.1", port: 0, maxConcurrentTurns: 24, subagentProfile: "auto-sub" },
|
|
125
126
|
ledger: { ...DEFAULT_CONFIG.ledger, path: ":memory:" },
|
|
126
127
|
logLevel: "silent",
|
|
127
128
|
};
|
|
@@ -221,3 +222,17 @@ describe("ollamaRunway", () => {
|
|
|
221
222
|
expect(ollamaRunway(null, 7, 1)).toBeNull();
|
|
222
223
|
});
|
|
223
224
|
});
|
|
225
|
+
|
|
226
|
+
describe("subagent profile", () => {
|
|
227
|
+
test("a subagent asking for the default profile is routed under server.subagentProfile; explicit profiles are honoured", () => {
|
|
228
|
+
const cfg = structuredClone(DEFAULT_CONFIG);
|
|
229
|
+
expect(resolveProfile(cfg, "auto", true).id).toBe("auto-sub");
|
|
230
|
+
expect(resolveProfile(cfg, "auto", false).id).toBe("auto");
|
|
231
|
+
expect(resolveProfile(cfg, "auto-max", true).id).toBe("auto-max");
|
|
232
|
+
expect(resolveProfile(cfg, "unknown", true).id).toBe("auto-sub"); // unknown ids fall back to the default, which a subagent remaps
|
|
233
|
+
cfg.server.subagentProfile = "";
|
|
234
|
+
expect(resolveProfile(cfg, "auto", true).id).toBe("auto");
|
|
235
|
+
cfg.server.subagentProfile = "nope";
|
|
236
|
+
expect(resolveProfile(cfg, "auto", true).id).toBe("auto");
|
|
237
|
+
});
|
|
238
|
+
});
|
package/test/escalate.test.ts
CHANGED
package/test/failover.test.ts
CHANGED
|
@@ -29,7 +29,7 @@ import type {
|
|
|
29
29
|
|
|
30
30
|
function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
|
|
31
31
|
return {
|
|
32
|
-
server: { host: "127.0.0.1", port: 8787, maxConcurrentTurns: 24 },
|
|
32
|
+
server: { host: "127.0.0.1", port: 8787, maxConcurrentTurns: 24, subagentProfile: "auto-sub" },
|
|
33
33
|
openrouter: { baseUrl: "https://openrouter.ai/api/v1", apiKey: "", title: "test", timeoutMs: 30_000, catalogTtlMs: 3_600_000, catalogRefreshMs: 0 },
|
|
34
34
|
ollama: { enabled: false, baseUrl: "http://127.0.0.1:11434/v1", apiKey: "", timeoutMs: 30_000, catalogTtlMs: 300_000, includeLocal: false, prices: {}, twins: {}, costBias: 1, biasUntilUsage: 0.9, usagePollMs: 0, quotaCooldownMs: 0, rateLimitCooldownMs: 0, planCreditsUsd: 0 },
|
|
35
35
|
benchmarks: { enabled: false, artificialAnalysisApiKey: "", benchlm: true, refreshMs: 86_400_000, timeoutMs: 30_000, useLocalScores: false },
|
|
@@ -46,7 +46,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
|
|
|
46
46
|
data: { axis: "intelligence", minQuality: 0 },
|
|
47
47
|
chat: { axis: "intelligence", minQuality: 0 },
|
|
48
48
|
},
|
|
49
|
-
filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, minTrustSamples: 5, trustScopedByHarness: false, trustWindowDays: 0, contextHeadroom: 1.2, latencyWeight: 0, latencyReferenceMs: 5000, latencyReferenceTokensPerSec: 30, cacheReliabilityMinSamples: 10, latencyMinSamples: 20, escalationCostWeight: 0 },
|
|
49
|
+
filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, feedbackWeight: 0, minTrustSamples: 5, trustScopedByHarness: false, trustWindowDays: 0, contextHeadroom: 1.2, latencyWeight: 0, latencyReferenceMs: 5000, latencyReferenceTokensPerSec: 30, cacheReliabilityMinSamples: 10, latencyMinSamples: 20, escalationCostWeight: 0 },
|
|
50
50
|
classifier: {
|
|
51
51
|
ambiguityThreshold: 0,
|
|
52
52
|
model: "test/adjudicator", learnedModelPath: "",
|
|
@@ -57,7 +57,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
|
|
|
57
57
|
toolAxis: "coding",
|
|
58
58
|
chatAxis: "intelligence",
|
|
59
59
|
agenticLoopDepth: 3,
|
|
60
|
-
mechanicalRetryFactor: 0.2,
|
|
60
|
+
mechanicalRetryFactor: 0.2, readOnlyToolWeight: 0,
|
|
61
61
|
reasoningWeights: { medium: 0.14, high: 0.24, xhigh: 0.3, max: 0.34 },
|
|
62
62
|
},
|
|
63
63
|
escalation: {
|
|
@@ -92,6 +92,7 @@ function mkReq(): NormRequest {
|
|
|
92
92
|
harnessId: "",
|
|
93
93
|
ompSessionId: "",
|
|
94
94
|
agentdoxScope: "",
|
|
95
|
+
isSubagent: false,
|
|
95
96
|
requestedModel: "auto",
|
|
96
97
|
messages: [{ role: "user", text: "hi", images: 0, textBytes: 2, toolCalls: [] }],
|
|
97
98
|
tools: [],
|
package/test/features.test.ts
CHANGED
|
@@ -337,3 +337,35 @@ describe("prompt anatomy", () => {
|
|
|
337
337
|
expect(a.olderHalfBytes).toBe(0);
|
|
338
338
|
});
|
|
339
339
|
});
|
|
340
|
+
|
|
341
|
+
describe("subagent and read-only tool loop signals", () => {
|
|
342
|
+
test("a tool-result tail behind read-only calls is flagged; a write call clears it", () => {
|
|
343
|
+
const reads = req([
|
|
344
|
+
SYSTEM,
|
|
345
|
+
{ role: "user", content: "find the retry helper" },
|
|
346
|
+
{ role: "assistant", content: null, tool_calls: [
|
|
347
|
+
{ id: "a", type: "function", function: { name: "grep", arguments: "{\"pattern\":\"retry\"}" } },
|
|
348
|
+
{ id: "b", type: "function", function: { name: "read", arguments: "{\"path\":\"x.ts\"}" } },
|
|
349
|
+
] },
|
|
350
|
+
{ role: "tool", tool_call_id: "a", content: "x.ts:12" },
|
|
351
|
+
{ role: "tool", tool_call_id: "b", content: "export function retry() {}" },
|
|
352
|
+
]);
|
|
353
|
+
expect(extractFeatures(reads, 1000).readOnlyToolTail).toBe(true);
|
|
354
|
+
const write = req([
|
|
355
|
+
SYSTEM,
|
|
356
|
+
{ role: "user", content: "fix it" },
|
|
357
|
+
toolCall("c", "edit", "{\"path\":\"x.ts\"}"),
|
|
358
|
+
{ role: "tool", tool_call_id: "c", content: "ok" },
|
|
359
|
+
]);
|
|
360
|
+
expect(extractFeatures(write, 1000).readOnlyToolTail).toBe(false);
|
|
361
|
+
// A fresh user turn is never a read-only tail, whatever came before.
|
|
362
|
+
expect(extractFeatures(req([SYSTEM, { role: "user", content: "now what?" }]), 100).readOnlyToolTail).toBe(false);
|
|
363
|
+
});
|
|
364
|
+
|
|
365
|
+
test("the subagent marker rides on the request", () => {
|
|
366
|
+
const r = parseChatRequest({ model: "auto", messages: [SYSTEM, { role: "user", content: "hi" }], tools: TOOLS }, new Headers({ "x-omp-subagent": "1" }));
|
|
367
|
+
expect(r.isSubagent).toBe(true);
|
|
368
|
+
expect(extractFeatures(r, 100).isSubagent).toBe(true);
|
|
369
|
+
expect(req([SYSTEM, { role: "user", content: "hi" }]).isSubagent).toBe(false);
|
|
370
|
+
});
|
|
371
|
+
});
|
|
@@ -10,7 +10,7 @@ describe("HTTP server resilience against dead streams", () => {
|
|
|
10
10
|
beforeAll(() => {
|
|
11
11
|
const cfg: RouterConfig = {
|
|
12
12
|
...DEFAULT_CONFIG,
|
|
13
|
-
server: { host: "127.0.0.1", port: 0, maxConcurrentTurns: 24 },
|
|
13
|
+
server: { host: "127.0.0.1", port: 0, maxConcurrentTurns: 24, subagentProfile: "auto-sub" },
|
|
14
14
|
ledger: { ...DEFAULT_CONFIG.ledger, path: ":memory:" },
|
|
15
15
|
context: { ...DEFAULT_CONFIG.context, enabled: false },
|
|
16
16
|
logLevel: "silent",
|
package/test/report-hub.test.ts
CHANGED
package/test/report.test.ts
CHANGED
|
@@ -197,6 +197,18 @@ describe("buildUsageReport", () => {
|
|
|
197
197
|
db.close();
|
|
198
198
|
});
|
|
199
199
|
|
|
200
|
+
test("subagent turns are counted with their spend", () => {
|
|
201
|
+
const { db, ledger } = seeded();
|
|
202
|
+
ledger.record(entry({ reportedUsd: 0.01, features: { isSubagent: true } }));
|
|
203
|
+
ledger.record(entry({ reportedUsd: 0.03, features: { isSubagent: false } }));
|
|
204
|
+
ledger.record(entry({ reportedUsd: 0.06 }));
|
|
205
|
+
const r = buildUsageReport(db, { windowDays: 7, nowMs: NOW });
|
|
206
|
+
expect(r.totals.subagentDispatches).toBe(1);
|
|
207
|
+
expect(r.totals.subagentSpendUsd).toBeCloseTo(0.01, 6);
|
|
208
|
+
expect(renderUsageReport(r)).toContain("subagents: 1 dispatches, $0.0100 (10% of spend)");
|
|
209
|
+
db.close();
|
|
210
|
+
});
|
|
211
|
+
|
|
200
212
|
test("empty ledger yields zeroed totals and null speeds", () => {
|
|
201
213
|
const { db } = seeded();
|
|
202
214
|
const r = buildUsageReport(db, { windowDays: 7, nowMs: NOW });
|
|
@@ -213,6 +225,8 @@ describe("buildUsageReport", () => {
|
|
|
213
225
|
aborted: 0,
|
|
214
226
|
modelSwitches: 0,
|
|
215
227
|
cacheEstimated: false,
|
|
228
|
+
subagentDispatches: 0,
|
|
229
|
+
subagentSpendUsd: 0,
|
|
216
230
|
});
|
|
217
231
|
expect(r.providers).toEqual([]);
|
|
218
232
|
expect(r.models).toEqual([]);
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { describe, expect, test } from "bun:test";
|
|
2
|
+
import { createFeedbackStore } from "../src/cost/feedback.ts";
|
|
2
3
|
|
|
3
4
|
import { loadConfig } from "../src/config/load.ts";
|
|
4
5
|
import { createLedger, LATENCY_WINDOW_ROWS } from "../src/cost/ledger.ts";
|
|
@@ -410,3 +411,55 @@ describe("cache reliability signal", () => {
|
|
|
410
411
|
db.close();
|
|
411
412
|
});
|
|
412
413
|
});
|
|
414
|
+
|
|
415
|
+
describe("feedback in trust", () => {
|
|
416
|
+
// A user verdict counts as filters.feedbackWeight attempts of that outcome.
|
|
417
|
+
function trustWith(weight: number, verdicts: Array<"good" | "bad">): { rate: number; good: number; bad: number } {
|
|
418
|
+
const db = openDb(":memory:");
|
|
419
|
+
try {
|
|
420
|
+
const c = structuredClone(cfg);
|
|
421
|
+
c.filters.feedbackWeight = weight;
|
|
422
|
+
const ledger = createLedger(db, c);
|
|
423
|
+
const fb = createFeedbackStore(db);
|
|
424
|
+
let last = "";
|
|
425
|
+
for (let i = 0; i < 10; i++) {
|
|
426
|
+
const e = entry({ error: null });
|
|
427
|
+
last = e.id;
|
|
428
|
+
ledger.record(e);
|
|
429
|
+
}
|
|
430
|
+
for (const v of verdicts) fb.record({ ledgerId: last, ompSessionId: "s", slug: "vendor/model", tier: "simple", verdict: v, note: "" });
|
|
431
|
+
const t = ledger.trust("vendor/model")!;
|
|
432
|
+
return { rate: t.successRate, good: t.feedbackGood ?? -1, bad: t.feedbackBad ?? -1 };
|
|
433
|
+
} finally {
|
|
434
|
+
db.close();
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
test("weight 0 records verdicts without moving the rate", () => {
|
|
439
|
+
const base = trustWith(0, []);
|
|
440
|
+
expect(base.rate).toBeCloseTo(11 / 12, 6); // (10 - 0 + 1) / (10 + 2)
|
|
441
|
+
expect(trustWith(0, ["bad", "bad"]).rate).toBeCloseTo(base.rate, 6);
|
|
442
|
+
});
|
|
443
|
+
|
|
444
|
+
test("a bad verdict counts as `weight` failures, a good one as `weight` successes", () => {
|
|
445
|
+
// 10 clean attempts + one bad verdict at weight 3: attempts 13, failures 3.
|
|
446
|
+
const bad = trustWith(3, ["bad"]);
|
|
447
|
+
expect(bad.rate).toBeCloseTo((13 - 3 + 1) / (13 + 2), 6);
|
|
448
|
+
expect(bad.bad).toBe(1);
|
|
449
|
+
const good = trustWith(3, ["good"]);
|
|
450
|
+
expect(good.rate).toBeCloseTo((13 - 0 + 1) / (13 + 2), 6);
|
|
451
|
+
expect(good.good).toBe(1);
|
|
452
|
+
// allTrust and signals agree with trust().
|
|
453
|
+
const db = openDb(":memory:");
|
|
454
|
+
const c = structuredClone(cfg);
|
|
455
|
+
c.filters.feedbackWeight = 3;
|
|
456
|
+
const ledger = createLedger(db, c);
|
|
457
|
+
const fb = createFeedbackStore(db);
|
|
458
|
+
const e = entry({ error: null });
|
|
459
|
+
ledger.record(e);
|
|
460
|
+
fb.record({ ledgerId: e.id, ompSessionId: "s", slug: "vendor/model", tier: "simple", verdict: "bad", note: "" });
|
|
461
|
+
expect(ledger.allTrust()[0]?.successRate).toBeCloseTo(ledger.trust("vendor/model")!.successRate, 9);
|
|
462
|
+
expect(ledger.signals?.(["vendor/model"]).get("vendor/model")?.trust?.successRate).toBeCloseTo(ledger.trust("vendor/model")!.successRate, 9);
|
|
463
|
+
db.close();
|
|
464
|
+
});
|
|
465
|
+
});
|
package/test/turn.test.ts
CHANGED
|
@@ -29,7 +29,7 @@ import type {
|
|
|
29
29
|
|
|
30
30
|
function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
|
|
31
31
|
return {
|
|
32
|
-
server: { host: "127.0.0.1", port: 8787, maxConcurrentTurns: 24 },
|
|
32
|
+
server: { host: "127.0.0.1", port: 8787, maxConcurrentTurns: 24, subagentProfile: "auto-sub" },
|
|
33
33
|
openrouter: { baseUrl: "https://openrouter.ai/api/v1", apiKey: "", title: "test", timeoutMs: 30_000, catalogTtlMs: 3_600_000, catalogRefreshMs: 0 },
|
|
34
34
|
ollama: { enabled: false, baseUrl: "http://127.0.0.1:11434/v1", apiKey: "", timeoutMs: 30_000, catalogTtlMs: 300_000, includeLocal: false, prices: {}, twins: {}, costBias: 1, biasUntilUsage: 0.9, usagePollMs: 0, quotaCooldownMs: 0, rateLimitCooldownMs: 0, planCreditsUsd: 0 },
|
|
35
35
|
benchmarks: { enabled: false, artificialAnalysisApiKey: "", benchlm: true, refreshMs: 86_400_000, timeoutMs: 30_000, useLocalScores: false },
|
|
@@ -46,7 +46,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
|
|
|
46
46
|
data: { axis: "intelligence", minQuality: 0 },
|
|
47
47
|
chat: { axis: "intelligence", minQuality: 0 },
|
|
48
48
|
},
|
|
49
|
-
filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, minTrustSamples: 5, trustScopedByHarness: false, trustWindowDays: 0, contextHeadroom: 1.2, latencyWeight: 0, latencyReferenceMs: 5000, latencyReferenceTokensPerSec: 30, cacheReliabilityMinSamples: 10, latencyMinSamples: 20, escalationCostWeight: 0 },
|
|
49
|
+
filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, feedbackWeight: 0, minTrustSamples: 5, trustScopedByHarness: false, trustWindowDays: 0, contextHeadroom: 1.2, latencyWeight: 0, latencyReferenceMs: 5000, latencyReferenceTokensPerSec: 30, cacheReliabilityMinSamples: 10, latencyMinSamples: 20, escalationCostWeight: 0 },
|
|
50
50
|
classifier: {
|
|
51
51
|
ambiguityThreshold: 0,
|
|
52
52
|
model: "test/adjudicator", learnedModelPath: "",
|
|
@@ -57,7 +57,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
|
|
|
57
57
|
toolAxis: "coding",
|
|
58
58
|
chatAxis: "intelligence",
|
|
59
59
|
agenticLoopDepth: 3,
|
|
60
|
-
mechanicalRetryFactor: 0.2,
|
|
60
|
+
mechanicalRetryFactor: 0.2, readOnlyToolWeight: 0,
|
|
61
61
|
reasoningWeights: { medium: 0.14, high: 0.24, xhigh: 0.3, max: 0.34 },
|
|
62
62
|
},
|
|
63
63
|
escalation: {
|
|
@@ -92,6 +92,7 @@ function mkReq(): NormRequest {
|
|
|
92
92
|
harnessId: "",
|
|
93
93
|
ompSessionId: "",
|
|
94
94
|
agentdoxScope: "",
|
|
95
|
+
isSubagent: false,
|
|
95
96
|
requestedModel: "auto",
|
|
96
97
|
messages: [{ role: "user", text: "hi", images: 0, textBytes: 2, toolCalls: [] }],
|
|
97
98
|
tools: [],
|