auto-model-router 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +124 -31
- package/hermes-plugin/__init__.py +17 -1
- package/omp-extension/router-embed.ts +14 -6
- package/omp-extension/router-toast.ts +6 -1
- package/omp-extension/toast-logic.ts +7 -0
- package/package.json +1 -1
- package/src/cost/ledger.ts +5 -2
- package/src/cost/types.ts +6 -0
- package/src/index.ts +0 -1
- package/src/server/turn.ts +1 -0
- package/src/util/sqlite.ts +12 -1
- package/src/wire/openai/request.ts +5 -0
- package/src/wire/types.ts +7 -0
- package/test/escalate.test.ts +1 -0
- package/test/failover.test.ts +1 -0
- package/test/toast-logic.test.ts +32 -0
- package/test/tokens.test.ts +1 -0
- package/test/trust-attribution.test.ts +16 -2
- package/test/turn.test.ts +1 -0
- package/test/wire-request.test.ts +11 -0
package/README.md
CHANGED
|
@@ -6,13 +6,14 @@ OpenRouter model **per turn** based on measured price and estimated task
|
|
|
6
6
|
complexity — including mid-conversation, when a session shifts from mechanical
|
|
7
7
|
tool-loop churn to genuine reasoning work.
|
|
8
8
|
|
|
9
|
-
All LLM inference is offloaded to OpenRouter. Nothing runs on-device except
|
|
10
|
-
routing arithmetic.
|
|
11
|
-
|
|
12
9
|
auto-model-router runs **embedded inside the omp process** (as an omp extension) — no
|
|
13
10
|
separate server, no orphaned process. It binds a free OS-assigned port and
|
|
14
11
|
lives and dies with the omp session.
|
|
15
12
|
|
|
13
|
+
For non-omp harnesses (Hermes, Claude, any OpenAI-compatible client), run it as
|
|
14
|
+
a standalone process with `auto-model-router serve --port <n>` — the same core,
|
|
15
|
+
on a fixed port, owned by you. See [Hermes](#hermes) below.
|
|
16
|
+
|
|
16
17
|
## Why this exists when OpenRouter already ships routers
|
|
17
18
|
|
|
18
19
|
OpenRouter has `openrouter/auto` (market-spend classifier) and
|
|
@@ -62,7 +63,7 @@ without touching routing.
|
|
|
62
63
|
| `src/router/` | Feature extraction, complexity classification, candidate filtering and scoring, hysteresis, cache-breakpoint placement, budget guard, probe planning. |
|
|
63
64
|
| `src/upstream/` | OpenRouter transport: streaming dispatch, `session_id` stickiness, error classification, fallback arrays. |
|
|
64
65
|
| `src/config/` | Configuration loading, schema validation, and the built-in defaults. |
|
|
65
|
-
| `src/cli/` | `stats`, `models`, `explain`, `config` commands. |
|
|
66
|
+
| `src/cli/` | `serve`, `stats`, `models`, `explain`, `config` commands. |
|
|
66
67
|
| `omp-extension/` | The omp extensions: `router-embed.ts`, `router-toast.ts`, `router-configure.ts`. |
|
|
67
68
|
|
|
68
69
|
### Two cost numbers, never conflated
|
|
@@ -73,13 +74,68 @@ without touching routing.
|
|
|
73
74
|
- **Reported** — `usage.cost` from OpenRouter, authoritative after the fact.
|
|
74
75
|
Drives the ledger, `stats`, and prediction-error calibration.
|
|
75
76
|
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
- **omp** (the Oh My Pi harness) — the router runs as an omp extension.
|
|
79
|
-
- **Bun** `>= 1.2.0` — omp itself is a Bun process; the router code runs inside
|
|
77
|
+
it. No separate Bun install is needed for the embedded path. The standalone
|
|
78
|
+
`serve` binary (`npm install -g auto-model-router`) bundles Bun.
|
|
80
79
|
it. No separate Bun install is needed for the embedded path.
|
|
81
80
|
|
|
82
|
-
|
|
81
|
+
Two ways to get the router into omp. The **npm package** is the modern path —
|
|
82
|
+
it installs the `auto-model-router` binary and wires the omp extensions; the
|
|
83
|
+
**repo-local installer** is for developing against the source.
|
|
84
|
+
|
|
85
|
+
### Via npm (installs the `auto-model-router` binary)
|
|
86
|
+
|
|
87
|
+
```bash
|
|
88
|
+
npm install -g auto-model-router
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
Then add the shipped extensions to omp's `~/.omp/agent/config.yml`
|
|
92
|
+
(`$PI_CODING_AGENT_DIR/config.yml` when that env var relocates the agent dir):
|
|
93
|
+
|
|
94
|
+
```yaml
|
|
95
|
+
# ~/.omp/agent/config.yml
|
|
96
|
+
extensions:
|
|
97
|
+
- auto-model-router/omp-extension/router-embed.ts
|
|
98
|
+
- auto-model-router/omp-extension/router-toast.ts # optional: chosen-model toasts
|
|
99
|
+
- auto-model-router/omp-extension/router-configure.ts # optional: /router command
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
### From the repo (cross-platform installer)
|
|
103
|
+
|
|
104
|
+
```bash
|
|
105
|
+
bun tools/install.ts
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
It wires the auto-model-router extensions into omp's `~/.omp/agent/config.yml`
|
|
109
|
+
(`$PI_CODING_AGENT_DIR/config.yml` when that env var relocates the agent dir),
|
|
110
|
+
backing up the previous file first. It is idempotent — re-running is a no-op.
|
|
111
|
+
|
|
112
|
+
Options:
|
|
113
|
+
|
|
114
|
+
```bash
|
|
115
|
+
bun tools/install.ts --no-toast --no-configure # only the required embed extension
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
The installer adds:
|
|
119
|
+
|
|
120
|
+
- `router-embed.ts` — **required**; runs the router in-process.
|
|
121
|
+
- `router-toast.ts` — optional; chosen-model toasts.
|
|
122
|
+
- `router-configure.ts` — optional; the `/router` command.
|
|
123
|
+
|
|
124
|
+
Or add the paths by hand to omp's `~/.omp/agent/config.yml`:
|
|
125
|
+
|
|
126
|
+
```yaml
|
|
127
|
+
# ~/.omp/agent/config.yml
|
|
128
|
+
extensions:
|
|
129
|
+
- /path/to/auto-model-router/omp-extension/router-embed.ts
|
|
130
|
+
- /path/to/auto-model-router/omp-extension/router-toast.ts # optional: chosen-model toasts
|
|
131
|
+
- /path/to/auto-model-router/omp-extension/router-configure.ts # optional: /router command
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
Then restart the omp session (extensions load at session start).
|
|
135
|
+
|
|
136
|
+
or install it from the marketplace (see below). The plugin declares all three
|
|
137
|
+
extensions (`router-embed`, `router-toast`, `router-configure`), so installing
|
|
138
|
+
it wires the router in without editing `config.yml` by hand.
|
|
83
139
|
|
|
84
140
|
There is nothing to install system-wide. Run the cross-platform installer
|
|
85
141
|
(Windows, macOS, Linux) from the repo:
|
|
@@ -162,37 +218,65 @@ npm publish
|
|
|
162
218
|
|
|
163
219
|
### Hermes
|
|
164
220
|
|
|
165
|
-
|
|
166
|
-
|
|
221
|
+
Install the router globally (puts the `serve` binary on PATH) and
|
|
222
|
+
install the native plugin, then point Hermes at it:
|
|
223
|
+
|
|
224
|
+
**1. Install the router binary:**
|
|
225
|
+
|
|
226
|
+
```bash
|
|
227
|
+
npm install -g auto-model-router
|
|
228
|
+
```
|
|
229
|
+
|
|
230
|
+
**2. Install the Hermes plugin.** Copy `hermes-plugin/` to
|
|
231
|
+
`$HERMES_HOME/plugins/model-providers/auto-model-router/` (where
|
|
232
|
+
`HERMES_HOME` is `C:\Users\<you>\AppData\Local\hermes` on Windows,
|
|
233
|
+
`~/.hermes` on macOS/Linux):
|
|
234
|
+
|
|
235
|
+
```bash
|
|
236
|
+
mkdir -p "$HERMES_HOME/plugins/model-providers"
|
|
237
|
+
cp -r hermes-plugin/ "$HERMES_HOME/plugins/model-providers/auto-model-router/"
|
|
238
|
+
```
|
|
239
|
+
|
|
240
|
+
**3. Surface the provider in Hermes's picker.** Hermes only lists providers
|
|
241
|
+
that have a credential. The router itself is keyless (it resolves its own
|
|
242
|
+
OpenRouter key), but to make Hermes show it as selectable, add a marker value
|
|
243
|
+
to `$HERMES_HOME/.env`:
|
|
244
|
+
|
|
245
|
+
```bash
|
|
246
|
+
echo "AUTO_MODEL_ROUTER_API_KEY=local" >> "$HERMES_HOME/.env"
|
|
247
|
+
```
|
|
248
|
+
|
|
249
|
+
**4. Restart Hermes.** On load, the plugin spawns the router (`auto-model-router
|
|
250
|
+
serve`) as a subprocess on port 8788 and registers the provider profile. Select
|
|
251
|
+
`auto-model-router/auto` as the model.
|
|
167
252
|
|
|
168
|
-
|
|
169
|
-
|
|
253
|
+
The plugin runs the router against its **own** config home
|
|
254
|
+
(`$HERMES_HOME/auto-model-router/`), separate from omp's
|
|
255
|
+
`~/.auto-model-router/`, so the two harnesses never share a ledger or
|
|
256
|
+
conversation state and don't leak routing toasts into each other's UIs.
|
|
257
|
+
|
|
258
|
+
The router serves `GET /v1/models` (returning the `auto`, `auto-cheap`,
|
|
259
|
+
`auto-max` profiles) and `POST /v1/chat/completions`, which Hermes's custom
|
|
260
|
+
endpoint discovery verifies. The router's own OpenRouter key resolution
|
|
261
|
+
(config → env → omp auth store) applies — Hermes does not need its own
|
|
262
|
+
OpenRouter key.
|
|
263
|
+
|
|
264
|
+
**Standalone alternative (no plugin):** run the router yourself, then add a
|
|
265
|
+
custom provider:
|
|
170
266
|
|
|
171
267
|
```bash
|
|
172
268
|
auto-model-router serve --port 8788
|
|
173
269
|
```
|
|
174
270
|
|
|
175
271
|
```yaml
|
|
176
|
-
#
|
|
272
|
+
# $HERMES_HOME/config.yaml
|
|
177
273
|
providers:
|
|
178
274
|
auto-model-router:
|
|
179
275
|
base_url: http://127.0.0.1:8788/v1
|
|
180
|
-
api_key:
|
|
276
|
+
api_key: local
|
|
181
277
|
default_model: auto
|
|
182
278
|
```
|
|
183
279
|
|
|
184
|
-
**Hermes plugin (native):** copy `hermes-plugin/` to
|
|
185
|
-
`$HERMES_HOME/plugins/model-providers/auto-model-router/` and restart Hermes.
|
|
186
|
-
The plugin spawns the router as a subprocess on load and registers the provider
|
|
187
|
-
profile, so Hermes routes each turn through the router automatically.
|
|
188
|
-
|
|
189
|
-
The router serves `GET /v1/models` (returning the `auto`, `auto-cheap`,
|
|
190
|
-
`auto-max` profiles) and `POST /v1/chat/completions`, which Hermes's custom
|
|
191
|
-
endpoint discovery verifies. Select `auto-model-router/auto` as the model and
|
|
192
|
-
the router routes each turn by price and complexity. The router's own OpenRouter
|
|
193
|
-
key resolution (config → env → omp auth store) applies — Hermes does not need
|
|
194
|
-
its own OpenRouter key.
|
|
195
|
-
|
|
196
280
|
### The OpenRouter key
|
|
197
281
|
|
|
198
282
|
There should be exactly one OpenRouter key on the machine, and omp already owns
|
|
@@ -307,7 +391,8 @@ disk and back up the previous file to a timestamped `.bak`.
|
|
|
307
391
|
| --- | --- | --- |
|
|
308
392
|
| `OPENROUTER_API_KEY` | OpenRouter key (overrides the auth store). | — |
|
|
309
393
|
| `AUTO_MODEL_ROUTER_HOME` | Config + database directory. | `~/.auto-model-router` |
|
|
310
|
-
| `
|
|
394
|
+
| `AUTO_MODEL_ROUTER_HOST` | Bind address override. | `127.0.0.1` |
|
|
395
|
+
| `AUTO_MODEL_ROUTER_LOG` | Log level: `silent`/`error`/`warn`/`info`/`debug`. | `info` |
|
|
311
396
|
| `AUTO_MODEL_ROUTER_LOG` | Log level: `silent`/`error`/`warn`/`info`/`debug`. | `info` |
|
|
312
397
|
| `AUTO_MODEL_ROUTER_DB` | Override the ledger path. | `$AUTO_MODEL_ROUTER_HOME/router.db` |
|
|
313
398
|
| `AUTO_MODEL_ROUTER_URL` | Toast/base URL override (the toast reads the shared port file first). | — |
|
|
@@ -472,8 +557,15 @@ on each other:
|
|
|
472
557
|
- **Per-harness daily budget** — each harness sends an `X-Omp-Harness` header
|
|
473
558
|
(from the provider block's `headers:`), and the router scopes the rolling
|
|
474
559
|
24h `perDayUsd` ceiling to it. One harness can't exhaust the day for another.
|
|
475
|
-
- **Per-
|
|
476
|
-
extension
|
|
560
|
+
- **Per-session toasts** — the toast surfaces only the decisions made by *its
|
|
561
|
+
own* omp session. The embed extension tags every request with an
|
|
562
|
+
`X-Omp-Session` header (`ctx.sessionManager.getSessionId()`), the router
|
|
563
|
+
records it on each ledger row, and the toast filters on it. Two concurrent
|
|
564
|
+
interactive sessions — even of the same harness — never surface each other's
|
|
565
|
+
model choices. This needs no configuration.
|
|
566
|
+
- **Per-harness toasts** — additionally set `OMP_HARNESS_ID` to the same value
|
|
567
|
+
so the extension only toasts that harness's model choices. Session scoping is
|
|
568
|
+
finer-grained; harness scoping still applies on top when set.
|
|
477
569
|
|
|
478
570
|
Configure a harness by setting `server.harnessId`; set the same id in that
|
|
479
571
|
harness's `OMP_HARNESS_ID` env var.
|
|
@@ -509,7 +601,8 @@ bound, even though it changes every session.
|
|
|
509
601
|
The toast logic is a pure, unit-tested module
|
|
510
602
|
(`omp-extension/toast-logic.ts`, covered by `test/toast-logic.test.ts`): it
|
|
511
603
|
toasts only decisions newer than the last seen one, skips `wasted` escalation
|
|
512
|
-
attempts,
|
|
604
|
+
attempts, prefers the actual serving slug over the requested one, and filters
|
|
605
|
+
to the toast's own omp session id (and harness id, when set).
|
|
513
606
|
|
|
514
607
|
---
|
|
515
608
|
|
|
@@ -35,10 +35,23 @@ BASE_URL = f"http://127.0.0.1:{PORT}/v1"
|
|
|
35
35
|
# The router binary, provided by `npm install -g auto-model-router`.
|
|
36
36
|
BIN = "auto-model-router"
|
|
37
37
|
|
|
38
|
+
# The router's config home. Must be SEPARATE from the default (~/.auto-model-router)
|
|
39
|
+
# which omp's embedded router shares: both read the same ledger + config there,
|
|
40
|
+
# so sharing it would leak omp's routing toasts into this harness (and vice versa)
|
|
41
|
+
# and let the two routers fight over conversation state. Use a hermes-owned home.
|
|
42
|
+
_ROUTER_HOME = os.environ.get(
|
|
43
|
+
"AUTO_MODEL_ROUTER_HOME",
|
|
44
|
+
os.path.join(
|
|
45
|
+
os.environ.get("HERMES_HOME", os.path.expanduser("~")), "auto-model-router"
|
|
46
|
+
),
|
|
47
|
+
)
|
|
48
|
+
|
|
38
49
|
|
|
39
50
|
def _router_running() -> bool:
|
|
40
51
|
try:
|
|
41
|
-
with urllib.request.urlopen(
|
|
52
|
+
with urllib.request.urlopen(
|
|
53
|
+
f"http://127.0.0.1:{PORT}/health", timeout=1
|
|
54
|
+
) as resp:
|
|
42
55
|
return resp.status == 200
|
|
43
56
|
except Exception:
|
|
44
57
|
return False
|
|
@@ -53,6 +66,7 @@ def _spawn_router() -> None:
|
|
|
53
66
|
"auto-model-router is not installed or not on PATH. "
|
|
54
67
|
"Run `npm install -g auto-model-router` first."
|
|
55
68
|
)
|
|
69
|
+
env = {**os.environ, "AUTO_MODEL_ROUTER_HOME": _ROUTER_HOME}
|
|
56
70
|
if sys.platform.startswith("win"):
|
|
57
71
|
# npm installs a `.cmd` shim that cannot be exec'd as a bare argv
|
|
58
72
|
# list; shell=True resolves it through the command shell.
|
|
@@ -62,6 +76,7 @@ def _spawn_router() -> None:
|
|
|
62
76
|
stderr=subprocess.DEVNULL,
|
|
63
77
|
shell=True,
|
|
64
78
|
start_new_session=True,
|
|
79
|
+
env=env,
|
|
65
80
|
)
|
|
66
81
|
else:
|
|
67
82
|
subprocess.Popen(
|
|
@@ -69,6 +84,7 @@ def _spawn_router() -> None:
|
|
|
69
84
|
stdout=subprocess.DEVNULL,
|
|
70
85
|
stderr=subprocess.DEVNULL,
|
|
71
86
|
start_new_session=True,
|
|
87
|
+
env=env,
|
|
72
88
|
)
|
|
73
89
|
# Wait for it to come up (bounded).
|
|
74
90
|
for _ in range(50):
|
|
@@ -43,15 +43,20 @@ import {
|
|
|
43
43
|
* Registers the auto-model-router provider (and its virtual models) into omp's model
|
|
44
44
|
* registry at a specific bound port.
|
|
45
45
|
*/
|
|
46
|
-
function registerRouterProvider(pi: ExtensionAPI, port: number, cfg: RouterConfig): void {
|
|
46
|
+
function registerRouterProvider(pi: ExtensionAPI, port: number, cfg: RouterConfig, sessionId: string): void {
|
|
47
47
|
const providerConfig = buildProviderConfig(port, cfg);
|
|
48
|
+
const headers: Record<string, string> = {};
|
|
49
|
+
if (providerConfig.harnessId !== undefined && providerConfig.harnessId !== "") {
|
|
50
|
+
headers["X-Omp-Harness"] = providerConfig.harnessId;
|
|
51
|
+
}
|
|
52
|
+
// Per-session scoping: lets the toast surface only this session's decisions
|
|
53
|
+
// even when several omp sessions share one embedded router's ledger.
|
|
54
|
+
if (sessionId !== "") headers["X-Omp-Session"] = sessionId;
|
|
48
55
|
pi.registerProvider(EMBED_PROVIDER_ID, {
|
|
49
56
|
baseUrl: providerConfig.baseUrl,
|
|
50
57
|
api: "openai-completions",
|
|
51
58
|
apiKey: EMBED_DUMMY_API_KEY,
|
|
52
|
-
...(
|
|
53
|
-
? { headers: { "X-Omp-Harness": providerConfig.harnessId } }
|
|
54
|
-
: {}),
|
|
59
|
+
...(Object.keys(headers).length > 0 ? { headers } : {}),
|
|
55
60
|
models: providerConfig.models.map((m) => ({
|
|
56
61
|
id: m.id,
|
|
57
62
|
name: m.name,
|
|
@@ -90,9 +95,12 @@ export default function (pi: ExtensionAPI): void {
|
|
|
90
95
|
// Subagents and headless sessions do not bind their own router; they
|
|
91
96
|
// route to the main's router via the shared port file. The main writes
|
|
92
97
|
// the file before spawning subagents, so the port is available here.
|
|
98
|
+
// The omp UI session id tags every request so the toast can scope its
|
|
99
|
+
// notifications to that exact session (see router-toast.ts).
|
|
100
|
+
const sessionId = ctx.sessionManager.getSessionId();
|
|
93
101
|
if (!ctx.hasUI) {
|
|
94
102
|
const port = readEmbedPort(portFile);
|
|
95
|
-
if (port !== null) registerRouterProvider(pi, port, cfg);
|
|
103
|
+
if (port !== null) registerRouterProvider(pi, port, cfg, sessionId);
|
|
96
104
|
return;
|
|
97
105
|
}
|
|
98
106
|
|
|
@@ -108,7 +116,7 @@ export default function (pi: ExtensionAPI): void {
|
|
|
108
116
|
|
|
109
117
|
// Publish the shared port; subagents and the toast read it from here.
|
|
110
118
|
writeEmbedPort(portFile, actualPort);
|
|
111
|
-
registerRouterProvider(pi, actualPort, cfg);
|
|
119
|
+
registerRouterProvider(pi, actualPort, cfg, sessionId);
|
|
112
120
|
|
|
113
121
|
pi.on("session_shutdown", () => {
|
|
114
122
|
void app?.stop().catch(() => {});
|
|
@@ -73,6 +73,11 @@ export default function (pi: ExtensionAPI): void {
|
|
|
73
73
|
// poll loop entirely rather than waking every 2s to do nothing.
|
|
74
74
|
if (!ctx.hasUI) return;
|
|
75
75
|
|
|
76
|
+
// This session's omp id. The embed extension tags every request with it
|
|
77
|
+
// (X-Omp-Session), so filtering on it scopes toasts to this session even
|
|
78
|
+
// when several omp sessions share one embedded router's ledger.
|
|
79
|
+
const sessionId = ctx.sessionManager.getSessionId();
|
|
80
|
+
|
|
76
81
|
// The poll request's own deadline (3s) exceeds the poll period (2s), so
|
|
77
82
|
// a slow router could let a second tick start while the first is still in
|
|
78
83
|
// flight — both read the same lastSeenId and raise duplicate toasts. An
|
|
@@ -115,7 +120,7 @@ export default function (pi: ExtensionAPI): void {
|
|
|
115
120
|
const entries = body.entries;
|
|
116
121
|
if (!Array.isArray(entries) || entries.length === 0) return;
|
|
117
122
|
|
|
118
|
-
for (const t of selectToasts(entries, lastSeenId, HARNESS_ID)) {
|
|
123
|
+
for (const t of selectToasts(entries, lastSeenId, HARNESS_ID, sessionId)) {
|
|
119
124
|
ctx.ui.notify(t.text, "info");
|
|
120
125
|
}
|
|
121
126
|
lastSeenId = newestId(entries) ?? lastSeenId;
|
|
@@ -83,6 +83,11 @@ export interface ToastDecision {
|
|
|
83
83
|
wasted: boolean;
|
|
84
84
|
/** Harness id from the request header; empty for the default harness. */
|
|
85
85
|
harnessId: string;
|
|
86
|
+
/**
|
|
87
|
+
* omp UI session id from the `X-Omp-Session` header; empty for the no-header
|
|
88
|
+
* default. Lets the toast scope to a single interactive session.
|
|
89
|
+
*/
|
|
90
|
+
ompSessionId?: string;
|
|
86
91
|
}
|
|
87
92
|
|
|
88
93
|
export interface ToastMessage {
|
|
@@ -111,6 +116,7 @@ export function selectToasts(
|
|
|
111
116
|
entries: ToastDecision[],
|
|
112
117
|
lastSeenId: string | null,
|
|
113
118
|
harnessId = "",
|
|
119
|
+
ompSessionId = "",
|
|
114
120
|
): ToastMessage[] {
|
|
115
121
|
if (lastSeenId === null) return [];
|
|
116
122
|
// `entries` is newest-first. Entries strictly newer than lastSeenId are the
|
|
@@ -124,6 +130,7 @@ export function selectToasts(
|
|
|
124
130
|
if (d === undefined) continue;
|
|
125
131
|
if (d.wasted) continue;
|
|
126
132
|
if (harnessId !== "" && d.harnessId !== harnessId) continue;
|
|
133
|
+
if (ompSessionId !== "" && d.ompSessionId !== ompSessionId) continue;
|
|
127
134
|
out.push({ model: d.servedSlug ?? d.slug, tier: d.tier, costUsd: d.reportedUsd, text: toToastText(d) });
|
|
128
135
|
}
|
|
129
136
|
return out;
|
package/package.json
CHANGED
package/src/cost/ledger.ts
CHANGED
|
@@ -32,6 +32,7 @@ interface LedgerRow {
|
|
|
32
32
|
turn: number;
|
|
33
33
|
requested_model: string;
|
|
34
34
|
harness_id: string;
|
|
35
|
+
omp_session_id: string;
|
|
35
36
|
slug: string;
|
|
36
37
|
served_slug: string | null;
|
|
37
38
|
tier: string;
|
|
@@ -128,6 +129,7 @@ function toEntry(row: LedgerRow): LedgerEntry {
|
|
|
128
129
|
turn: row.turn,
|
|
129
130
|
requestedModel: row.requested_model,
|
|
130
131
|
harnessId: row.harness_id,
|
|
132
|
+
ompSessionId: row.omp_session_id,
|
|
131
133
|
slug: row.slug,
|
|
132
134
|
servedSlug: row.served_slug,
|
|
133
135
|
tier: row.tier,
|
|
@@ -151,11 +153,11 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
|
|
|
151
153
|
// Prepared once: record() runs on every turn.
|
|
152
154
|
const insertStmt = db.query(
|
|
153
155
|
`INSERT INTO ledger (
|
|
154
|
-
id, created_at_ms, conversation_key, session_id, turn, requested_model, harness_id, slug, served_slug,
|
|
156
|
+
id, created_at_ms, conversation_key, session_id, turn, requested_model, harness_id, omp_session_id, slug, served_slug,
|
|
155
157
|
tier, classification_source, reasons, predicted_usd, reported_usd, usage, cost_breakdown,
|
|
156
158
|
attempt, escalation_signal, latency_ms, ttft_ms, finish_reason, wasted, upstream_generation_id, error,
|
|
157
159
|
error_kind
|
|
158
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
160
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
159
161
|
);
|
|
160
162
|
const calibrationStmt = db.query(
|
|
161
163
|
`INSERT INTO token_calibration (tokenizer, est_bytes, actual_tokens, samples) VALUES (?, ?, ?, 1)
|
|
@@ -215,6 +217,7 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
|
|
|
215
217
|
entry.turn,
|
|
216
218
|
entry.requestedModel,
|
|
217
219
|
entry.harnessId,
|
|
220
|
+
entry.ompSessionId,
|
|
218
221
|
entry.slug,
|
|
219
222
|
entry.servedSlug,
|
|
220
223
|
entry.tier,
|
package/src/cost/types.ts
CHANGED
|
@@ -74,6 +74,12 @@ export interface LedgerEntry {
|
|
|
74
74
|
requestedModel: string;
|
|
75
75
|
/** Harness id from the request header; empty for the default harness. */
|
|
76
76
|
harnessId: string;
|
|
77
|
+
/**
|
|
78
|
+
* omp UI session id from the `X-Omp-Session` request header; empty when the
|
|
79
|
+
* client sends no header. Scopes toasts to a single interactive session so
|
|
80
|
+
* concurrent sessions sharing one ledger don't surface each other's choices.
|
|
81
|
+
*/
|
|
82
|
+
ompSessionId: string;
|
|
77
83
|
/** Concrete slug we dispatched to. */
|
|
78
84
|
slug: string;
|
|
79
85
|
/** Slug that actually served it, per the response `model` field. */
|
package/src/index.ts
CHANGED
|
@@ -33,7 +33,6 @@ Global options:
|
|
|
33
33
|
|
|
34
34
|
serve --port <n> --host <addr> --log <level>
|
|
35
35
|
stats --days <n> --json
|
|
36
|
-
stats --days <n> --json
|
|
37
36
|
models --tier <trivial|simple|moderate|hard> --limit <n> --json
|
|
38
37
|
explain --file <request.json> --json (reads stdin when --file is absent)
|
|
39
38
|
config --print --write --path <models.yml> --config <router-config.yml>
|
package/src/server/turn.ts
CHANGED
package/src/util/sqlite.ts
CHANGED
|
@@ -18,7 +18,7 @@ import { mkdirSync } from "node:fs";
|
|
|
18
18
|
import { dirname } from "node:path";
|
|
19
19
|
|
|
20
20
|
/** Bump when a migration is added; guarded below so reopening never regresses it. */
|
|
21
|
-
const USER_VERSION =
|
|
21
|
+
const USER_VERSION = 5;
|
|
22
22
|
|
|
23
23
|
const MIGRATIONS = `
|
|
24
24
|
CREATE TABLE IF NOT EXISTS catalog_cache (
|
|
@@ -37,6 +37,7 @@ CREATE TABLE IF NOT EXISTS ledger (
|
|
|
37
37
|
turn INTEGER NOT NULL,
|
|
38
38
|
requested_model TEXT NOT NULL,
|
|
39
39
|
harness_id TEXT NOT NULL DEFAULT '',
|
|
40
|
+
omp_session_id TEXT NOT NULL DEFAULT '',
|
|
40
41
|
slug TEXT NOT NULL,
|
|
41
42
|
served_slug TEXT,
|
|
42
43
|
tier TEXT NOT NULL,
|
|
@@ -118,6 +119,15 @@ END
|
|
|
118
119
|
WHERE error IS NOT NULL;
|
|
119
120
|
`;
|
|
120
121
|
|
|
122
|
+
// v5: ledger gains omp_session_id, so the toast extension can scope decisions to
|
|
123
|
+
// its own omp session. Before this, the only scoping was per-harness, so two
|
|
124
|
+
// interactive omp sessions of the same harness (the default: empty) each
|
|
125
|
+
// surfaced the other's routing toasts from the shared ledger. Existing rows
|
|
126
|
+
// backfill to '' (unknown session), matching the no-header default.
|
|
127
|
+
const MIGRATE_V5 = `
|
|
128
|
+
ALTER TABLE ledger ADD COLUMN omp_session_id TEXT NOT NULL DEFAULT '';
|
|
129
|
+
`;
|
|
130
|
+
|
|
121
131
|
export function openDb(path: string): Database {
|
|
122
132
|
// ":memory:" has no parent directory to create.
|
|
123
133
|
if (path !== ":memory:") mkdirSync(dirname(path), { recursive: true });
|
|
@@ -134,6 +144,7 @@ export function openDb(path: string): Database {
|
|
|
134
144
|
const ledgerCols = db.query("PRAGMA table_info(ledger)").all() as { name: string }[];
|
|
135
145
|
if (!ledgerCols.some((c) => c.name === "harness_id")) db.exec(MIGRATE_V3);
|
|
136
146
|
if (!ledgerCols.some((c) => c.name === "error_kind")) db.exec(MIGRATE_V4);
|
|
147
|
+
if (!ledgerCols.some((c) => c.name === "omp_session_id")) db.exec(MIGRATE_V5);
|
|
137
148
|
db.exec(`PRAGMA user_version = ${USER_VERSION}`);
|
|
138
149
|
}
|
|
139
150
|
return db;
|
|
@@ -204,6 +204,10 @@ export function parseChatRequest(body: unknown, headers: Headers): NormRequest {
|
|
|
204
204
|
// this via the provider block's `headers:` override; absent ⇒ single harness.
|
|
205
205
|
const harnessId = (headers.get("x-omp-harness") ?? "").trim();
|
|
206
206
|
|
|
207
|
+
// omp UI session id for per-session toast scoping. The embed extension sets
|
|
208
|
+
// this header to ctx.sessionManager.getSessionId(); absent ⇒ unknown session.
|
|
209
|
+
const ompSessionId = (headers.get("x-omp-session") ?? "").trim();
|
|
210
|
+
|
|
207
211
|
if (typeof b.model !== "string" || b.model.length === 0) {
|
|
208
212
|
throw invalidRequest("model must be a non-empty string");
|
|
209
213
|
}
|
|
@@ -262,6 +266,7 @@ export function parseChatRequest(body: unknown, headers: Headers): NormRequest {
|
|
|
262
266
|
protocol: "openai-chat",
|
|
263
267
|
conversationKey,
|
|
264
268
|
harnessId,
|
|
269
|
+
ompSessionId,
|
|
265
270
|
requestedModel,
|
|
266
271
|
messages,
|
|
267
272
|
tools,
|
package/src/wire/types.ts
CHANGED
|
@@ -68,6 +68,13 @@ export interface NormRequest {
|
|
|
68
68
|
* client sends no header (single-harness default).
|
|
69
69
|
*/
|
|
70
70
|
harnessId: string;
|
|
71
|
+
/**
|
|
72
|
+
* omp UI session id from the `X-Omp-Session` request header, when the client
|
|
73
|
+
* sends one. Scopes toasts to a single interactive session so concurrent
|
|
74
|
+
* sessions sharing one router don't surface each other's choices. Empty when
|
|
75
|
+
* the client sends no header.
|
|
76
|
+
*/
|
|
77
|
+
ompSessionId: string;
|
|
71
78
|
/** Virtual model the client selected, e.g. `auto`, `auto-cheap`, `auto-max`. */
|
|
72
79
|
requestedModel: string;
|
|
73
80
|
messages: NormMessage[];
|
package/test/escalate.test.ts
CHANGED
package/test/failover.test.ts
CHANGED
package/test/toast-logic.test.ts
CHANGED
|
@@ -141,6 +141,38 @@ describe("selectToasts", () => {
|
|
|
141
141
|
];
|
|
142
142
|
expect(selectToasts(entries, "", "")).toHaveLength(2);
|
|
143
143
|
});
|
|
144
|
+
|
|
145
|
+
test("filters to the requesting omp session when one is set", () => {
|
|
146
|
+
// Two interactive omp sessions sharing one router's ledger: session-a's
|
|
147
|
+
// toast must not surface session-b's decisions.
|
|
148
|
+
const entries = [
|
|
149
|
+
dec({ id: "d3", slug: "mine", ompSessionId: "sess-a" }),
|
|
150
|
+
dec({ id: "d2", slug: "other", ompSessionId: "sess-b" }),
|
|
151
|
+
dec({ id: "d1", slug: "prior", ompSessionId: "sess-a" }),
|
|
152
|
+
];
|
|
153
|
+
const toasts = selectToasts(entries, "d1", "", "sess-a");
|
|
154
|
+
expect(toasts).toHaveLength(1);
|
|
155
|
+
expect(toasts[0]?.model).toBe("mine");
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
test("empty omp session id toasts every session", () => {
|
|
159
|
+
const entries = [
|
|
160
|
+
dec({ id: "d2", slug: "a", ompSessionId: "sess-a" }),
|
|
161
|
+
dec({ id: "d1", slug: "b", ompSessionId: "sess-b" }),
|
|
162
|
+
];
|
|
163
|
+
expect(selectToasts(entries, "", "", "")).toHaveLength(2);
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
test("harness and session filters compose", () => {
|
|
167
|
+
const entries = [
|
|
168
|
+
dec({ id: "d3", slug: "keep", harnessId: "h", ompSessionId: "sess-a" }),
|
|
169
|
+
dec({ id: "d2", slug: "wrong-session", harnessId: "h", ompSessionId: "sess-b" }),
|
|
170
|
+
dec({ id: "d1", slug: "wrong-harness", harnessId: "other", ompSessionId: "sess-a" }),
|
|
171
|
+
];
|
|
172
|
+
const toasts = selectToasts(entries, "", "h", "sess-a");
|
|
173
|
+
expect(toasts).toHaveLength(1);
|
|
174
|
+
expect(toasts[0]?.model).toBe("keep");
|
|
175
|
+
});
|
|
144
176
|
});
|
|
145
177
|
|
|
146
178
|
describe("toToastText", () => {
|
package/test/tokens.test.ts
CHANGED
|
@@ -16,6 +16,7 @@ function entry(over: Partial<LedgerEntry>): LedgerEntry {
|
|
|
16
16
|
turn: 1,
|
|
17
17
|
requestedModel: "auto",
|
|
18
18
|
harnessId: "",
|
|
19
|
+
ompSessionId: "",
|
|
19
20
|
slug: "vendor/model",
|
|
20
21
|
servedSlug: "vendor/model",
|
|
21
22
|
tier: "simple",
|
|
@@ -163,11 +164,24 @@ describe("v4 migration", () => {
|
|
|
163
164
|
}
|
|
164
165
|
});
|
|
165
166
|
|
|
166
|
-
test("schema is at user_version
|
|
167
|
+
test("schema is at user_version 5", () => {
|
|
167
168
|
const db = openDb(":memory:");
|
|
168
169
|
try {
|
|
169
170
|
const row = db.query("PRAGMA user_version").get() as { user_version: number };
|
|
170
|
-
expect(row.user_version).toBe(
|
|
171
|
+
expect(row.user_version).toBe(5);
|
|
172
|
+
} finally {
|
|
173
|
+
db.close();
|
|
174
|
+
}
|
|
175
|
+
});
|
|
176
|
+
|
|
177
|
+
test("persists omp_session_id and returns it via recentEntries", () => {
|
|
178
|
+
const db = openDb(":memory:");
|
|
179
|
+
try {
|
|
180
|
+
const ledger = createLedger(db, cfg);
|
|
181
|
+
ledger.record(entry({ ompSessionId: "sess-a" }));
|
|
182
|
+
ledger.record(entry({ ompSessionId: "" }));
|
|
183
|
+
const got = ledger.recentEntries(10).map((e) => e.ompSessionId).sort();
|
|
184
|
+
expect(got).toEqual(["", "sess-a"]);
|
|
171
185
|
} finally {
|
|
172
186
|
db.close();
|
|
173
187
|
}
|
package/test/turn.test.ts
CHANGED
|
@@ -58,6 +58,17 @@ describe("parseChatRequest normalization", () => {
|
|
|
58
58
|
expect(prefixed.requestedModel).toBe("auto");
|
|
59
59
|
});
|
|
60
60
|
|
|
61
|
+
test("reads harness and omp session ids from headers, trimmed", () => {
|
|
62
|
+
const headers = new Headers({ "x-omp-harness": " prod-a ", "x-omp-session": " sess-1 " });
|
|
63
|
+
const req = parseChatRequest(userBody("hi"), headers);
|
|
64
|
+
expect(req.harnessId).toBe("prod-a");
|
|
65
|
+
expect(req.ompSessionId).toBe("sess-1");
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
test("omp session id defaults to empty when the header is absent", () => {
|
|
69
|
+
expect(parseChatRequest(userBody("hi"), HEADERS).ompSessionId).toBe("");
|
|
70
|
+
});
|
|
71
|
+
|
|
61
72
|
test("tool schemas, names, and descriptions contribute to promptBytes", () => {
|
|
62
73
|
const parameters = { type: "object", properties: { path: { type: "string" } } };
|
|
63
74
|
const withTools = parseChatRequest(
|