omp-llm-role 1.2.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -6
- package/docs/dev/architecture.md +42 -19
- package/docs/dev/ci.md +9 -0
- package/docs/dev/current-state.md +126 -20
- package/docs/dev/explorer.md +126 -23
- package/docs/dev/releasing.md +66 -3
- package/docs/dev/scoring.md +76 -8
- package/docs/dev/testing.md +105 -14
- package/docs/dev/writes-and-state.md +161 -26
- package/docs/llm-role-rankings.md +115 -115
- package/package.json +19 -1
- package/src/agent-create.ts +4 -2
- package/src/availability.ts +40 -7
- package/src/engine.ts +33 -2
- package/src/explorer/boot.ts +43 -1
- package/src/explorer/explain.ts +101 -11
- package/src/explorer/levers.ts +100 -0
- package/src/explorer/pool.ts +46 -0
- package/src/explorer/server.ts +92 -21
- package/src/extension.ts +132 -2
- package/src/settings.ts +50 -0
- package/src/updater.ts +132 -33
- package/src/usage-ledger.ts +127 -0
- package/web/app.js +139 -1
- package/web/style.css +3 -0
package/docs/dev/explorer.md
CHANGED
|
@@ -12,7 +12,9 @@ explorer never reimplements `value = q − λ·$/M`.
|
|
|
12
12
|
| File | Role |
|
|
13
13
|
|---|---|
|
|
14
14
|
| `src/explorer/server.ts` | Zero-dependency `node:http` surface: static SPA + JSON API. Exports `createExplorerServer`, `ExplorerOpts`. |
|
|
15
|
-
| `src/explorer/explain.ts` | Pure explanation layer (no I/O): rank rows with baseline deltas, per-model decomposition, inverse-cardinal targets,
|
|
15
|
+
| `src/explorer/explain.ts` | Pure explanation layer (no I/O): rank rows with baseline deltas, per-model decomposition, inverse-cardinal targets, the keyed-catalog availability overlay, and the display-only per-task price. Exports `rankRows`, `explainModel`, `perTaskPrice`, `inverseCardinal`, `METRIC_META`, `WEIGHTABLE_METRICS`. |
|
|
16
|
+
| `src/explorer/levers.ts` | Pure price-lever readouts (no I/O): the leader-flip threshold and the thinking level's effect on the leader, computed on the pool they are handed. Exports `leaderFlip`, `thinkingLevelEffect`, `LeaderFlip`, `ThinkingLevelRow`. |
|
|
17
|
+
| `src/explorer/pool.ts` | The as-written pool (no I/O): the dataset filtered exactly as the updater filters it before ranking (catalog/tier gate + allowed-providers prune). Exports `asWrittenPool`, `PoolMode`, `PoolInputs`. |
|
|
16
18
|
| `src/explorer/scopes.ts` | Scope resolution (no HTTP): the user-level scope, the registry's projects, and the session's project; the per-scope read; the dataset union. Exports `Scope`, `resolveScopes`, `defaultScopeId`, `resolveScope`, `readScopeRoles`, `unionRoles`. |
|
|
17
19
|
| `src/explorer/boot.ts` | Shared launcher: bind/port fallback, scope resolution, browser open, close. Exports `startExplorer`, `EXPLORER_DEFAULT_PORT`, `ExplorerHandle`, `ExplorerBootOpts`. |
|
|
18
20
|
| `src/project-registry.ts` | The global project registry (`llm-role-projects.json` in the agent dir): read/register/prune, atomic write. Exports `PROJECT_REGISTRY_FILE`, `readProjectRegistry`, `registerProject`. |
|
|
@@ -47,12 +49,29 @@ type ExplorerOpts = {
|
|
|
47
49
|
availability: KeyAvailability; // keyed-catalog overlay (re-derived on refresh)
|
|
48
50
|
};
|
|
49
51
|
refresh(): Promise<void>; // re-runs loadRankData({ refresh: true, roles, extraMetrics }) and swaps rank
|
|
52
|
+
// The as-written pool's inputs: the omp catalog, the key's tier, and the
|
|
53
|
+
// account's allowed-providers whitelist (null = no whitelist / harvest failed,
|
|
54
|
+
// which leaves every route in place). Re-derived per call, so a project's
|
|
55
|
+
// whitelist cannot leak into another scope's view.
|
|
56
|
+
getPoolInputs(): PoolInputs;
|
|
57
|
+
// The recorded usage rows the per-task price resolves its token profile from,
|
|
58
|
+
// for the request's active scope (a project scope reads its own `<root>/.omp`
|
|
59
|
+
// ledger). Optional: a host without a ledger supplies none.
|
|
60
|
+
getUsageRows?(scope: Scope): readonly UsageRow[];
|
|
61
|
+
// The user-set token profile the per-task column falls back to (the
|
|
62
|
+
// `tokenProfile` setting); null (the default) = unset.
|
|
63
|
+
tokenProfile?: TokenProfile | null;
|
|
50
64
|
};
|
|
51
65
|
```
|
|
52
66
|
|
|
53
|
-
The server holds the active scope id
|
|
54
|
-
`rank`/`availability`); `activeScope` resolves
|
|
55
|
-
fallback when the active scope became unavailable.
|
|
67
|
+
The server holds the active scope id **and the pool mode** as mutable state
|
|
68
|
+
(alongside the mutable `rank`/`availability`); `activeScope` resolves the scope
|
|
69
|
+
per request, persisting the fallback when the active scope became unavailable.
|
|
70
|
+
`poolRank` applies the session's pool mode to the dataset before ranking — the
|
|
71
|
+
`as-written` mode runs `asWrittenPool(rank, opts.getPoolInputs())`, the `raw`
|
|
72
|
+
mode (the default, the historical behaviour) passes the fetch through. The pool
|
|
73
|
+
mode is **never** applied to the export path, which validates a pin against the
|
|
74
|
+
raw dataset.
|
|
56
75
|
|
|
57
76
|
Endpoints:
|
|
58
77
|
|
|
@@ -62,8 +81,9 @@ Endpoints:
|
|
|
62
81
|
| `GET /app.js`, `/style.css` | — | static asset (traversal-guarded) |
|
|
63
82
|
| `GET /api/bootstrap` | — | `bootstrapPayload(opts, activeScope)` |
|
|
64
83
|
| `POST /api/scope` | `{scope}` | switch the active scope, then the bootstrap payload (404 unknown id, 409 non-present) |
|
|
65
|
-
| `POST /api/
|
|
66
|
-
| `POST /api/
|
|
84
|
+
| `POST /api/pool` | `{mode}` | switch the ranked pool (`"raw"` \| `"as-written"`), then the bootstrap payload (400 on any other mode) |
|
|
85
|
+
| `POST /api/rank` | `{role, def}` | `{eligible, lambda, derivedLambda, rows, errors}` — each row carries the display-only `perTask` |
|
|
86
|
+
| `POST /api/explain` | `{role, def, modelId}` | `explainModel(...)` spread + `leaderFlip` + `thinkingLevels` + `errors` |
|
|
67
87
|
| `POST /api/export` | `{roles}` | `{ok:true, backupPath, roles, lockPath}` or `{ok:false, errors}` |
|
|
68
88
|
| `POST /api/refresh` | — | `await opts.refresh()` (re-ranks and re-derives the availability overlay) then the bootstrap payload |
|
|
69
89
|
|
|
@@ -75,7 +95,9 @@ Endpoints:
|
|
|
75
95
|
focus metrics over the resolved dataset), `levels` (`Object.keys(SUFFIX_LEVELS)`), `thinkingFactors`
|
|
76
96
|
(per-level billed-blend multiplier from the engine's own `thinkingPriceFactor`,
|
|
77
97
|
so the UI readout cannot drift), `fetchedAt`, `modelCount`, `orMatched`,
|
|
78
|
-
`orPriced`, `
|
|
98
|
+
`orPriced`, `poolMode` (the ranked pool the counts describe — `"raw"` or
|
|
99
|
+
`"as-written"`), `tokenProfile` (the user-set per-task token profile, or
|
|
100
|
+
`null`), `lockPath` (the **active scope's** lock file), `scopes` (id, label,
|
|
79
101
|
kind, present per known scope), `activeScope` (the active scope id),
|
|
80
102
|
the active scope's global capability flags and the feature registry (the
|
|
81
103
|
`FEATURES` rows: id, label, description, recommended bundle, buys), so the SPA
|
|
@@ -124,38 +146,71 @@ Pure, no I/O; all ranking math is delegated to `src/engine.ts`.
|
|
|
124
146
|
declared?)`** merges the shipped table with a derived entry per external metric.
|
|
125
147
|
- **`weightableMetrics(extra?)`** — the shipped keys plus any external metric in
|
|
126
148
|
`extra`; exactly what the validator accepts.
|
|
127
|
-
- **`rankRows(def, models, baseline, availability?)`** — `rankRole(def, models)` +
|
|
149
|
+
- **`rankRows(def, models, baseline, availability?, profile?)`** — `rankRole(def, models)` +
|
|
128
150
|
`paretoFrontier`, annotating each row with `baselineRank`/`delta`
|
|
129
151
|
(`baselineRank − rank`; positive = moved up) against the effective role's
|
|
130
152
|
ranking. Rows carry `priceEff` (the thinking-adjusted price the ranking
|
|
131
153
|
penalized), `parts`, `missing` (weighted non-price metrics whose raw value is
|
|
132
|
-
null), `frontier`,
|
|
154
|
+
null), `frontier`, `key` — the availability overlay verdict
|
|
133
155
|
(`usable`/`blocked`/`unknown`), orthogonal to `missing` and to the eligibility
|
|
134
|
-
gates
|
|
135
|
-
|
|
156
|
+
gates — and `perTask` (the display-only dollars-per-task, or `null`).
|
|
157
|
+
`availability` and `profile` are optional and pure (no I/O); omitted or
|
|
158
|
+
inactive ⇒ every row reads `unknown` / `perTask: null`. Neither affects the
|
|
159
|
+
ranking order or the deltas.
|
|
136
160
|
- **`inverseCardinal(metric, t)`** — inverse of `cardinalMetric` for the "what
|
|
137
161
|
would it take" targets: `index` → `t*80 − 20`; `throughput` → `10 * 30**t`
|
|
138
162
|
(null outside `[0,1]`, because the forward transform clamps); everything else
|
|
139
163
|
(benchmark, percentile, price) → identity. See *Open defect* below for the
|
|
140
164
|
`gpqa` exception.
|
|
141
|
-
- **`
|
|
165
|
+
- **`perTaskPrice(model, def, profile)`** — the display-only dollars-per-task:
|
|
166
|
+
`p_in` is the role's priced route's input price (cache-adjusted by the role's
|
|
167
|
+
`cacheHitRate`, `routeInputPrice`) and `p_out` that same route's output price —
|
|
168
|
+
the inverse 3:1 blend of it (`routeOutputPrice`), never re-derived here — and
|
|
169
|
+
the output term is scaled by the role's thinking factor
|
|
170
|
+
(`thinkingAdjustedPrice(model, def, 1)`, so the per-task price and the row's
|
|
171
|
+
`priceEff` cannot disagree). When the role prices on the `1/price²` blend
|
|
172
|
+
(`roleRoute` returns `null`) the model's billed blend and the input blend over
|
|
173
|
+
the routes that survive the role's endpoint filters stand in — the same pool the
|
|
174
|
+
ranking prices on, so `p_in` and the billed blend never mix two route pools.
|
|
175
|
+
`null` when there is no usable route price — **absent, never a silent
|
|
176
|
+
zero**. The SPA renders `—` in both cases: no token profile resolves (no
|
|
177
|
+
recorded usage and no `tokenProfile` set), **or** the model has no usable
|
|
178
|
+
route price (e.g. every route pruned by the as-written whitelist — the model
|
|
179
|
+
still ranks on its blend price, but the per-task column has no route to
|
|
180
|
+
price). It never feeds the ranking's sort key, `value`, or the explanation's
|
|
181
|
+
`cost` block.
|
|
182
|
+
- **`explainModel(def, models, modelId, roleName, availability?, profile?)`** — the full
|
|
142
183
|
decomposition, or the eligibility gates it failed (same gates `rankRole`
|
|
143
184
|
applies, same order: missing required, image filter, no billed price). On
|
|
144
185
|
success: `contributions`
|
|
145
186
|
(raw → cardinal `t` → renormalized weight → contribution → share of `q`, with
|
|
146
187
|
a `fillNote` when the value is a capability fill), `cost` (`priceEff`, the
|
|
147
188
|
cache-adjusted billed price, λ, penalty, `q`, `value`), `role.cacheHitRate`,
|
|
148
|
-
`
|
|
189
|
+
`perTask` (the `PerTaskPrice` breakdown, or `null` when no profile was
|
|
190
|
+
supplied or the model has no usable route price), `gapAbove`/`gapToTop`/`above`, `closing` (the
|
|
149
191
|
per-metric raw target that would close the gap, via `inverseCardinal`, with
|
|
150
192
|
unreachable/extrapolated notes), `dominators` (models both cheaper and at
|
|
151
193
|
least as good on `q`, top 3), `ownProvider` (true when the model was priced on
|
|
152
|
-
the soft best route rather than the blend), `autoPin` (the
|
|
153
|
-
|
|
194
|
+
the soft best route rather than the blend), `autoPin` (the **candidate** route's
|
|
195
|
+
provider slug — `bestRoute(model, def)?.providerSlug`, the max-`vfm`
|
|
154
196
|
route (scoring.md, *Route ordering*), when the role sets
|
|
155
|
-
`preferOwnProvider` and carries no manual `providerPin`, else `null`
|
|
197
|
+
`preferOwnProvider` and carries no manual `providerPin`, else `null` — the
|
|
198
|
+
first route the updater's daily walk tries, **not** a promise of the written
|
|
199
|
+
pin: the updater probes it and skips to the next route when OpenRouter does not
|
|
200
|
+
serve it, or drops the pin entirely when none is routable; writes-and-state.md,
|
|
201
|
+
*The provider-pin auto-configuration*), and the
|
|
156
202
|
availability overlay `key`/`keyReason`
|
|
157
203
|
(the same verdict as `rankRows`, plus the reason it reads that way). It mirrors
|
|
158
204
|
`rankRole`'s capability fill so the contributions sum exactly to `q`.
|
|
205
|
+
- **`leaderFlip(def, models)`** / **`thinkingLevelEffect(def, models)`**
|
|
206
|
+
(`src/explorer/levers.ts`) — the two price-lever readouts, computed on the pool
|
|
207
|
+
they are handed (so the readout and the ranking above it agree). `leaderFlip`
|
|
208
|
+
returns the price weight at which a cheaper challenger overtakes the value
|
|
209
|
+
leader (`{ wPrice, flipW, distance, challenger }`), or `null` when no cheaper
|
|
210
|
+
challenger exists. `thinkingLevelEffect` returns one `{ level, factor, leader }`
|
|
211
|
+
row per thinking level, or `null` for a bare role. Both reuse the engine's own
|
|
212
|
+
`roleLambda`/`rankRole`/`SUFFIX_LEVELS`/`thinkingPriceFactor`; the formulas are
|
|
213
|
+
in [`scoring.md`](scoring.md).
|
|
159
214
|
|
|
160
215
|
## Shared boot path (`src/explorer/boot.ts`)
|
|
161
216
|
|
|
@@ -202,6 +257,22 @@ launcher `/explore-roles` uses.
|
|
|
202
257
|
factor on the omp catalog; `refresh()` re-runs `opts.reload(true)`,
|
|
203
258
|
re-enriches, and — when `opts.reloadAvailability` is supplied — re-derives the
|
|
204
259
|
availability overlay.
|
|
260
|
+
- **As-written pool inputs.** `ExplorerBootOpts` carries `tier` (default
|
|
261
|
+
`"billed"`), `allowedProviders` (`null` = no pruning), and
|
|
262
|
+
`reloadAllowedProviders` (re-derived on `refresh()`). `getPoolInputs()` returns
|
|
263
|
+
`{ catalog: opts.catalog, tier, allowed }` per call, so a project's whitelist
|
|
264
|
+
cannot leak into another scope's view. Both are **best-effort**: an absent tier
|
|
265
|
+
reads as `billed`, and a failed re-harvest (`reloadAllowedProviders` throwing
|
|
266
|
+
or returning `null`) degrades the as-written pool to no route pruning rather
|
|
267
|
+
than failing the refresh. `/explore-roles` builds them in-process from the
|
|
268
|
+
session's OpenRouter key (the canary must be a **catalog** slug, not a ranking
|
|
269
|
+
id).
|
|
270
|
+
- **Usage ledger.** `getUsageRows(scope)` reads the active scope's ledger: a
|
|
271
|
+
project scope reads its own `<root>/.omp` ledger, the user scope reads
|
|
272
|
+
`usageDir` (default `agentDir()`), so a project's usage never mixes with the
|
|
273
|
+
user-level profile. Re-read per call, so a turn recorded while the explorer is
|
|
274
|
+
open shows up without a restart. `tokenProfile` is the user-set fallback the
|
|
275
|
+
per-task column uses when a role has no recorded usage.
|
|
205
276
|
- `opts.availability` is the boot-time keyed-catalog overlay; `getState(scope)`
|
|
206
277
|
returns it on every request, so `/api/rank` and `/api/explain` annotate rows
|
|
207
278
|
without a second fetch. It is scope-independent and carried across switches
|
|
@@ -246,12 +317,23 @@ a scraped third-party site, so the DOM is built with
|
|
|
246
317
|
`textContent`/`createElement` only (the `el()` helper).
|
|
247
318
|
|
|
248
319
|
- **Boot**: `boot()` → `GET /api/bootstrap` → `applyBootstrap(data)`, which fills
|
|
249
|
-
`state` (`scopes`, `activeScope`, `
|
|
320
|
+
`state` (`scopes`, `activeScope`, `poolMode`, `tokenProfile`, `universe`,
|
|
321
|
+
`effective` = each entry's `def`,
|
|
250
322
|
`defs` = a structuredClone of it, `defaults`, `metrics`, `metricMeta`,
|
|
251
323
|
`focusAssessments`, `levels`, `thinkingFactors`, `lockPath`, `availability`),
|
|
252
324
|
renders the header/roles/table, and selects the previously selected role when
|
|
253
325
|
the new scope has it, else the first role. `boot()` then wires the
|
|
254
326
|
filter/topn/hide-key-blocked/export/copy/download controls.
|
|
327
|
+
- **Pool toggle**: the header renders a `#pool` `<select>` (`raw` / `as-written`)
|
|
328
|
+
from `state.poolMode`. Changing it confirms first when there are unsaved edits
|
|
329
|
+
(mirroring the scope switcher), then `POST /api/pool {mode}` and
|
|
330
|
+
`applyBootstrap` on the returned payload — so the header's model count, the
|
|
331
|
+
table, the focus assessments and the two readouts all describe the pool the
|
|
332
|
+
user is now looking at. The toggle is **session-level** state — the server
|
|
333
|
+
holds one pool mode, kept across scope switches (not persisted) — while the
|
|
334
|
+
pool *inputs* are re-derived per call, so a project's whitelist never leaks
|
|
335
|
+
into another scope's view. Export is unaffected: it always
|
|
336
|
+
writes the authored def, never the pool.
|
|
255
337
|
- **Scope switcher**: the header renders a `#scope` `<select>` from
|
|
256
338
|
`state.scopes` (the user-level scope plus every known project); a scope whose
|
|
257
339
|
lock file is gone is `disabled` with a reason `title`. Changing the selection
|
|
@@ -282,15 +364,36 @@ a scraped third-party site, so the DOM is built with
|
|
|
282
364
|
- **Recompute**: `selectRole` → `renderRoles`/`renderEditor`/`renderExplain` +
|
|
283
365
|
`recompute()`. `recompute()` → `POST /api/rank {role, def}` → `state.rows`,
|
|
284
366
|
`lambda`, `derivedLambda`, `errors`; re-renders the table and readouts, and
|
|
285
|
-
reloads the explain panel if the selected model is still present.
|
|
286
|
-
|
|
367
|
+
reloads the explain panel if the selected model is still present. The table
|
|
368
|
+
gains a **`$/task`** column (`r.perTask`, rendered `—` when `null`), beside the
|
|
369
|
+
`$/M` column. Editor changes call `scheduleRecompute()` (120 ms debounce).
|
|
287
370
|
- **Explain**: `selectModel(id)` → `POST /api/explain {role, def, modelId}` →
|
|
288
|
-
`renderExplain()` (composition table, cost line,
|
|
289
|
-
dominators,
|
|
371
|
+
`renderExplain()` (composition table, cost line, **per-task price** line, "why
|
|
372
|
+
not higher", dominators, the **Price levers** section, and the availability
|
|
373
|
+
line). The cost line appends `· best route`
|
|
290
374
|
when the model was priced on the soft best route (`ownProvider`), and a
|
|
291
375
|
**pin line** shows the pin in force: `pin: <slug> (manual)` when the edited def
|
|
292
|
-
carries a `providerPin`, else `auto pin: <slug>` from the
|
|
293
|
-
`autoPin` (read-only — the manual field is the only way to set
|
|
376
|
+
carries a `providerPin`, else `auto pin (candidate): <slug>` from the
|
|
377
|
+
explanation's `autoPin` (read-only — the manual field is the only way to set
|
|
378
|
+
one). The label says *candidate* on purpose: the value is the first route the
|
|
379
|
+
updater's value-ordered walk *tries*, not a promise of the written pin — the
|
|
380
|
+
updater probes that route and skips to the next when OpenRouter does not serve
|
|
381
|
+
it, or drops the pin entirely (bare selector) when none is routable, so the
|
|
382
|
+
written pin can be a later route or absent altogether (spec #90;
|
|
383
|
+
writes-and-state.md, *The provider-pin auto-configuration*). The explorer is
|
|
384
|
+
read-only and offline and cannot probe, so it names the candidate and says so
|
|
385
|
+
in the tooltip. The candidate equals the written pin when every route probes
|
|
386
|
+
clean; with the account's allowed-providers whitelist active, the panel and the
|
|
387
|
+
written pin agree only on the **as-written** pool, which applies the same prune
|
|
388
|
+
the updater applies before walking (in the raw pool the panel can name a route
|
|
389
|
+
the prune removes). The
|
|
390
|
+
per-task line shows the breakdown (`<in> in × $<p_in>/M + <out> out ×
|
|
391
|
+
$<p_out>/M × <factor> = $<total>`), or `— (no token profile — record usage or
|
|
392
|
+
set tokenProfile)` when `perTask` is `null`. The **Price levers** section
|
|
393
|
+
renders the leader-flip line (`w* = <flipW> · now <wPrice> · distance
|
|
394
|
+
<±distance> (<ratio>)`) and a per-level table of the thinking level's effect on
|
|
395
|
+
the leader; each is omitted when the server sends `null` (no cheaper
|
|
396
|
+
challenger, or a bare role).
|
|
294
397
|
- **Provider pin field**: the editor renders a text input for
|
|
295
398
|
`roles.<role>.providerPin` (a tiered slug like `deepinfra/fp8` is kept
|
|
296
399
|
verbatim; empty clears the key, resuming the automatic choice). It is the
|
package/docs/dev/releasing.md
CHANGED
|
@@ -122,6 +122,68 @@ path, plus `marketplace update` + `upgrade` to a bumped catalog version), and
|
|
|
122
122
|
push publishes to npm over OIDC and creates the GitHub Release. The guard was
|
|
123
123
|
verified with a negative control — tag `v0.0.1` against version `1.1.0` failed
|
|
124
124
|
in `notes` before any publish.
|
|
125
|
+
- **1.3.0 published by CD 2026-10-19** (run 39222193609, tag push): the guard
|
|
126
|
+
and the npm floor passed, `npm publish` completed over OIDC trusted publishing
|
|
127
|
+
with a provenance attestation (`npm audit signatures`: 1 package has a
|
|
128
|
+
verified attestation), and the GitHub Release was created with the 1.3.0
|
|
129
|
+
CHANGELOG section as its body
|
|
130
|
+
(`https://github.com/Gott50/omp-llm-role/releases/tag/v1.3.0`). Verified:
|
|
131
|
+
`dist-tags.latest` = `1.3.0`, npm's `gitHead` = `55416d4` (the tag's commit),
|
|
132
|
+
and the publish step's log carries the direct-publish signature (`Publishing
|
|
133
|
+
to https://registry.npmjs.org/ with tag latest and default access`, `Signed
|
|
134
|
+
provenance statement`, `+ omp-llm-role@1.3.0`). The cut (`55416d4`, tagged
|
|
135
|
+
`v1.3.0`) bumped the four lockstep fields, renamed `## [Unreleased]` to
|
|
136
|
+
`## [1.3.0] - 2026-10-19`, and regenerated the report (408 models, matched
|
|
137
|
+
149/408, priced 148 — byte-identical to the 2026-10-18 report apart from its
|
|
138
|
+
date line; `docs/dev/current-state.md` refreshed to the same measurement).
|
|
139
|
+
The cut also carried the #89 fix (the `GIT_*` scrub in the pre-commit hook and
|
|
140
|
+
`tests/ci.test.ts`), so the release commit went through the hook with the real
|
|
141
|
+
repository untouched.
|
|
142
|
+
- **1.3.0 install routes smoke-tested** (2026-10-19, omp 18.12.2, throwaway
|
|
143
|
+
`HOME`s, cwd outside the real home tree): npm
|
|
144
|
+
(`omp plugin install omp-llm-role`), git
|
|
145
|
+
(`omp plugin install github:Gott50/omp-llm-role`) and marketplace
|
|
146
|
+
(`omp plugin marketplace add Gott50/omp-llm-role` +
|
|
147
|
+
`omp plugin install omp-llm-role@gott50-plugins`) each installed 1.3.0
|
|
148
|
+
(`omp plugin doctor`: 4 ok / 0 warnings / 0 errors for npm and git; the
|
|
149
|
+
marketplace install's one warning is the expected
|
|
150
|
+
`package_manifest: Not created yet`, and it landed at
|
|
151
|
+
`…/cache/plugins/gott50-plugins___omp-llm-role___1.3.0`). Each post-install
|
|
152
|
+
session run (`omp -p "say ok"` with `OPENROUTER_API_KEY` from
|
|
153
|
+
`omp token openrouter`) wrote `modelRoles` (9 roles),
|
|
154
|
+
`retry.fallbackChains` and `task.disabledAgents: [designer]` into that HOME's
|
|
155
|
+
`config.yml`, landed `llm-role-state.json` + `llm-role-history.jsonl` +
|
|
156
|
+
`llm-role-usage.jsonl` (the spec #81 ledger, live for the first time), and
|
|
157
|
+
answered.
|
|
158
|
+
- **1.2.0 published by CD 2026-10-12** (run 38299141569, tag push): the guard
|
|
159
|
+
and the npm floor passed, `npm publish` completed over OIDC trusted publishing
|
|
160
|
+
with a provenance attestation (`npm audit signatures`: 1 package has a
|
|
161
|
+
verified attestation), and the GitHub Release was created with the 1.2.0
|
|
162
|
+
CHANGELOG section as its body
|
|
163
|
+
(`https://github.com/Gott50/omp-llm-role/releases/tag/v1.2.0`). Verified:
|
|
164
|
+
`dist-tags.latest` = `1.2.0`, npm's `gitHead` = `50fce66` (the tag's commit),
|
|
165
|
+
and the publish step's log carries the direct-publish signature (`Publishing
|
|
166
|
+
to https://registry.npmjs.org/ with tag latest`, `Signed provenance
|
|
167
|
+
statement`, `+ omp-llm-role@1.2.0`). The cut (`50fce66`, tagged `v1.2.0`)
|
|
168
|
+
bumped the four lockstep fields, renamed `## [Unreleased]` to
|
|
169
|
+
`## [1.2.0] - 2026-10-12`, and regenerated the report (408 models, matched
|
|
170
|
+
150/408, priced 148; `docs/dev/current-state.md` refreshed to the same
|
|
171
|
+
measurement).
|
|
172
|
+
- **1.2.0 install routes smoke-tested** (2026-10-12, omp 18.10.3, throwaway
|
|
173
|
+
`HOME`s, cwd outside the real home tree): npm
|
|
174
|
+
(`omp plugin install omp-llm-role`), git
|
|
175
|
+
(`omp plugin install github:Gott50/omp-llm-role`) and marketplace
|
|
176
|
+
(`omp plugin marketplace add Gott50/omp-llm-role` +
|
|
177
|
+
`omp plugin install omp-llm-role@gott50-plugins`) each installed 1.2.0
|
|
178
|
+
(`omp plugin doctor`: 4 ok / 0 warnings / 0 errors for npm and git; the
|
|
179
|
+
marketplace install's one warning is the expected
|
|
180
|
+
`package_manifest: Not created yet`). Each post-install session run
|
|
181
|
+
(`omp -p "say ok"` with `OPENROUTER_API_KEY` from `omp token openrouter`)
|
|
182
|
+
wrote `modelRoles` (9 roles), `retry.fallbackChains` and
|
|
183
|
+
`task.disabledAgents: [designer]` into that HOME's `config.yml`, landed
|
|
184
|
+
`llm-role-state.json` + `llm-role-history.jsonl`, and answered. The npm-route
|
|
185
|
+
run also exercised the spec #65 whitelist harvest live (`OpenRouter
|
|
186
|
+
allowed-providers whitelist active — 11 providers; 542 routes pruned`).
|
|
125
187
|
- **1.1.0 published by CD 2026-10-09** (run 37897647817,
|
|
126
188
|
`gh workflow run ci.yml --ref v1.1.0`): the guard and the npm floor passed,
|
|
127
189
|
`npm publish` completed over OIDC trusted publishing with a provenance
|
|
@@ -171,6 +233,7 @@ path, plus `marketplace update` + `upgrade` to a bumped catalog version), and
|
|
|
171
233
|
- **Stray registry placeholder.** The registry also carries a `0.0.0-stage`
|
|
172
234
|
version ("Temporary package placeholder for staged publishing", 334 B,
|
|
173
235
|
published 2026-10-03). It is not from this tree and does not affect installs
|
|
174
|
-
(`latest` is `1.
|
|
175
|
-
- **Next release:
|
|
176
|
-
|
|
236
|
+
(`latest` is `1.3.0`).
|
|
237
|
+
- **Next release: none owed.** `## [Unreleased]` is empty — the 1.3.0 cut moved
|
|
238
|
+
its entries under `## [1.3.0] - 2026-10-19`; the number comes from the
|
|
239
|
+
Unreleased scope, not from the open issues.
|
package/docs/dev/scoring.md
CHANGED
|
@@ -101,10 +101,13 @@ cache reads; the knob prices that, opt-in, without touching any shipped role.
|
|
|
101
101
|
- `endpointFilteredModel(model, filters, cacheHitRate)` recomputes the blend when
|
|
102
102
|
the role declares endpoint filters **or** a non-zero hit rate; the hit rate
|
|
103
103
|
composes with the filters (filter the pool first, then cache-price it);
|
|
104
|
-
- `
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
104
|
+
- the route-price accessors — `routeInputPrice` (the cache-adjusted input term),
|
|
105
|
+
`routeEffectivePrice` (the billed 3:1 blend, which reuses the input accessor),
|
|
106
|
+
`routeOutputPrice`/`blendOutputPrice` (the inverse blend) — are what the
|
|
107
|
+
blend, the pinned-route path, `roleMetricValue` and the explorer's per-task
|
|
108
|
+
price share; a pinned role prices its route's cache-adjusted billed price, and
|
|
109
|
+
`maxPriceUsdPerM` caps the thinking-adjusted `priceEff` (the same cap as the
|
|
110
|
+
blend path);
|
|
108
111
|
- `explainModel` mirrors all of it, carries `role.cacheHitRate`, and reports the
|
|
109
112
|
cache-adjusted `cost.billedPrice`; the SPA cost line appends `· cache <rate>`
|
|
110
113
|
when the rate is set;
|
|
@@ -149,9 +152,22 @@ while every other weighted metric stays the model's own.
|
|
|
149
152
|
### Route ordering (the value-for-money score)
|
|
150
153
|
|
|
151
154
|
`rankedRoutes(model, def)` (`src/engine.ts`) is the one function that orders a
|
|
152
|
-
model's candidate routes; `bestRoute` is `rankedRoutes[0]`, and `roleRoute`
|
|
153
|
-
route `rankRole` prices the model on) and the updater's auto-pin
|
|
154
|
-
|
|
155
|
+
model's candidate routes; `bestRoute` is `rankedRoutes[0]`, and both `roleRoute`
|
|
156
|
+
(the route `rankRole` prices the model on) and the updater's auto-pin read that
|
|
157
|
+
one order. The priced route is the score-max candidate; the **written pin is the
|
|
158
|
+
first route in that order the route probe does not reject** (writes-and-state.md,
|
|
159
|
+
*The provider-pin auto-configuration*), so the two can differ — the probe is a
|
|
160
|
+
routability check on the write side only and never feeds the ranking. The
|
|
161
|
+
fallback chain follows the pin: it excludes the route the pin names (by slug, not
|
|
162
|
+
by position) and, with a dropped pin, excludes none. Which **models** the chain
|
|
163
|
+
falls back to is a separate choice (spec #95): its `fallbackChainDepth` fallback
|
|
164
|
+
models come from the role's **ranked candidate list** below the chosen model —
|
|
165
|
+
the cheaper-or-equal candidates that retain at least `fallbackQualityFloor` of
|
|
166
|
+
its `q` first, then the pricier ones, each bucket in value order, truncated to
|
|
167
|
+
the depth — **not** from the probe window, so the chain can reach a
|
|
168
|
+
cheaper-or-equal model ranked below the `1 + fallbackChainDepth` window the walk
|
|
169
|
+
probes (key-blocked and probe-`blocked` candidates are excluded; unprobed ones
|
|
170
|
+
stay eligible — writes-and-state.md, *The provider-pin auto-configuration*).
|
|
155
171
|
|
|
156
172
|
- **Candidates.** A route is a candidate when it is non-degraded (`status === 0`),
|
|
157
173
|
passes `routePassesEndpointFilters(route, def.filters)`, carries a usable billed
|
|
@@ -191,6 +207,54 @@ filter, not a re-pricing: the role-independent `1/price²` blend (`m.price`/
|
|
|
191
207
|
`m.throughput`) is left as the account-agnostic expected value, and a model with
|
|
192
208
|
no allowed route keeps that blend (the soft basis never drops a model).
|
|
193
209
|
|
|
210
|
+
## Price levers (the explorer's readouts)
|
|
211
|
+
|
|
212
|
+
Two readouts make the price posture visible (spec #81). Both are pure
|
|
213
|
+
(`src/explorer/levers.ts`), computed on the pool the explorer is viewing, and
|
|
214
|
+
reuse the engine's own `roleLambda`/`rankRole`/`SUFFIX_LEVELS`/
|
|
215
|
+
`thinkingPriceFactor` — never a re-derivation.
|
|
216
|
+
|
|
217
|
+
### The leader-flip threshold
|
|
218
|
+
|
|
219
|
+
The price weight at which a **cheaper** challenger overtakes the value leader.
|
|
220
|
+
Let `L = rankRole(def, models)[0]` (the value leader). For every challenger `X`
|
|
221
|
+
with `pEff_X < pEff_L` (strictly cheaper), the flip λ is
|
|
222
|
+
`(q_L − q_X)/(pEff_L − pEff_X)`; the threshold is the minimum over those
|
|
223
|
+
challengers:
|
|
224
|
+
|
|
225
|
+
```
|
|
226
|
+
λ* = min over cheaper challengers X of (q_L − q_X)/(pEff_L − pEff_X)
|
|
227
|
+
w* = 20·λ* / (1 + 20·λ*)
|
|
228
|
+
```
|
|
229
|
+
|
|
230
|
+
`w*` is the inverse of `roleLambda`'s `λ = w/(1−w)/20` (`P_REF_USD = 20`). The
|
|
231
|
+
readout reports the role's current `wPrice`, the flip `w*`, the distance
|
|
232
|
+
`w* − wPrice`, and the challenger that would take over. **No cheaper challenger
|
|
233
|
+
⇒ `null`** — the readout is absent, never invented. The degenerate case (the
|
|
234
|
+
value leader is also the quality leader, where the naive
|
|
235
|
+
`(q_QL − q_VL)/(p_QL − p_VL)` is `0/0`) is handled by this same min formula: it
|
|
236
|
+
is *undefined*, not zero.
|
|
237
|
+
|
|
238
|
+
**Exactness.** The threshold is exact when the role's price basis is
|
|
239
|
+
weight-independent — the `1/price²` blend, or a hard `providerPin`. Under the
|
|
240
|
+
soft `preferOwnProvider` basis the route choice itself moves with the price
|
|
241
|
+
weight, so the threshold is a **first-order estimate**: measured on `smol`,
|
|
242
|
+
moving `w` from 0.45 to 0.9586 moves the leader's route `coreweave/fp8 →
|
|
243
|
+
decart/fp4`, so the flip is approximate there.
|
|
244
|
+
|
|
245
|
+
### The thinking level's effect
|
|
246
|
+
|
|
247
|
+
`thinkingLevelEffect(def, models)` re-ranks the role at each thinking level and
|
|
248
|
+
reports the leader per level, with the level's price factor
|
|
249
|
+
`thinkingPriceFactor(level) = (1 + 3·0.25 + THINKING_TOKEN_OVERHEAD[level]) /
|
|
250
|
+
(1 + 3·0.25)`. It is **inert for a bare role** (no thinking level) — the readout
|
|
251
|
+
is `null`. The factor is a **price-sensitivity amplifier, not a hidden cost**: it
|
|
252
|
+
is multiplicative on `priceEff`, so it penalizes expensive models more in
|
|
253
|
+
absolute λ-units — which is why `auto` favours the cheap model. On the user's
|
|
254
|
+
live `default` def: `off` → Gemini 4 Argon $2.99, `auto` → Gemini 4 Argon $5.55,
|
|
255
|
+
`high` → DeepSeek-V4.1-Flash $0.17. The readout shows this effect; it never
|
|
256
|
+
frames the level as a cost to remove.
|
|
257
|
+
|
|
194
258
|
## Capability presets (feature flags)
|
|
195
259
|
|
|
196
260
|
One named flag per opt-in capability (`features.<id>` global,
|
|
@@ -232,7 +296,11 @@ on the 2026-10-11 cache):
|
|
|
232
296
|
routes that pass the role's endpoint filters and the account's
|
|
233
297
|
allowed-providers whitelist (see *Whitelist-aware route selection*) — falling
|
|
234
298
|
back to
|
|
235
|
-
the default `1/price²` blend only when the model has no candidate route.
|
|
299
|
+
the default `1/price²` blend only when the model has no candidate route. The
|
|
300
|
+
**written** pin is the first route in that same order the route probe does not
|
|
301
|
+
reject (spec #90; writes-and-state.md, *The provider-pin auto-configuration*) —
|
|
302
|
+
the probe is a routability gate on the write side, so the priced (score-max)
|
|
303
|
+
route and the written pin can differ. It is
|
|
236
304
|
a **per-model policy, not a scalar** — the sensible pin differs per model, so
|
|
237
305
|
the bundle carries a policy field evaluated against the model's own routes at
|
|
238
306
|
rank time, not a slug baked into the lock file. `bestRoute` (`src/engine.ts`)
|
package/docs/dev/testing.md
CHANGED
|
@@ -34,17 +34,38 @@ passing.
|
|
|
34
34
|
best; cost override (`switched-cost`); `priceSwitchFraction: 0` keeps the
|
|
35
35
|
margin in charge; an inside-margin challenger short of the fraction stays
|
|
36
36
|
kept; the day gate.
|
|
37
|
-
6. **Chain pruning** (`chain-pruning.test.ts
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
the chosen
|
|
37
|
+
6. **Chain pruning + the quality floor** (`chain-pruning.test.ts`,
|
|
38
|
+
`chain-quality-floor.test.ts`) — plugin-written stale keys removed;
|
|
39
|
+
owner-written keys preserved; `writeFallbackChains: false`; a role that left
|
|
40
|
+
the managed set is pruned; the chain prefers candidates at or below the chosen
|
|
41
|
+
price. The floor (spec #95): a cheaper rank-4 candidate that clears the floor
|
|
42
|
+
becomes the first fallback where the pricier rank 2/3 do not, even though the
|
|
43
|
+
probe walk never reaches it; a cheap primary whose cheaper-or-equal candidates
|
|
44
|
+
are all below the floor keeps the next-best-by-value models;
|
|
45
|
+
`fallbackQualityFloor: 0` admits any cheaper-or-equal candidate and `1` only an
|
|
46
|
+
exactly-equal-quality one; an equal-priced sub-floor candidate is in neither
|
|
47
|
+
bucket and is dropped; a key-blocked cheaper candidate never enters the chain;
|
|
48
|
+
the chain stops at `fallbackChainDepth` in the cheaper set's value order; the
|
|
49
|
+
manual-pin path and the auto-pin chain shape are unchanged; an out-of-range
|
|
50
|
+
floor is a settings error that writes nothing; and a `kept-margin` role's chain
|
|
51
|
+
is built from the kept model's `q` and the candidates below it, with the
|
|
52
|
+
selector and the decision reason unaffected.
|
|
41
53
|
7. **Chain suffixes** (`chain-suffix.test.ts`) — chain values carry the role's
|
|
42
54
|
suffix, the key stays bare; an entry without thinking support gets no suffix;
|
|
43
55
|
two roles on one model share a level-free chain.
|
|
44
56
|
8. **Probe gate** (`probe-gate.test.ts`) — 200 → ok; the no-allowed-providers 404
|
|
45
57
|
→ blocked; any other failure → usable; blocked best → next adopted; blocked
|
|
46
|
-
current → replaced;
|
|
47
|
-
|
|
58
|
+
current → replaced; a probe-`blocked` candidate never enters a chain
|
|
59
|
+
(`fallback chains contain only probe-clean candidates`); verdicts cached across
|
|
60
|
+
roles; all-blocked → role untouched with a note. Spec #95 changed what the
|
|
61
|
+
chain's model set is built from: "blocked candidates beyond a kept current do
|
|
62
|
+
not count toward chain depth" was **deleted** (it encoded the removed
|
|
63
|
+
probe-window membership rule), "walk fills a kept role's chain to full depth
|
|
64
|
+
with probe-clean entries" was **renamed** to "a kept role's chain is filled to
|
|
65
|
+
full depth from the candidates below the kept model", and a new "the chain
|
|
66
|
+
reaches a cheaper candidate the walk never probes" pins that an **unprobed**
|
|
67
|
+
cheap candidate is chained while the walk still stops at
|
|
68
|
+
`1 + fallbackChainDepth` clean candidates.
|
|
48
69
|
9. **Keyed catalog** (`keyed-catalog.test.ts`) — key-blocked candidates pruned
|
|
49
70
|
from the walk; only candidates in neither catalog are probed; key-blocked
|
|
50
71
|
never in a chain; no-filter/unavailable degrade to the probe walk.
|
|
@@ -78,7 +99,8 @@ passing.
|
|
|
78
99
|
floor sorts a below-floor route last **without dropping it**; a route with no
|
|
79
100
|
p50 uses the model's blended throughput (for both the score and the floor
|
|
80
101
|
check); equal scores break by `providerSlug` then `id`; `explainModel`'s
|
|
81
|
-
`autoPin` is the score-max route
|
|
102
|
+
`autoPin` is the score-max route — the auto-pin's first **candidate**, which
|
|
103
|
+
the updater probes before writing it (spec #90).
|
|
82
104
|
14. **Thinking price** (`thinking-price.test.ts`) — bare/off unadjusted; a level
|
|
83
105
|
scales only thinking-capable models; the frontier uses the effective price;
|
|
84
106
|
`resolveSettings` validates thinking and rejects the legacy `suffixes` map;
|
|
@@ -137,11 +159,22 @@ passing.
|
|
|
137
159
|
`filters.preferredMinThroughput: 50` (the registry row and the expanded def
|
|
138
160
|
agree), an explicit `filters.preferredMinThroughput` beats the preset, and
|
|
139
161
|
the schema carries the `roles.<role>.filters.preferredMinThroughput` row
|
|
140
|
-
(number, min 0, default 0).
|
|
162
|
+
(number, min 0, default 0). The `tokenProfile` setting (spec #81) defaults to
|
|
163
|
+
`null`, accepts a record with finite `inputTokens`/`outputTokens` ≥ 0, and
|
|
164
|
+
rejects a non-record or a negative/non-finite field; the schema carries the
|
|
165
|
+
`tokenProfile.inputTokens`/`tokenProfile.outputTokens` rows (number, min 0).
|
|
141
166
|
23. **Explorer** (`explorer.test.ts`, `explorer-scopes.test.ts`) — rank deltas;
|
|
142
167
|
contributions sum to `q`; `inverseCardinal` round-trips; Export preserves
|
|
143
168
|
siblings and normalizes nested roles; the scope switcher; the availability
|
|
144
|
-
overlay.
|
|
169
|
+
overlay. The per-task price (spec #81): `rankRows`/`explainModel` fill
|
|
170
|
+
`perTask` from a supplied profile and leave it `null` without one; the
|
|
171
|
+
`POST /api/pool` toggle returns the gated/pruned dataset in `as-written` mode
|
|
172
|
+
and the full one in `raw`; the bootstrap payload carries `poolMode` and
|
|
173
|
+
`tokenProfile`; the explain response carries `leaderFlip`/`thinkingLevels`.
|
|
174
|
+
The auto-pin indicator (spec #90): `explainModel`'s `autoPin` is the route the
|
|
175
|
+
updater's walk pins when every route probes clean, and it deliberately differs
|
|
176
|
+
from the written pin when the walk skips a blocked leader — the panel names
|
|
177
|
+
the *candidate*.
|
|
145
178
|
24. **Project scope** (`project-updater.test.ts`, `project-setup.test.ts`,
|
|
146
179
|
`project-roles-command.test.ts`) — project mode scopes config/state/history/
|
|
147
180
|
lock to `<cwd>/.omp`; the day gate is per scope; the registry write;
|
|
@@ -201,13 +234,30 @@ passing.
|
|
|
201
234
|
and `cut` bumping the four fields, cutting the changelog, committing and
|
|
202
235
|
tagging while refusing a dirty tree, an existing tag, an empty
|
|
203
236
|
`## [Unreleased]` and a HEAD that is not `origin/main`.
|
|
204
|
-
31. **Auto-pin** (`auto-pin.test.ts`) — the selector gains the
|
|
205
|
-
|
|
206
|
-
|
|
237
|
+
31. **Auto-pin** (`auto-pin.test.ts`) — the selector gains the `@<slug>` of the
|
|
238
|
+
first route in the role's `vfm` order that the route probe does not reject
|
|
239
|
+
(spec #90): a `blocked` leader pins the next route and records the skipped
|
|
240
|
+
slug (`routeBlockedSlugs`); `unknown` is usable and stays pinned; every route
|
|
241
|
+
blocked writes a bare selector and sets `pinDropped`; the notify lines carry
|
|
242
|
+
`; route-blocked: …` and `; pin dropped (no routable route)`; an all-blocked
|
|
243
|
+
pool issues at most `ROUTE_PROBE_BUDGET` probes; a verdict cached for one role
|
|
244
|
+
is reused by another; two blocked routes sharing a provider slug are recorded
|
|
245
|
+
once; and the capability off, a manual pin, a locked role, a disabled role and
|
|
246
|
+
a model with no gate-passing route never consult the probe (a throwing stub
|
|
247
|
+
proves it) and leave the config byte-identical. The chain carries every
|
|
248
|
+
gate-passing route of the primary except the pinned **slug** (not the pinned
|
|
249
|
+
position) then of each fallback model, in score order (the `vfm` route
|
|
207
250
|
score desc, spec #71), deduped by
|
|
208
251
|
provider slug; a degraded / over-`maxPriceUsdPerM` / endpoint-filter-failing /
|
|
209
|
-
whitelist-blocked route is excluded; the manual pin wins
|
|
210
|
-
|
|
252
|
+
whitelist-blocked route is excluded; the manual pin wins (its chain is the
|
|
253
|
+
fallback models only, every value carrying the manual slug) and the no-pin
|
|
254
|
+
path is bare (fallback models only); a re-run is idempotent and a pin change
|
|
255
|
+
prunes the old key. Since spec #95 the chain's fallback **models** come from
|
|
256
|
+
the ranked candidate list below the chosen model with the quality floor
|
|
257
|
+
(item 6) rather than the probe window, so the fixtures price each route at its
|
|
258
|
+
model's own price plus its position — routes all priced `1..n` would collapse
|
|
259
|
+
every model to one effective price and the floor would drop the weaker
|
|
260
|
+
fallbacks at equal price.
|
|
211
261
|
32. **Provider display** (`provider-display.test.ts`,
|
|
212
262
|
`provider-display-wiring.test.ts`) — `providerPinOf` reads the single
|
|
213
263
|
`openRouterRouting.only[0]` (else `vercelGatewayRouting`), preferring
|
|
@@ -224,6 +274,35 @@ passing.
|
|
|
224
274
|
from the live model on `turn_start` (the silent-restore case), strips when
|
|
225
275
|
neither source carries a pin, never calls `pi.setModel`, and contains a
|
|
226
276
|
throwing live-model read.
|
|
277
|
+
33. **Usage ledger** (`usage-ledger.test.ts`, `usage-ledger-wiring.test.ts`) —
|
|
278
|
+
`appendUsageRow` writes one row with the expected fields; `readUsageRows`
|
|
279
|
+
degrades to `[]` on a missing/malformed file and skips an unparseable or
|
|
280
|
+
wrongly-shaped line; the cap holds (the newest `USAGE_LEDGER_CAP` rows
|
|
281
|
+
survive a compaction); `roleTokenProfile` is the role's median per-turn
|
|
282
|
+
counts (or `null` with no rows) and `resolveTokenProfile` prefers the
|
|
283
|
+
recorded usage over the setting. The wiring (a fake `pi`/`ctx` with a
|
|
284
|
+
`sessionManager.getBranch()`) records one row per assistant turn on
|
|
285
|
+
`turn_end` and `session_shutdown`, dedupes by `ts|model` so the two hooks
|
|
286
|
+
cannot double-record, writes into `resolveProjectDir() ?? agentDir()`, and
|
|
287
|
+
contains a throwing branch read.
|
|
288
|
+
34. **Explorer price levers** (`explorer-levers.test.ts`) — `leaderFlip` returns
|
|
289
|
+
the min-λ threshold over the strictly cheaper challengers, the current
|
|
290
|
+
`wPrice`, the distance and the challenger; `null` when no cheaper challenger
|
|
291
|
+
exists; the degenerate case (value leader = quality leader) is handled by the
|
|
292
|
+
same min formula, not `0/0`. `thinkingLevelEffect` returns one row per level
|
|
293
|
+
with the engine's own factor and the leader at that level, and `null` for a
|
|
294
|
+
bare role.
|
|
295
|
+
35. **As-written pool** (`explorer-pool.test.ts`) — `asWrittenPool` applies the
|
|
296
|
+
catalog/tier gate (`resolveVariant`) and the allowed-providers prune
|
|
297
|
+
(`providerAllowed`), leaves the routes untouched on a `null` whitelist,
|
|
298
|
+
yields an empty pool on `tier: "none"`, and never mutates its input.
|
|
299
|
+
36. **Route probe** (`route-probe.test.ts`) — `probeRoute` sends the exact
|
|
300
|
+
one-token completion shape (POST, bearer, `max_tokens: 1`, `provider: { only:
|
|
301
|
+
[slug] }`): 200 → `ok`; the no-allowed-providers 404 → `blocked`; a 404 with
|
|
302
|
+
any other body, a 5xx, a 429 and a transport throw → `unknown` (usable). The
|
|
303
|
+
walk that consumes it (skip a `blocked` route, accept `unknown`, drop the pin
|
|
304
|
+
when nothing is clean, the per-role budget, the cross-role verdict cache) is
|
|
305
|
+
covered by the auto-pin cases above.
|
|
227
306
|
|
|
228
307
|
## Live checks
|
|
229
308
|
|
|
@@ -244,6 +323,18 @@ passing.
|
|
|
244
323
|
derived metadata.
|
|
245
324
|
- **Agent removal** — `/remove-agent --name <n>` in a session deletes both
|
|
246
325
|
artifacts and the updater pass removes the config entries.
|
|
326
|
+
- **Explorer pool / per-task / levers** — `/explore-roles` in a session with a
|
|
327
|
+
recorded usage ledger: confirm the header's pool toggle switches the table
|
|
328
|
+
between the raw and as-written pools (the model count and the leader change),
|
|
329
|
+
the `$/task` column renders a value (or `—` with no profile), and the explain
|
|
330
|
+
panel shows the per-task breakdown and the **Price levers** section (the
|
|
331
|
+
leader-flip line and the per-level leader table). Verify in headless Chromium
|
|
332
|
+
(the SPA is not unit-tested).
|
|
333
|
+
- **Usage ledger** — run a real omp turn, then confirm `llm-role-usage.jsonl`
|
|
334
|
+
gained one row per assistant turn (the model, the role, the provider, the
|
|
335
|
+
token counts, the cost) in the plugin's state dir (`<project>/.omp` in project
|
|
336
|
+
mode, `agentDir()` otherwise), and that the explorer's `$/task` column then
|
|
337
|
+
fills for that role. A session with no completed turn writes no rows.
|
|
247
338
|
- **CI/CD** — push a branch and open a PR (or push to `main`) and `gh run watch`
|
|
248
339
|
the `test` job green; push a tag that disagrees with the package version and
|
|
249
340
|
confirm the `release` job fails in the `notes` guard *before* any publish;
|