omp-llm-role 1.1.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +75 -13
- package/docs/dev/architecture.md +35 -11
- package/docs/dev/ci.md +21 -1
- package/docs/dev/current-state.md +99 -27
- package/docs/dev/data-sources.md +33 -4
- package/docs/dev/explorer.md +126 -21
- package/docs/dev/quirks.md +32 -0
- package/docs/dev/releasing.md +63 -32
- package/docs/dev/scoring.md +153 -39
- package/docs/dev/testing.md +112 -24
- package/docs/dev/writes-and-state.md +86 -0
- package/docs/llm-role-rankings.md +126 -126
- package/package.json +92 -22
- package/src/agent-create.ts +4 -2
- package/src/availability.ts +87 -9
- package/src/cli/create-role.ts +1 -1
- package/src/engine.ts +190 -78
- package/src/explorer/boot.ts +43 -1
- package/src/explorer/explain.ts +113 -10
- package/src/explorer/levers.ts +100 -0
- package/src/explorer/pool.ts +46 -0
- package/src/explorer/server.ts +107 -23
- package/src/extension.ts +189 -16
- package/src/features.ts +9 -7
- package/src/provider-display.ts +67 -0
- package/src/settings.ts +39 -2
- package/src/updater.ts +97 -14
- package/src/usage-ledger.ts +127 -0
- package/web/app.js +156 -1
- package/web/style.css +3 -0
package/docs/dev/explorer.md
CHANGED
|
@@ -12,7 +12,9 @@ explorer never reimplements `value = q − λ·$/M`.
|
|
|
12
12
|
| File | Role |
|
|
13
13
|
|---|---|
|
|
14
14
|
| `src/explorer/server.ts` | Zero-dependency `node:http` surface: static SPA + JSON API. Exports `createExplorerServer`, `ExplorerOpts`. |
|
|
15
|
-
| `src/explorer/explain.ts` | Pure explanation layer (no I/O): rank rows with baseline deltas, per-model decomposition, inverse-cardinal targets,
|
|
15
|
+
| `src/explorer/explain.ts` | Pure explanation layer (no I/O): rank rows with baseline deltas, per-model decomposition, inverse-cardinal targets, the keyed-catalog availability overlay, and the display-only per-task price. Exports `rankRows`, `explainModel`, `perTaskPrice`, `inverseCardinal`, `METRIC_META`, `WEIGHTABLE_METRICS`. |
|
|
16
|
+
| `src/explorer/levers.ts` | Pure price-lever readouts (no I/O): the leader-flip threshold and the thinking level's effect on the leader, computed on the pool they are handed. Exports `leaderFlip`, `thinkingLevelEffect`, `LeaderFlip`, `ThinkingLevelRow`. |
|
|
17
|
+
| `src/explorer/pool.ts` | The as-written pool (no I/O): the dataset filtered exactly as the updater filters it before ranking (catalog/tier gate + allowed-providers prune). Exports `asWrittenPool`, `PoolMode`, `PoolInputs`. |
|
|
16
18
|
| `src/explorer/scopes.ts` | Scope resolution (no HTTP): the user-level scope, the registry's projects, and the session's project; the per-scope read; the dataset union. Exports `Scope`, `resolveScopes`, `defaultScopeId`, `resolveScope`, `readScopeRoles`, `unionRoles`. |
|
|
17
19
|
| `src/explorer/boot.ts` | Shared launcher: bind/port fallback, scope resolution, browser open, close. Exports `startExplorer`, `EXPLORER_DEFAULT_PORT`, `ExplorerHandle`, `ExplorerBootOpts`. |
|
|
18
20
|
| `src/project-registry.ts` | The global project registry (`llm-role-projects.json` in the agent dir): read/register/prune, atomic write. Exports `PROJECT_REGISTRY_FILE`, `readProjectRegistry`, `registerProject`. |
|
|
@@ -47,12 +49,29 @@ type ExplorerOpts = {
|
|
|
47
49
|
availability: KeyAvailability; // keyed-catalog overlay (re-derived on refresh)
|
|
48
50
|
};
|
|
49
51
|
refresh(): Promise<void>; // re-runs loadRankData({ refresh: true, roles, extraMetrics }) and swaps rank
|
|
52
|
+
// The as-written pool's inputs: the omp catalog, the key's tier, and the
|
|
53
|
+
// account's allowed-providers whitelist (null = no whitelist / harvest failed,
|
|
54
|
+
// which leaves every route in place). Re-derived per call, so a project's
|
|
55
|
+
// whitelist cannot leak into another scope's view.
|
|
56
|
+
getPoolInputs(): PoolInputs;
|
|
57
|
+
// The recorded usage rows the per-task price resolves its token profile from,
|
|
58
|
+
// for the request's active scope (a project scope reads its own `<root>/.omp`
|
|
59
|
+
// ledger). Optional: a host without a ledger supplies none.
|
|
60
|
+
getUsageRows?(scope: Scope): readonly UsageRow[];
|
|
61
|
+
// The user-set token profile the per-task column falls back to (the
|
|
62
|
+
// `tokenProfile` setting); null (the default) = unset.
|
|
63
|
+
tokenProfile?: TokenProfile | null;
|
|
50
64
|
};
|
|
51
65
|
```
|
|
52
66
|
|
|
53
|
-
The server holds the active scope id
|
|
54
|
-
`rank`/`availability`); `activeScope` resolves
|
|
55
|
-
fallback when the active scope became unavailable.
|
|
67
|
+
The server holds the active scope id **and the pool mode** as mutable state
|
|
68
|
+
(alongside the mutable `rank`/`availability`); `activeScope` resolves the scope
|
|
69
|
+
per request, persisting the fallback when the active scope became unavailable.
|
|
70
|
+
`poolRank` applies the session's pool mode to the dataset before ranking — the
|
|
71
|
+
`as-written` mode runs `asWrittenPool(rank, opts.getPoolInputs())`, the `raw`
|
|
72
|
+
mode (the default, the historical behaviour) passes the fetch through. The pool
|
|
73
|
+
mode is **never** applied to the export path, which validates a pin against the
|
|
74
|
+
raw dataset.
|
|
56
75
|
|
|
57
76
|
Endpoints:
|
|
58
77
|
|
|
@@ -62,8 +81,9 @@ Endpoints:
|
|
|
62
81
|
| `GET /app.js`, `/style.css` | — | static asset (traversal-guarded) |
|
|
63
82
|
| `GET /api/bootstrap` | — | `bootstrapPayload(opts, activeScope)` |
|
|
64
83
|
| `POST /api/scope` | `{scope}` | switch the active scope, then the bootstrap payload (404 unknown id, 409 non-present) |
|
|
65
|
-
| `POST /api/
|
|
66
|
-
| `POST /api/
|
|
84
|
+
| `POST /api/pool` | `{mode}` | switch the ranked pool (`"raw"` \| `"as-written"`), then the bootstrap payload (400 on any other mode) |
|
|
85
|
+
| `POST /api/rank` | `{role, def}` | `{eligible, lambda, derivedLambda, rows, errors}` — each row carries the display-only `perTask` |
|
|
86
|
+
| `POST /api/explain` | `{role, def, modelId}` | `explainModel(...)` spread + `leaderFlip` + `thinkingLevels` + `errors` |
|
|
67
87
|
| `POST /api/export` | `{roles}` | `{ok:true, backupPath, roles, lockPath}` or `{ok:false, errors}` |
|
|
68
88
|
| `POST /api/refresh` | — | `await opts.refresh()` (re-ranks and re-derives the availability overlay) then the bootstrap payload |
|
|
69
89
|
|
|
@@ -75,7 +95,9 @@ Endpoints:
|
|
|
75
95
|
focus metrics over the resolved dataset), `levels` (`Object.keys(SUFFIX_LEVELS)`), `thinkingFactors`
|
|
76
96
|
(per-level billed-blend multiplier from the engine's own `thinkingPriceFactor`,
|
|
77
97
|
so the UI readout cannot drift), `fetchedAt`, `modelCount`, `orMatched`,
|
|
78
|
-
`orPriced`, `
|
|
98
|
+
`orPriced`, `poolMode` (the ranked pool the counts describe — `"raw"` or
|
|
99
|
+
`"as-written"`), `tokenProfile` (the user-set per-task token profile, or
|
|
100
|
+
`null`), `lockPath` (the **active scope's** lock file), `scopes` (id, label,
|
|
79
101
|
kind, present per known scope), `activeScope` (the active scope id),
|
|
80
102
|
the active scope's global capability flags and the feature registry (the
|
|
81
103
|
`FEATURES` rows: id, label, description, recommended bundle, buys), so the SPA
|
|
@@ -99,8 +121,12 @@ Endpoints:
|
|
|
99
121
|
expansion the preview would omit the preset the baseline includes and every
|
|
100
122
|
row's Δ would be a lie. The posted def is the **authored** def (flags + explicit
|
|
101
123
|
keys); the expansion is applied only for the ranking call, never persisted.
|
|
102
|
-
- `handleExport`
|
|
103
|
-
|
|
124
|
+
- `handleExport` validates every dirty role first — `validateRole` plus a
|
|
125
|
+
**provider-pin check**: a `providerPin` that matches no route in the loaded
|
|
126
|
+
dataset is rejected (`role "<name>": provider pin "<slug>" matches no route in
|
|
127
|
+
the loaded dataset`) before any write, so an exported role cannot carry an
|
|
128
|
+
unroutable pin. It then calls `writeRoleSettings(activeScope.lockPath,
|
|
129
|
+
body.roles)` and returns its result verbatim plus the written `lockPath`.
|
|
104
130
|
- Unknown `/api/*` → 404; wrong method → 405; non-API non-GET → 405.
|
|
105
131
|
- `readBody` caps at `MAX_BODY = 1 MiB` (413); `readJson` → 400 on invalid JSON.
|
|
106
132
|
- `serveStatic` resolves against `webDir` and rejects traversal (`full` must be
|
|
@@ -120,33 +146,67 @@ Pure, no I/O; all ranking math is delegated to `src/engine.ts`.
|
|
|
120
146
|
declared?)`** merges the shipped table with a derived entry per external metric.
|
|
121
147
|
- **`weightableMetrics(extra?)`** — the shipped keys plus any external metric in
|
|
122
148
|
`extra`; exactly what the validator accepts.
|
|
123
|
-
- **`rankRows(def, models, baseline, availability?)`** — `rankRole(def, models)` +
|
|
149
|
+
- **`rankRows(def, models, baseline, availability?, profile?)`** — `rankRole(def, models)` +
|
|
124
150
|
`paretoFrontier`, annotating each row with `baselineRank`/`delta`
|
|
125
151
|
(`baselineRank − rank`; positive = moved up) against the effective role's
|
|
126
152
|
ranking. Rows carry `priceEff` (the thinking-adjusted price the ranking
|
|
127
153
|
penalized), `parts`, `missing` (weighted non-price metrics whose raw value is
|
|
128
|
-
null), `frontier`,
|
|
154
|
+
null), `frontier`, `key` — the availability overlay verdict
|
|
129
155
|
(`usable`/`blocked`/`unknown`), orthogonal to `missing` and to the eligibility
|
|
130
|
-
gates
|
|
131
|
-
|
|
156
|
+
gates — and `perTask` (the display-only dollars-per-task, or `null`).
|
|
157
|
+
`availability` and `profile` are optional and pure (no I/O); omitted or
|
|
158
|
+
inactive ⇒ every row reads `unknown` / `perTask: null`. Neither affects the
|
|
159
|
+
ranking order or the deltas.
|
|
132
160
|
- **`inverseCardinal(metric, t)`** — inverse of `cardinalMetric` for the "what
|
|
133
161
|
would it take" targets: `index` → `t*80 − 20`; `throughput` → `10 * 30**t`
|
|
134
162
|
(null outside `[0,1]`, because the forward transform clamps); everything else
|
|
135
163
|
(benchmark, percentile, price) → identity. See *Open defect* below for the
|
|
136
164
|
`gpqa` exception.
|
|
137
|
-
- **`
|
|
165
|
+
- **`perTaskPrice(model, def, profile)`** — the display-only dollars-per-task:
|
|
166
|
+
`p_in` is the role's priced route's input price (cache-adjusted by the role's
|
|
167
|
+
`cacheHitRate`, `routeInputPrice`) and `p_out` that same route's output price —
|
|
168
|
+
the inverse 3:1 blend of it (`routeOutputPrice`), never re-derived here — and
|
|
169
|
+
the output term is scaled by the role's thinking factor
|
|
170
|
+
(`thinkingAdjustedPrice(model, def, 1)`, so the per-task price and the row's
|
|
171
|
+
`priceEff` cannot disagree). When the role prices on the `1/price²` blend
|
|
172
|
+
(`roleRoute` returns `null`) the model's billed blend and the input blend over
|
|
173
|
+
the routes that survive the role's endpoint filters stand in — the same pool the
|
|
174
|
+
ranking prices on, so `p_in` and the billed blend never mix two route pools.
|
|
175
|
+
`null` when there is no usable route price — **absent, never a silent
|
|
176
|
+
zero**. The SPA renders `—` in both cases: no token profile resolves (no
|
|
177
|
+
recorded usage and no `tokenProfile` set), **or** the model has no usable
|
|
178
|
+
route price (e.g. every route pruned by the as-written whitelist — the model
|
|
179
|
+
still ranks on its blend price, but the per-task column has no route to
|
|
180
|
+
price). It never feeds the ranking's sort key, `value`, or the explanation's
|
|
181
|
+
`cost` block.
|
|
182
|
+
- **`explainModel(def, models, modelId, roleName, availability?, profile?)`** — the full
|
|
138
183
|
decomposition, or the eligibility gates it failed (same gates `rankRole`
|
|
139
184
|
applies, same order: missing required, image filter, no billed price). On
|
|
140
185
|
success: `contributions`
|
|
141
186
|
(raw → cardinal `t` → renormalized weight → contribution → share of `q`, with
|
|
142
187
|
a `fillNote` when the value is a capability fill), `cost` (`priceEff`, the
|
|
143
188
|
cache-adjusted billed price, λ, penalty, `q`, `value`), `role.cacheHitRate`,
|
|
144
|
-
`
|
|
189
|
+
`perTask` (the `PerTaskPrice` breakdown, or `null` when no profile was
|
|
190
|
+
supplied or the model has no usable route price), `gapAbove`/`gapToTop`/`above`, `closing` (the
|
|
145
191
|
per-metric raw target that would close the gap, via `inverseCardinal`, with
|
|
146
192
|
unreachable/extrapolated notes), `dominators` (models both cheaper and at
|
|
147
|
-
least as good on `q`, top 3),
|
|
193
|
+
least as good on `q`, top 3), `ownProvider` (true when the model was priced on
|
|
194
|
+
the soft best route rather than the blend), `autoPin` (the provider slug the
|
|
195
|
+
updater would write — `bestRoute(model, def)?.providerSlug`, the max-`vfm`
|
|
196
|
+
route (scoring.md, *Route ordering*), when the role sets
|
|
197
|
+
`preferOwnProvider` and carries no manual `providerPin`, else `null`), and the
|
|
198
|
+
availability overlay `key`/`keyReason`
|
|
148
199
|
(the same verdict as `rankRows`, plus the reason it reads that way). It mirrors
|
|
149
200
|
`rankRole`'s capability fill so the contributions sum exactly to `q`.
|
|
201
|
+
- **`leaderFlip(def, models)`** / **`thinkingLevelEffect(def, models)`**
|
|
202
|
+
(`src/explorer/levers.ts`) — the two price-lever readouts, computed on the pool
|
|
203
|
+
they are handed (so the readout and the ranking above it agree). `leaderFlip`
|
|
204
|
+
returns the price weight at which a cheaper challenger overtakes the value
|
|
205
|
+
leader (`{ wPrice, flipW, distance, challenger }`), or `null` when no cheaper
|
|
206
|
+
challenger exists. `thinkingLevelEffect` returns one `{ level, factor, leader }`
|
|
207
|
+
row per thinking level, or `null` for a bare role. Both reuse the engine's own
|
|
208
|
+
`roleLambda`/`rankRole`/`SUFFIX_LEVELS`/`thinkingPriceFactor`; the formulas are
|
|
209
|
+
in [`scoring.md`](scoring.md).
|
|
150
210
|
|
|
151
211
|
## Shared boot path (`src/explorer/boot.ts`)
|
|
152
212
|
|
|
@@ -193,6 +253,22 @@ launcher `/explore-roles` uses.
|
|
|
193
253
|
factor on the omp catalog; `refresh()` re-runs `opts.reload(true)`,
|
|
194
254
|
re-enriches, and — when `opts.reloadAvailability` is supplied — re-derives the
|
|
195
255
|
availability overlay.
|
|
256
|
+
- **As-written pool inputs.** `ExplorerBootOpts` carries `tier` (default
|
|
257
|
+
`"billed"`), `allowedProviders` (`null` = no pruning), and
|
|
258
|
+
`reloadAllowedProviders` (re-derived on `refresh()`). `getPoolInputs()` returns
|
|
259
|
+
`{ catalog: opts.catalog, tier, allowed }` per call, so a project's whitelist
|
|
260
|
+
cannot leak into another scope's view. Both are **best-effort**: an absent tier
|
|
261
|
+
reads as `billed`, and a failed re-harvest (`reloadAllowedProviders` throwing
|
|
262
|
+
or returning `null`) degrades the as-written pool to no route pruning rather
|
|
263
|
+
than failing the refresh. `/explore-roles` builds them in-process from the
|
|
264
|
+
session's OpenRouter key (the canary must be a **catalog** slug, not a ranking
|
|
265
|
+
id).
|
|
266
|
+
- **Usage ledger.** `getUsageRows(scope)` reads the active scope's ledger: a
|
|
267
|
+
project scope reads its own `<root>/.omp` ledger, the user scope reads
|
|
268
|
+
`usageDir` (default `agentDir()`), so a project's usage never mixes with the
|
|
269
|
+
user-level profile. Re-read per call, so a turn recorded while the explorer is
|
|
270
|
+
open shows up without a restart. `tokenProfile` is the user-set fallback the
|
|
271
|
+
per-task column uses when a role has no recorded usage.
|
|
196
272
|
- `opts.availability` is the boot-time keyed-catalog overlay; `getState(scope)`
|
|
197
273
|
returns it on every request, so `/api/rank` and `/api/explain` annotate rows
|
|
198
274
|
without a second fetch. It is scope-independent and carried across switches
|
|
@@ -237,12 +313,23 @@ a scraped third-party site, so the DOM is built with
|
|
|
237
313
|
`textContent`/`createElement` only (the `el()` helper).
|
|
238
314
|
|
|
239
315
|
- **Boot**: `boot()` → `GET /api/bootstrap` → `applyBootstrap(data)`, which fills
|
|
240
|
-
`state` (`scopes`, `activeScope`, `
|
|
316
|
+
`state` (`scopes`, `activeScope`, `poolMode`, `tokenProfile`, `universe`,
|
|
317
|
+
`effective` = each entry's `def`,
|
|
241
318
|
`defs` = a structuredClone of it, `defaults`, `metrics`, `metricMeta`,
|
|
242
319
|
`focusAssessments`, `levels`, `thinkingFactors`, `lockPath`, `availability`),
|
|
243
320
|
renders the header/roles/table, and selects the previously selected role when
|
|
244
321
|
the new scope has it, else the first role. `boot()` then wires the
|
|
245
322
|
filter/topn/hide-key-blocked/export/copy/download controls.
|
|
323
|
+
- **Pool toggle**: the header renders a `#pool` `<select>` (`raw` / `as-written`)
|
|
324
|
+
from `state.poolMode`. Changing it confirms first when there are unsaved edits
|
|
325
|
+
(mirroring the scope switcher), then `POST /api/pool {mode}` and
|
|
326
|
+
`applyBootstrap` on the returned payload — so the header's model count, the
|
|
327
|
+
table, the focus assessments and the two readouts all describe the pool the
|
|
328
|
+
user is now looking at. The toggle is **session-level** state — the server
|
|
329
|
+
holds one pool mode, kept across scope switches (not persisted) — while the
|
|
330
|
+
pool *inputs* are re-derived per call, so a project's whitelist never leaks
|
|
331
|
+
into another scope's view. Export is unaffected: it always
|
|
332
|
+
writes the authored def, never the pool.
|
|
246
333
|
- **Scope switcher**: the header renders a `#scope` `<select>` from
|
|
247
334
|
`state.scopes` (the user-level scope plus every known project); a scope whose
|
|
248
335
|
lock file is gone is `disabled` with a reason `title`. Changing the selection
|
|
@@ -273,11 +360,29 @@ a scraped third-party site, so the DOM is built with
|
|
|
273
360
|
- **Recompute**: `selectRole` → `renderRoles`/`renderEditor`/`renderExplain` +
|
|
274
361
|
`recompute()`. `recompute()` → `POST /api/rank {role, def}` → `state.rows`,
|
|
275
362
|
`lambda`, `derivedLambda`, `errors`; re-renders the table and readouts, and
|
|
276
|
-
reloads the explain panel if the selected model is still present.
|
|
277
|
-
|
|
363
|
+
reloads the explain panel if the selected model is still present. The table
|
|
364
|
+
gains a **`$/task`** column (`r.perTask`, rendered `—` when `null`), beside the
|
|
365
|
+
`$/M` column. Editor changes call `scheduleRecompute()` (120 ms debounce).
|
|
278
366
|
- **Explain**: `selectModel(id)` → `POST /api/explain {role, def, modelId}` →
|
|
279
|
-
`renderExplain()` (composition table, cost line,
|
|
280
|
-
dominators, and the availability
|
|
367
|
+
`renderExplain()` (composition table, cost line, **per-task price** line, "why
|
|
368
|
+
not higher", dominators, the **Price levers** section, and the availability
|
|
369
|
+
line). The cost line appends `· best route`
|
|
370
|
+
when the model was priced on the soft best route (`ownProvider`), and a
|
|
371
|
+
**pin line** shows the pin in force: `pin: <slug> (manual)` when the edited def
|
|
372
|
+
carries a `providerPin`, else `auto pin: <slug>` from the explanation's
|
|
373
|
+
`autoPin` (read-only — the manual field is the only way to set one). The
|
|
374
|
+
per-task line shows the breakdown (`<in> in × $<p_in>/M + <out> out ×
|
|
375
|
+
$<p_out>/M × <factor> = $<total>`), or `— (no token profile — record usage or
|
|
376
|
+
set tokenProfile)` when `perTask` is `null`. The **Price levers** section
|
|
377
|
+
renders the leader-flip line (`w* = <flipW> · now <wPrice> · distance
|
|
378
|
+
<±distance> (<ratio>)`) and a per-level table of the thinking level's effect on
|
|
379
|
+
the leader; each is omitted when the server sends `null` (no cheaper
|
|
380
|
+
challenger, or a bare role).
|
|
381
|
+
- **Provider pin field**: the editor renders a text input for
|
|
382
|
+
`roles.<role>.providerPin` (a tiered slug like `deepinfra/fp8` is kept
|
|
383
|
+
verbatim; empty clears the key, resuming the automatic choice). It is the
|
|
384
|
+
manual override of the auto-pin; Export rejects a slug that matches no route in
|
|
385
|
+
the loaded dataset (see *Export path*).
|
|
281
386
|
- **Availability overlay**: each row's `key` renders as a three-state badge in
|
|
282
387
|
the `key` column; the **hide key-blocked** checkbox beside the filter/top-n
|
|
283
388
|
controls drops `blocked` rows from the table (a client-side filter — the
|
package/docs/dev/quirks.md
CHANGED
|
@@ -146,6 +146,22 @@ things that only bite someone changing or debugging the code.
|
|
|
146
146
|
update (or a `link` to a changed tree), a session that was already running
|
|
147
147
|
still executes the module it loaded at start; a write from that stale code can
|
|
148
148
|
revert a newer write. Restart the session after updating the plugin.
|
|
149
|
+
- **omp's restore-to-primary is silent to extensions.** After a fallback
|
|
150
|
+
cooldown, omp restores the primary model at prompt admission
|
|
151
|
+
(`#maybeRestoreRetryFallbackPrimary`) with **no** session event, and
|
|
152
|
+
`model_changed` is not delivered to extensions — so a display derived from the
|
|
153
|
+
fallback events alone would keep the fallback's provider pin after a restore.
|
|
154
|
+
The provider display (`src/provider-display.ts`) therefore re-derives the pin
|
|
155
|
+
from the live model on `turn_start`, which *is* extension-delivered and fires
|
|
156
|
+
after the restore. The same re-derive covers a manual `/model` change (with a
|
|
157
|
+
one-turn lag). The primary and a fallback may be the same registry object
|
|
158
|
+
(same model id, different route), so the strip-then-append is idempotent and
|
|
159
|
+
reads the pin from the live model's routing block, never a cached value.
|
|
160
|
+
`retry_fallback_applied`/`retry_fallback_succeeded` are also emitted for
|
|
161
|
+
**advisor** fallbacks, where only the advisor's model changes and the session
|
|
162
|
+
model is untouched, so the event's selector is trusted only when it names the
|
|
163
|
+
live session model (`liveModelIsSelector`); otherwise the pin is re-derived
|
|
164
|
+
from the live model's routing block.
|
|
149
165
|
- **Testing a registered slash command needs `--mode rpc` or a TUI** — `omp -p`
|
|
150
166
|
(print mode) sends a leading `/name` as a *prompt* to the model, so the handler
|
|
151
167
|
never runs and `ctx.ui.notify` is a no-op anyway. RPC mode does dispatch them:
|
|
@@ -154,6 +170,22 @@ things that only bite someone changing or debugging the code.
|
|
|
154
170
|
stderr; `get_available_commands` confirms registration. Verified this way for
|
|
155
171
|
`/create-agent` and `/explore-roles` (omp 18.4.8).
|
|
156
172
|
|
|
173
|
+
## Repository plumbing
|
|
174
|
+
|
|
175
|
+
- **The https credential cannot *introduce* a workflow change.** `origin` is
|
|
176
|
+
https and its credential is a `gh` OAuth token with scopes
|
|
177
|
+
`admin:public_key, gist, read:org, repo` — no `workflow` — so a push that adds
|
|
178
|
+
or modifies `.github/workflows/*` is rejected ("refusing to allow an OAuth App
|
|
179
|
+
to create or update workflow … without `workflow` scope"), and the REST
|
|
180
|
+
contents API is rejected the same way. The check is a diff against the ref's
|
|
181
|
+
base, not a property of the ref: a new branch or tag whose commit already
|
|
182
|
+
contains the workflow, with `main` also containing it, pushes fine over https
|
|
183
|
+
(verified 2026-10-08). So only the commit that lands a workflow edit needs the
|
|
184
|
+
SSH route — `git push git@github.com:Gott50/omp-llm-role.git main` — and
|
|
185
|
+
because that bypasses the `origin` remote it leaves `refs/remotes/origin/main`
|
|
186
|
+
stale, which makes `scripts/release.ts cut` refuse ("HEAD is not origin/main")
|
|
187
|
+
until `git fetch origin` runs.
|
|
188
|
+
|
|
157
189
|
## Historical
|
|
158
190
|
|
|
159
191
|
- **Bare roles were priced as if thinking were free** (fixed 2026-09-30).
|
package/docs/dev/releasing.md
CHANGED
|
@@ -116,14 +116,64 @@ path, plus `marketplace update` + `upgrade` to a bumped catalog version), and
|
|
|
116
116
|
|
|
117
117
|
## Status
|
|
118
118
|
|
|
119
|
-
- **CI/CD landed 2026-10-08** (`fa7e546
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
against version `1.1.0` failed
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
119
|
+
- **CI/CD landed 2026-10-08** (`fa7e546`, docs follow-ups `4f78ce2`, `1d20ef9`,
|
|
120
|
+
`9244fdc`, `3c0125a`): every pull request and every `main` push runs the suite
|
|
121
|
+
on GitHub (verified green on PR #57 and on the `main` pushes), and a `v*` tag
|
|
122
|
+
push publishes to npm over OIDC and creates the GitHub Release. The guard was
|
|
123
|
+
verified with a negative control — tag `v0.0.1` against version `1.1.0` failed
|
|
124
|
+
in `notes` before any publish.
|
|
125
|
+
- **1.2.0 published by CD 2026-10-12** (run 38299141569, tag push): the guard
|
|
126
|
+
and the npm floor passed, `npm publish` completed over OIDC trusted publishing
|
|
127
|
+
with a provenance attestation (`npm audit signatures`: 1 package has a
|
|
128
|
+
verified attestation), and the GitHub Release was created with the 1.2.0
|
|
129
|
+
CHANGELOG section as its body
|
|
130
|
+
(`https://github.com/Gott50/omp-llm-role/releases/tag/v1.2.0`). Verified:
|
|
131
|
+
`dist-tags.latest` = `1.2.0`, npm's `gitHead` = `50fce66` (the tag's commit),
|
|
132
|
+
and the publish step's log carries the direct-publish signature (`Publishing
|
|
133
|
+
to https://registry.npmjs.org/ with tag latest`, `Signed provenance
|
|
134
|
+
statement`, `+ omp-llm-role@1.2.0`). The cut (`50fce66`, tagged `v1.2.0`)
|
|
135
|
+
bumped the four lockstep fields, renamed `## [Unreleased]` to
|
|
136
|
+
`## [1.2.0] - 2026-10-12`, and regenerated the report (408 models, matched
|
|
137
|
+
150/408, priced 148; `docs/dev/current-state.md` refreshed to the same
|
|
138
|
+
measurement).
|
|
139
|
+
- **1.2.0 install routes smoke-tested** (2026-10-12, omp 18.10.3, throwaway
|
|
140
|
+
`HOME`s, cwd outside the real home tree): npm
|
|
141
|
+
(`omp plugin install omp-llm-role`), git
|
|
142
|
+
(`omp plugin install github:Gott50/omp-llm-role`) and marketplace
|
|
143
|
+
(`omp plugin marketplace add Gott50/omp-llm-role` +
|
|
144
|
+
`omp plugin install omp-llm-role@gott50-plugins`) each installed 1.2.0
|
|
145
|
+
(`omp plugin doctor`: 4 ok / 0 warnings / 0 errors for npm and git; the
|
|
146
|
+
marketplace install's one warning is the expected
|
|
147
|
+
`package_manifest: Not created yet`). Each post-install session run
|
|
148
|
+
(`omp -p "say ok"` with `OPENROUTER_API_KEY` from `omp token openrouter`)
|
|
149
|
+
wrote `modelRoles` (9 roles), `retry.fallbackChains` and
|
|
150
|
+
`task.disabledAgents: [designer]` into that HOME's `config.yml`, landed
|
|
151
|
+
`llm-role-state.json` + `llm-role-history.jsonl`, and answered. The npm-route
|
|
152
|
+
run also exercised the spec #65 whitelist harvest live (`OpenRouter
|
|
153
|
+
allowed-providers whitelist active — 11 providers; 542 routes pruned`).
|
|
154
|
+
- **1.1.0 published by CD 2026-10-09** (run 37897647817,
|
|
155
|
+
`gh workflow run ci.yml --ref v1.1.0`): the guard and the npm floor passed,
|
|
156
|
+
`npm publish` completed over OIDC trusted publishing with a provenance
|
|
157
|
+
attestation (`npm audit signatures`: 0 invalid, 0 missing), and the GitHub
|
|
158
|
+
Release was created with the 1.1.0 CHANGELOG section as its body
|
|
159
|
+
(`https://github.com/Gott50/omp-llm-role/releases/tag/v1.1.0`). Verified:
|
|
160
|
+
`dist-tags.latest` = `1.1.0`, npm's `gitHead` = `4f78ce2` (the tag's commit),
|
|
161
|
+
`_npmUser` = `GitHub Actions` carrying the trusted-publisher OIDC config, the
|
|
162
|
+
provenance statement in the transparency log at
|
|
163
|
+
`https://search.sigstore.dev/?logIndex=3159690884`, and the tarball is 56
|
|
164
|
+
files / 308.4 kB. The `v1.1.0` tag was repointed to
|
|
165
|
+
`4f78ce2` first — its original commit predates the workflow file, so the
|
|
166
|
+
dispatch was rejected (see [`ci.md`](ci.md)).
|
|
167
|
+
- **npm route smoke-tested for 1.1.0** (2026-10-09, omp 18.8.6, throwaway
|
|
168
|
+
`HOME`, cwd outside the real home tree): `omp plugin install omp-llm-role`
|
|
169
|
+
installed 1.1.0 from the registry (`omp plugin doctor`: 4 ok, 0 warnings,
|
|
170
|
+
0 errors) and the post-install session run (`omp -p "say ok"` with
|
|
171
|
+
`OPENROUTER_API_KEY` from `omp token openrouter`) wrote `modelRoles`
|
|
172
|
+
(9 roles), `retry.fallbackChains` and `task.disabledAgents: [designer]` into
|
|
173
|
+
that HOME's `config.yml`, landed `llm-role-state.json` +
|
|
174
|
+
`llm-role-history.jsonl`, and answered.
|
|
175
|
+
- **1.1.0 cut 2026-10-08** (`c013918`, tagged `v1.1.0` — repointed to `4f78ce2`,
|
|
176
|
+
see the publish bullet). `package.json` and
|
|
127
177
|
`.omp-plugin/marketplace.json` bumped in lockstep (plus `package-lock.json`
|
|
128
178
|
via `npm install --package-lock-only` — the repo keeps it in lockstep, see
|
|
129
179
|
`8811309`), `## [Unreleased]` renamed to `## [1.1.0] - 2026-10-08`, and the
|
|
@@ -141,26 +191,6 @@ path, plus `marketplace update` + `upgrade` to a bumped catalog version), and
|
|
|
141
191
|
`modelRoles` (9 roles), `retry.fallbackChains` and
|
|
142
192
|
`task.disabledAgents: [designer]` into that HOME's `config.yml`, landed
|
|
143
193
|
`llm-role-state.json` + `llm-role-history.jsonl`, and the session answered.
|
|
144
|
-
The npm route has **not** been smoke-tested for 1.1.0 — it is still serving
|
|
145
|
-
1.0.0.
|
|
146
|
-
- **npm publish pending a valid token.** The `~/.npmrc` token is rejected
|
|
147
|
-
(`npm whoami` → 401 "your authentication token seems to be invalid",
|
|
148
|
-
`npm publish` → 404 on `PUT /omp-llm-role`), so `omp-llm-role@1.1.0` is not
|
|
149
|
-
on the registry yet, `dist-tags.latest` still points at `1.0.0`, and **npm's
|
|
150
|
-
`gitHead` for 1.1.0 does not exist yet** — the tag currently stands on its
|
|
151
|
-
own. The token is the only credential source (no repo `.npmrc`, no
|
|
152
|
-
`/opt/homebrew/etc/npmrc`, no `NPM_TOKEN` in the environment or the shell
|
|
153
|
-
profiles, no keychain entry) and it is a literal value, not a `${NPM_TOKEN}`
|
|
154
|
-
indirection, so the cause is a rejected/revoked/expired token — **not** the
|
|
155
|
-
2026-07-31 GAT restriction, which covers account/org/package-management
|
|
156
|
-
actions and not `whoami`, and not direct publish either (that loss is
|
|
157
|
-
targeted January 2027). `npm ping` and `npm owner ls` still succeed because
|
|
158
|
-
they are unauthenticated reads. Fix: the release workflow — OIDC trusted
|
|
159
|
-
publishing, no token — once the trusted publisher is registered (see
|
|
160
|
-
[`ci.md`](ci.md)). The tag was cut before the publish (the documented order is
|
|
161
|
-
publish-then-tag) precisely so the pinned git ref and the changelog link
|
|
162
|
-
resolve meanwhile; that also means the tag's commit predates the workflow
|
|
163
|
-
file, so the workflow cannot be dispatched against it.
|
|
164
194
|
- **1.0.0 on all three channels** (2026-10-03): `omp-llm-role@1.0.0` on npm
|
|
165
195
|
(`dist-tags.latest` = `1.0.0`), tagged `v1.0.0` at `fe98b76` (npm's
|
|
166
196
|
`gitHead`); all three routes verified end-to-end against the real URLs
|
|
@@ -169,7 +199,8 @@ path, plus `marketplace update` + `upgrade` to a bumped catalog version), and
|
|
|
169
199
|
landed the three daily caches in the installed copy.
|
|
170
200
|
- **Stray registry placeholder.** The registry also carries a `0.0.0-stage`
|
|
171
201
|
version ("Temporary package placeholder for staged publishing", 334 B,
|
|
172
|
-
published 2026-10-03). It is not from this tree
|
|
173
|
-
`1.
|
|
174
|
-
- **Next release:
|
|
175
|
-
|
|
202
|
+
published 2026-10-03). It is not from this tree and does not affect installs
|
|
203
|
+
(`latest` is `1.2.0`).
|
|
204
|
+
- **Next release: none owed.** `## [Unreleased]` is empty — the 1.2.0 cut moved
|
|
205
|
+
its entries under `## [1.2.0] - 2026-10-12`; the number comes from the
|
|
206
|
+
Unreleased scope, not from the open issues.
|