omp-llm-role 1.1.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +75 -13
- package/docs/dev/architecture.md +35 -11
- package/docs/dev/ci.md +21 -1
- package/docs/dev/current-state.md +99 -27
- package/docs/dev/data-sources.md +33 -4
- package/docs/dev/explorer.md +126 -21
- package/docs/dev/quirks.md +32 -0
- package/docs/dev/releasing.md +63 -32
- package/docs/dev/scoring.md +153 -39
- package/docs/dev/testing.md +112 -24
- package/docs/dev/writes-and-state.md +86 -0
- package/docs/llm-role-rankings.md +126 -126
- package/package.json +92 -22
- package/src/agent-create.ts +4 -2
- package/src/availability.ts +87 -9
- package/src/cli/create-role.ts +1 -1
- package/src/engine.ts +190 -78
- package/src/explorer/boot.ts +43 -1
- package/src/explorer/explain.ts +113 -10
- package/src/explorer/levers.ts +100 -0
- package/src/explorer/pool.ts +46 -0
- package/src/explorer/server.ts +107 -23
- package/src/extension.ts +189 -16
- package/src/features.ts +9 -7
- package/src/provider-display.ts +67 -0
- package/src/settings.ts +39 -2
- package/src/updater.ts +97 -14
- package/src/usage-ledger.ts +127 -0
- package/web/app.js +156 -1
- package/web/style.css +3 -0
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Price-lever readouts for the ranking explorer: the leader-flip threshold (the
|
|
3
|
+
* price weight at which a cheaper challenger overtakes the value leader) and the
|
|
4
|
+
* thinking level's effect on the leader.
|
|
5
|
+
*
|
|
6
|
+
* Both are pure and computed on the pool they are handed, so the caller decides
|
|
7
|
+
* raw vs as-written. The math is the engine's own — `roleLambda`, `rankRole`,
|
|
8
|
+
* `SUFFIX_LEVELS` and `thinkingPriceFactor` are reused, never re-derived.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import {
|
|
12
|
+
rankRole,
|
|
13
|
+
SUFFIX_LEVELS,
|
|
14
|
+
thinkingPriceFactor,
|
|
15
|
+
type Model,
|
|
16
|
+
type Ranked,
|
|
17
|
+
type RoleDef,
|
|
18
|
+
type SuffixLevel,
|
|
19
|
+
} from "../engine.ts";
|
|
20
|
+
|
|
21
|
+
/** The price weight at which a cheaper challenger overtakes the value leader. */
|
|
22
|
+
export type LeaderFlip = {
|
|
23
|
+
/** the role's current price weight */
|
|
24
|
+
wPrice: number;
|
|
25
|
+
/** the price weight at which a cheaper challenger overtakes */
|
|
26
|
+
flipW: number;
|
|
27
|
+
/** `flipW − wPrice` */
|
|
28
|
+
distance: number;
|
|
29
|
+
challenger: { id: string; name: string; value: number; priceEff: number };
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* The price weight at which a cheaper challenger overtakes the value leader, or
|
|
34
|
+
* `null` when no cheaper challenger exists (the readout is absent, never
|
|
35
|
+
* invented).
|
|
36
|
+
*
|
|
37
|
+
* `λ* = min over cheaper challengers X of (q_L − q_X)/(pEff_L − pEff_X)`, then
|
|
38
|
+
* `w* = 20·λ* ÷ (1 + 20·λ*)` — the inverse of `roleLambda`'s
|
|
39
|
+
* `λ = w/(1−w)/20`. The degenerate case — the value leader is also the quality
|
|
40
|
+
* leader, where the naive `(q_QL − q_VL)/(p_QL − p_VL)` is 0/0 — is handled by
|
|
41
|
+
* this same min formula; it is *undefined*, not zero.
|
|
42
|
+
*
|
|
43
|
+
* The threshold is exact when the role's price basis is weight-independent (the
|
|
44
|
+
* 1/price² blend, or a hard `providerPin`). With the soft `preferOwnProvider`
|
|
45
|
+
* basis the route choice itself moves with the price weight, so the threshold is
|
|
46
|
+
* a first-order estimate.
|
|
47
|
+
*/
|
|
48
|
+
export function leaderFlip(def: RoleDef, models: Model[]): LeaderFlip | null {
|
|
49
|
+
const ranked = rankRole(def, models);
|
|
50
|
+
const leader = ranked[0];
|
|
51
|
+
if (leader === undefined) return null;
|
|
52
|
+
let best: { lambda: number; challenger: Ranked } | null = null;
|
|
53
|
+
for (const x of ranked) {
|
|
54
|
+
if (!(x.priceEff < leader.priceEff)) continue; // cheaper challengers only
|
|
55
|
+
const lambda = (leader.q - x.q) / (leader.priceEff - x.priceEff);
|
|
56
|
+
if (best === null || lambda < best.lambda) best = { lambda, challenger: x };
|
|
57
|
+
}
|
|
58
|
+
if (best === null) return null;
|
|
59
|
+
const wPrice = def.weights.price ?? 0;
|
|
60
|
+
const flipW = (20 * best.lambda) / (1 + 20 * best.lambda);
|
|
61
|
+
return {
|
|
62
|
+
wPrice,
|
|
63
|
+
flipW,
|
|
64
|
+
distance: flipW - wPrice,
|
|
65
|
+
challenger: {
|
|
66
|
+
id: best.challenger.model.id,
|
|
67
|
+
name: best.challenger.model.name,
|
|
68
|
+
value: best.challenger.value,
|
|
69
|
+
priceEff: best.challenger.priceEff,
|
|
70
|
+
},
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** One thinking level's effect on the ranking: the level, its price factor, and
|
|
75
|
+
* the leader at that level (`null` when the level ranks nothing). */
|
|
76
|
+
export type ThinkingLevelRow = {
|
|
77
|
+
level: SuffixLevel;
|
|
78
|
+
factor: number;
|
|
79
|
+
leader: { id: string; name: string; value: number } | null;
|
|
80
|
+
};
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* The leader at each thinking level, with the level's price factor, or `null`
|
|
84
|
+
* for a bare role (`def.thinking === undefined` — a role with no thinking level
|
|
85
|
+
* is unaffected).
|
|
86
|
+
*
|
|
87
|
+
* The factor is a price-sensitivity amplifier, not a hidden cost: it multiplies
|
|
88
|
+
* `priceEff`, so it penalizes expensive models more in absolute λ-units.
|
|
89
|
+
*/
|
|
90
|
+
export function thinkingLevelEffect(def: RoleDef, models: Model[]): ThinkingLevelRow[] | null {
|
|
91
|
+
if (def.thinking === undefined) return null;
|
|
92
|
+
return (Object.keys(SUFFIX_LEVELS) as SuffixLevel[]).map((level) => {
|
|
93
|
+
const leader = rankRole({ ...def, thinking: level }, models)[0];
|
|
94
|
+
return {
|
|
95
|
+
level,
|
|
96
|
+
factor: thinkingPriceFactor(level),
|
|
97
|
+
leader: leader === undefined ? null : { id: leader.model.id, name: leader.model.name, value: leader.value },
|
|
98
|
+
};
|
|
99
|
+
});
|
|
100
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The as-written pool: the explorer's dataset filtered exactly as the updater
|
|
3
|
+
* filters it before ranking (spec #81). Two filters, both sourced from the
|
|
4
|
+
* availability primitives so the two pools cannot drift:
|
|
5
|
+
*
|
|
6
|
+
* 1. the catalog/tier gate — `filterCatalog(catalog, tier)`, then keep only
|
|
7
|
+
* models whose ranking id resolves to a tier-eligible entry
|
|
8
|
+
* (`resolveVariant`), mirroring `src/updater.ts:300` + `:376`;
|
|
9
|
+
* 2. the allowed-providers whitelist prune — each kept model's routes are
|
|
10
|
+
* filtered to allowed providers (`providerAllowed`), mirroring
|
|
11
|
+
* `src/updater.ts:349-357`. A null whitelist (no whitelist set / harvest
|
|
12
|
+
* failed) leaves the routes untouched.
|
|
13
|
+
*
|
|
14
|
+
* Pure: the input `RankData` and its models are never mutated. The fetch
|
|
15
|
+
* metadata (`fetchedAt`, `source`, `orMatched`, `orPriced`) describes the fetch,
|
|
16
|
+
* not the pool, and is carried through unchanged.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import { filterCatalog, providerAllowed, resolveVariant, type CatalogEntry, type Tier } from "../availability.ts";
|
|
20
|
+
import type { RankData } from "../engine.ts";
|
|
21
|
+
|
|
22
|
+
/** Which dataset the explorer ranks: the raw fetch, or the pool the updater writes from. */
|
|
23
|
+
export type PoolMode = "raw" | "as-written";
|
|
24
|
+
|
|
25
|
+
/** The inputs the as-written pool needs beyond the ranking data: the omp catalog,
|
|
26
|
+
* the key's tier, and the account's allowed-providers whitelist (`null` = no
|
|
27
|
+
* whitelist / harvest failed). */
|
|
28
|
+
export type PoolInputs = { catalog: CatalogEntry[]; tier: Tier; allowed: ReadonlySet<string> | null };
|
|
29
|
+
|
|
30
|
+
/** The as-written pool: `rank` with the catalog/tier gate and the
|
|
31
|
+
* allowed-providers prune applied. `tier: "none"` yields an empty pool. */
|
|
32
|
+
export function asWrittenPool(rank: RankData, inputs: PoolInputs): RankData {
|
|
33
|
+
const tier = inputs.tier;
|
|
34
|
+
const eligible = filterCatalog(inputs.catalog, tier);
|
|
35
|
+
const gated = tier === "none" ? [] : rank.models.filter((m) => resolveVariant(m.id, eligible, tier) !== null);
|
|
36
|
+
const allowed = inputs.allowed;
|
|
37
|
+
const models =
|
|
38
|
+
allowed === null
|
|
39
|
+
? gated
|
|
40
|
+
: gated.map((m) => {
|
|
41
|
+
const routes = m.routes;
|
|
42
|
+
if (routes === undefined) return m;
|
|
43
|
+
return { ...m, routes: routes.filter((r) => providerAllowed(allowed, r.providerSlug)) };
|
|
44
|
+
});
|
|
45
|
+
return { ...rank, models };
|
|
46
|
+
}
|
package/src/explorer/server.ts
CHANGED
|
@@ -10,9 +10,13 @@
|
|
|
10
10
|
* capability flags, and the feature registry
|
|
11
11
|
* POST /api/scope -> { scope } -> switch the active scope, return the
|
|
12
12
|
* new bootstrap payload
|
|
13
|
-
* POST /api/
|
|
14
|
-
*
|
|
15
|
-
* POST /api/
|
|
13
|
+
* POST /api/pool -> { mode } -> switch the ranked pool (raw vs
|
|
14
|
+
* as-written), return the new bootstrap payload
|
|
15
|
+
* POST /api/rank -> { role, def } -> rows with baseline deltas, a
|
|
16
|
+
* per-row `key` badge (usable/blocked/unknown), and
|
|
17
|
+
* the display-only per-task price
|
|
18
|
+
* POST /api/explain -> { role, def, modelId } -> full decomposition plus
|
|
19
|
+
* the two price-lever readouts
|
|
16
20
|
* POST /api/export -> { roles } -> validated, backed-up write of the
|
|
17
21
|
* active scope's lock file
|
|
18
22
|
* POST /api/refresh -> refetch the dataset, return the bootstrap payload
|
|
@@ -26,12 +30,15 @@ import { createServer, type IncomingMessage, type Server, type ServerResponse }
|
|
|
26
30
|
import { extname, resolve, sep } from "node:path";
|
|
27
31
|
import type { KeyAvailability } from "../availability.ts";
|
|
28
32
|
import { loadDeclaredSources } from "../benchmark-sources.ts";
|
|
29
|
-
import { SUFFIX_LEVELS, rankRole, roleLambda, thinkingPriceFactor, type RankData, type RoleDef, type SuffixLevel } from "../engine.ts";
|
|
33
|
+
import { pinnedRoute, SUFFIX_LEVELS, rankRole, roleLambda, thinkingPriceFactor, type RankData, type RoleDef, type SuffixLevel } from "../engine.ts";
|
|
30
34
|
import { expandFeatures, FEATURES } from "../features.ts";
|
|
31
35
|
import { isRecord } from "../guards.ts";
|
|
32
36
|
import { validateRole, writeRoleSettings } from "../role-settings.ts";
|
|
33
37
|
import { KNOWN_METRICS, type UniverseEntry } from "../settings.ts";
|
|
38
|
+
import { resolveTokenProfile, type TokenProfile, type UsageRow } from "../usage-ledger.ts";
|
|
34
39
|
import { explainModel, focusAssessments, metricMetaFor, rankRows, weightableMetrics } from "./explain.ts";
|
|
40
|
+
import { leaderFlip, thinkingLevelEffect } from "./levers.ts";
|
|
41
|
+
import { asWrittenPool, type PoolInputs, type PoolMode } from "./pool.ts";
|
|
35
42
|
import type { Scope } from "./scopes.ts";
|
|
36
43
|
|
|
37
44
|
export type ExplorerOpts = {
|
|
@@ -63,10 +70,33 @@ export type ExplorerOpts = {
|
|
|
63
70
|
};
|
|
64
71
|
/** Re-runs loadRankData({ refresh: true }) and swaps the snapshot. */
|
|
65
72
|
refresh(): Promise<void>;
|
|
73
|
+
/**
|
|
74
|
+
* The inputs the `as-written` pool ranks with: the omp catalog, the key's
|
|
75
|
+
* tier, and the account's allowed-providers whitelist (`null` = no whitelist /
|
|
76
|
+
* harvest failed, which leaves every route in place). Re-derived per call, so
|
|
77
|
+
* a project's whitelist can never leak into another scope's view.
|
|
78
|
+
*/
|
|
79
|
+
getPoolInputs(): PoolInputs;
|
|
80
|
+
/**
|
|
81
|
+
* The recorded usage rows the per-task price resolves its token profile from,
|
|
82
|
+
* for the request's active scope — a project scope reads its own
|
|
83
|
+
* `<root>/.omp` ledger, so a project's usage never leaks into the user-level
|
|
84
|
+
* view. Optional: a host without a ledger supplies none, and the profile falls
|
|
85
|
+
* back to `tokenProfile` (or stays absent). Re-read per call, so a turn
|
|
86
|
+
* recorded while the explorer is open shows up without a restart.
|
|
87
|
+
*/
|
|
88
|
+
getUsageRows?(scope: Scope): readonly UsageRow[];
|
|
89
|
+
/**
|
|
90
|
+
* The user-set token profile — the fallback when a role has no recorded usage
|
|
91
|
+
* (the `tokenProfile` setting). `null` (the default) = unset.
|
|
92
|
+
*/
|
|
93
|
+
tokenProfile?: TokenProfile | null;
|
|
66
94
|
};
|
|
67
95
|
|
|
68
|
-
/** Per-server mutable state: the active scope id (null = the default scope)
|
|
69
|
-
|
|
96
|
+
/** Per-server mutable state: the active scope id (null = the default scope) and
|
|
97
|
+
* the ranked pool. Both are session state, so a scope switch and a refresh keep
|
|
98
|
+
* them (the pool inputs themselves are re-derived per call). */
|
|
99
|
+
type ExplorerSession = { activeScopeId: string | null; poolMode: PoolMode };
|
|
70
100
|
|
|
71
101
|
const MIME: Record<string, string> = {
|
|
72
102
|
".html": "text/html; charset=utf-8",
|
|
@@ -157,8 +187,18 @@ function activeScope(opts: ExplorerOpts, session: ExplorerSession): Scope {
|
|
|
157
187
|
return scope;
|
|
158
188
|
}
|
|
159
189
|
|
|
160
|
-
|
|
190
|
+
/** The dataset the session's pool mode ranks: the raw fetch, or the
|
|
191
|
+
* gated+pruned pool the updater writes from. Applied before ranking, never to
|
|
192
|
+
* the export path (which validates a pin against the raw dataset). */
|
|
193
|
+
function poolRank(opts: ExplorerOpts, session: ExplorerSession, rank: RankData): RankData {
|
|
194
|
+
return session.poolMode === "as-written" ? asWrittenPool(rank, opts.getPoolInputs()) : rank;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
function bootstrapPayload(opts: ExplorerOpts, session: ExplorerSession, scope: Scope): object {
|
|
161
198
|
const state = opts.getState(scope);
|
|
199
|
+
// Every count and assessment below describes the pool the user is looking at,
|
|
200
|
+
// so the header's model count matches the table's rows.
|
|
201
|
+
const rank = poolRank(opts, session, state.rank);
|
|
162
202
|
// The metric universe the UI may weight: the shipped keys plus any external
|
|
163
203
|
// metric a resolved role weights, with a derived metadata entry for each.
|
|
164
204
|
const external = new Set<string>();
|
|
@@ -170,7 +210,7 @@ function bootstrapPayload(opts: ExplorerOpts, scope: Scope): object {
|
|
|
170
210
|
// Five-axis assessment of each role's focus metrics, over the same resolved
|
|
171
211
|
// dataset the updater ranks on (so the explorer and the updater agree).
|
|
172
212
|
const focus = Object.fromEntries(
|
|
173
|
-
Object.entries(state.roles).map(([role, def]) => [role, focusAssessments(
|
|
213
|
+
Object.entries(state.roles).map(([role, def]) => [role, focusAssessments(rank.models, def, declared)]),
|
|
174
214
|
);
|
|
175
215
|
return {
|
|
176
216
|
roles: state.roles,
|
|
@@ -185,10 +225,14 @@ function bootstrapPayload(opts: ExplorerOpts, scope: Scope): object {
|
|
|
185
225
|
focusAssessments: focus,
|
|
186
226
|
levels: Object.keys(SUFFIX_LEVELS),
|
|
187
227
|
thinkingFactors: thinkingFactors(),
|
|
188
|
-
fetchedAt:
|
|
189
|
-
modelCount:
|
|
190
|
-
orMatched:
|
|
191
|
-
orPriced:
|
|
228
|
+
fetchedAt: rank.fetchedAt,
|
|
229
|
+
modelCount: rank.models.length,
|
|
230
|
+
orMatched: rank.orMatched,
|
|
231
|
+
orPriced: rank.orPriced,
|
|
232
|
+
// The ranked pool the counts above describe; the SPA's header toggle reads it.
|
|
233
|
+
poolMode: session.poolMode,
|
|
234
|
+
// The user-set token profile the per-task column falls back to (null = unset).
|
|
235
|
+
tokenProfile: opts.tokenProfile ?? null,
|
|
192
236
|
availability: {
|
|
193
237
|
active: state.availability.active,
|
|
194
238
|
reason: state.availability.reason,
|
|
@@ -210,7 +254,16 @@ async function handleScope(req: IncomingMessage, res: ServerResponse, opts: Expl
|
|
|
210
254
|
if (scope === undefined) throw new HttpError(404, `unknown scope: ${body.scope}`);
|
|
211
255
|
if (!scope.present) throw new HttpError(409, `scope unavailable: ${body.scope}`);
|
|
212
256
|
session.activeScopeId = scope.id;
|
|
213
|
-
sendJson(res, 200, bootstrapPayload(opts, scope));
|
|
257
|
+
sendJson(res, 200, bootstrapPayload(opts, session, scope));
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
async function handlePool(req: IncomingMessage, res: ServerResponse, opts: ExplorerOpts, session: ExplorerSession): Promise<void> {
|
|
261
|
+
const body = await readJson(req);
|
|
262
|
+
if (!isRecord(body) || (body.mode !== "raw" && body.mode !== "as-written")) {
|
|
263
|
+
throw new HttpError(400, 'expected { mode: "raw" | "as-written" }');
|
|
264
|
+
}
|
|
265
|
+
session.poolMode = body.mode;
|
|
266
|
+
sendJson(res, 200, bootstrapPayload(opts, session, activeScope(opts, session)));
|
|
214
267
|
}
|
|
215
268
|
|
|
216
269
|
async function handleRank(req: IncomingMessage, res: ServerResponse, opts: ExplorerOpts, session: ExplorerSession): Promise<void> {
|
|
@@ -220,7 +273,9 @@ async function handleRank(req: IncomingMessage, res: ServerResponse, opts: Explo
|
|
|
220
273
|
}
|
|
221
274
|
const role = body.role;
|
|
222
275
|
const authored = body.def as unknown as RoleDef;
|
|
223
|
-
const
|
|
276
|
+
const scope = activeScope(opts, session);
|
|
277
|
+
const state = opts.getState(scope);
|
|
278
|
+
const rank = poolRank(opts, session, state.rank);
|
|
224
279
|
// The posted def is AUTHORED (flags + explicit keys); expand it with the
|
|
225
280
|
// scope's global flags so the preview matches the resolved baseline
|
|
226
281
|
// (`state.roles[role]`) — otherwise every row's Δ would be a lie.
|
|
@@ -229,8 +284,8 @@ async function handleRank(req: IncomingMessage, res: ServerResponse, opts: Explo
|
|
|
229
284
|
// role against its effective def (shipped defaults merged with its overrides) so
|
|
230
285
|
// selecting it still shows deltas, not an empty baseline.
|
|
231
286
|
const baselineDef = state.roles[role] ?? state.universe[role]?.def ?? state.defaults[role];
|
|
232
|
-
const baseline = baselineDef ? rankRole(baselineDef,
|
|
233
|
-
const rows = rankRows(def,
|
|
287
|
+
const baseline = baselineDef ? rankRole(baselineDef, rank.models) : [];
|
|
288
|
+
const rows = rankRows(def, rank.models, baseline, state.availability, resolveTokenProfile(role, opts.getUsageRows?.(scope) ?? [], opts.tokenProfile ?? null));
|
|
234
289
|
sendJson(res, 200, {
|
|
235
290
|
eligible: rows.length,
|
|
236
291
|
lambda: roleLambda(def),
|
|
@@ -248,19 +303,42 @@ async function handleExplain(req: IncomingMessage, res: ServerResponse, opts: Ex
|
|
|
248
303
|
throw new HttpError(400, "expected { role: string, def: object, modelId: string }");
|
|
249
304
|
}
|
|
250
305
|
const authored = body.def as unknown as RoleDef;
|
|
251
|
-
const
|
|
306
|
+
const scope = activeScope(opts, session);
|
|
307
|
+
const state = opts.getState(scope);
|
|
308
|
+
const rank = poolRank(opts, session, state.rank);
|
|
252
309
|
// Same expansion as handleRank: the posted def is authored, the baseline is
|
|
253
310
|
// resolved, so the preview must apply the scope's global flags first.
|
|
254
311
|
const def = expandFeatures(authored, state.features ?? {});
|
|
255
|
-
const explanation = explainModel(def,
|
|
256
|
-
|
|
312
|
+
const explanation = explainModel(def, rank.models, body.modelId, body.role, state.availability, resolveTokenProfile(body.role, opts.getUsageRows?.(scope) ?? [], opts.tokenProfile ?? null));
|
|
313
|
+
// The two price-lever readouts are role-level, so they ride beside the
|
|
314
|
+
// per-model decomposition — computed on the active pool, so the readout and
|
|
315
|
+
// the ranking above it agree.
|
|
316
|
+
sendJson(res, 200, {
|
|
317
|
+
...explanation,
|
|
318
|
+
leaderFlip: leaderFlip(def, rank.models),
|
|
319
|
+
thinkingLevels: thinkingLevelEffect(def, rank.models),
|
|
320
|
+
errors: validateRole(body.role, authored),
|
|
321
|
+
});
|
|
257
322
|
}
|
|
258
323
|
|
|
259
324
|
async function handleExport(req: IncomingMessage, res: ServerResponse, opts: ExplorerOpts, session: ExplorerSession): Promise<void> {
|
|
260
325
|
const body = await readJson(req);
|
|
261
326
|
if (!isRecord(body) || !isRecord(body.roles)) throw new HttpError(400, "expected { roles: object }");
|
|
262
327
|
const scope = activeScope(opts, session);
|
|
263
|
-
const
|
|
328
|
+
const dirty = body.roles as unknown as Record<string, RoleDef>;
|
|
329
|
+
// A manual provider pin must name a route the loaded dataset actually has, or
|
|
330
|
+
// the exported role would be unroutable. Validate before any write; the
|
|
331
|
+
// plugin's own validator (empty/@/:) still runs, so both error sets surface.
|
|
332
|
+
const state = opts.getState(scope);
|
|
333
|
+
const errors: string[] = [];
|
|
334
|
+
for (const [name, def] of Object.entries(dirty)) {
|
|
335
|
+
errors.push(...validateRole(name, def));
|
|
336
|
+
if (def.providerPin === undefined) continue;
|
|
337
|
+
const known = state.rank.models.some((m) => pinnedRoute(m, def.providerPin) !== null);
|
|
338
|
+
if (!known) errors.push(`role "${name}": provider pin "${def.providerPin}" matches no route in the loaded dataset`);
|
|
339
|
+
}
|
|
340
|
+
if (errors.length > 0) return sendJson(res, 200, { ok: false, errors });
|
|
341
|
+
const result = writeRoleSettings(scope.lockPath, dirty);
|
|
264
342
|
if (!result.ok) return sendJson(res, 200, { ok: false, errors: result.errors });
|
|
265
343
|
sendJson(res, 200, { ok: true, backupPath: result.backupPath, roles: result.roles, lockPath: scope.lockPath });
|
|
266
344
|
}
|
|
@@ -273,12 +351,16 @@ async function handle(req: IncomingMessage, res: ServerResponse, opts: ExplorerO
|
|
|
273
351
|
if (path.startsWith("/api/")) {
|
|
274
352
|
if (path === "/api/bootstrap") {
|
|
275
353
|
if (method !== "GET") return sendText(res, 405, "method not allowed");
|
|
276
|
-
return sendJson(res, 200, bootstrapPayload(opts, activeScope(opts, session)));
|
|
354
|
+
return sendJson(res, 200, bootstrapPayload(opts, session, activeScope(opts, session)));
|
|
277
355
|
}
|
|
278
356
|
if (path === "/api/scope") {
|
|
279
357
|
if (method !== "POST") return sendText(res, 405, "method not allowed");
|
|
280
358
|
return handleScope(req, res, opts, session);
|
|
281
359
|
}
|
|
360
|
+
if (path === "/api/pool") {
|
|
361
|
+
if (method !== "POST") return sendText(res, 405, "method not allowed");
|
|
362
|
+
return handlePool(req, res, opts, session);
|
|
363
|
+
}
|
|
282
364
|
if (path === "/api/rank") {
|
|
283
365
|
if (method !== "POST") return sendText(res, 405, "method not allowed");
|
|
284
366
|
return handleRank(req, res, opts, session);
|
|
@@ -294,7 +376,7 @@ async function handle(req: IncomingMessage, res: ServerResponse, opts: ExplorerO
|
|
|
294
376
|
if (path === "/api/refresh") {
|
|
295
377
|
if (method !== "POST") return sendText(res, 405, "method not allowed");
|
|
296
378
|
await opts.refresh();
|
|
297
|
-
return sendJson(res, 200, bootstrapPayload(opts, activeScope(opts, session)));
|
|
379
|
+
return sendJson(res, 200, bootstrapPayload(opts, session, activeScope(opts, session)));
|
|
298
380
|
}
|
|
299
381
|
return sendText(res, 404, "not found");
|
|
300
382
|
}
|
|
@@ -304,7 +386,9 @@ async function handle(req: IncomingMessage, res: ServerResponse, opts: ExplorerO
|
|
|
304
386
|
}
|
|
305
387
|
|
|
306
388
|
export function createExplorerServer(opts: ExplorerOpts): Server {
|
|
307
|
-
|
|
389
|
+
// `raw` is the historical behaviour, so a fresh server ranks exactly what it
|
|
390
|
+
// ranked before the pool toggle existed.
|
|
391
|
+
const session: ExplorerSession = { activeScopeId: null, poolMode: "raw" };
|
|
308
392
|
return createServer((req, res) => {
|
|
309
393
|
handle(req, res, opts, session).catch((err: unknown) => {
|
|
310
394
|
if (res.headersSent) {
|
package/src/extension.ts
CHANGED
|
@@ -20,16 +20,20 @@ import { generateAgentSpec, generateProjectProfile } from "./agent-architect.ts"
|
|
|
20
20
|
import { authorBenchmarkSource } from "./benchmark-author.ts";
|
|
21
21
|
import { applyBenchmarkScores, BENCHMARK_ENTRY_CAP, cachedSourceInfo, declarationToSource, declaredSourceForLink, dryRunDeclaration, fetchJson, loadBenchmarkCatalog, loadBenchmarkScores, loadDeclaredSources, resolveBenchmarkSource, saveDeclaredSource, sourceForMetric, validateDeclaration, type BenchmarkCatalogEntry, type BenchmarkPayloadMeta, type BenchmarkSource, type SourceDeclaration } from "./benchmark-sources.ts";
|
|
22
22
|
import { judgeBenchmarkRelevance } from "./benchmark-discovery.ts";
|
|
23
|
-
import {
|
|
23
|
+
import { catalogFromOmpModelsJson, currentRankingId, fetchAllowedProviders, fetchKeyAvailability, fetchKeyMeta, filterCatalog, splitThinkingSuffix, tierGate, type KeyAvailability, type Tier } from "./availability.ts";
|
|
24
|
+
import { parseConfig } from "./config-edit.ts";
|
|
24
25
|
import { loadRankData, rankRole, type Model } from "./engine.ts";
|
|
25
26
|
import { defaultErrorReportDeps, reportUnexpectedError, type ErrorReportInput } from "./error-report.ts";
|
|
26
27
|
import { formatFeatures } from "./features.ts";
|
|
27
28
|
import { startExplorer, type ExplorerHandle } from "./explorer/boot.ts";
|
|
28
29
|
import { resolveScopes, unionRoles } from "./explorer/scopes.ts";
|
|
29
30
|
import { isRecord } from "./guards.ts";
|
|
31
|
+
import { applyProviderDisplay, providerPinOf, providerPinOfSelector } from "./provider-display.ts";
|
|
30
32
|
import { applyProfileOverrides, formatProjectRolesReport, parseProjectRolesArgs, PROJECT_ROLES_USAGE, setupProject, type ProjectProfile } from "./project-setup.ts";
|
|
31
33
|
import { DEFAULT_ROLES, findProjectAnchor, KNOWN_METRICS, PLUGIN_SETTINGS_PATH, projectLockPath, readPluginSettingsMap, resolveSettings } from "./settings.ts";
|
|
32
|
-
import {
|
|
34
|
+
import { agentDir } from "./state.ts";
|
|
35
|
+
import { resolveProjectDir, runUpdater, type Deps } from "./updater.ts";
|
|
36
|
+
import { appendUsageRow, readUsageRows, type UsageRow } from "./usage-ledger.ts";
|
|
33
37
|
|
|
34
38
|
/** SPA directory shipped beside this extension (repo `web/`). */
|
|
35
39
|
const WEB_DIR = fileURLToPath(new URL("../web", import.meta.url));
|
|
@@ -71,17 +75,128 @@ function hasEmptyConversation(ctx: ExtContext): boolean {
|
|
|
71
75
|
return true;
|
|
72
76
|
}
|
|
73
77
|
|
|
74
|
-
/**
|
|
75
|
-
|
|
78
|
+
/**
|
|
79
|
+
* True when `selector` names the live session model: the selector's base
|
|
80
|
+
* (thinking suffix stripped) resolves to a model with the same `provider` and
|
|
81
|
+
* `id` as `ctx.models.current()`. Any thinking level or provider pin is
|
|
82
|
+
* ignored. Used both to detect a session booted on the old default role and to
|
|
83
|
+
* decide whether a fallback event's selector is authoritative for the session
|
|
84
|
+
* (advisor fallbacks emit the same events with the advisor's selector).
|
|
85
|
+
*/
|
|
86
|
+
function liveModelIsSelector(ctx: ExtContext, selector: string): boolean {
|
|
76
87
|
const current = ctx.models?.current();
|
|
77
|
-
const
|
|
78
|
-
const
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
if (
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
88
|
+
const { base } = splitThinkingSuffix(selector);
|
|
89
|
+
const target = ctx.models?.resolve(base);
|
|
90
|
+
if (current === undefined || current === null || target === undefined || target === null) return false;
|
|
91
|
+
if (typeof current !== "object" || typeof target !== "object") return false;
|
|
92
|
+
if (!("provider" in current) || !("id" in current) || !("provider" in target) || !("id" in target)) return false;
|
|
93
|
+
return current.provider === target.provider && current.id === target.id;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Apply the provider display suffix to the live session model (spec #77).
|
|
98
|
+
* `selector` is the event's authoritative selector string when the event
|
|
99
|
+
* carries one; the live model's routing block is the fallback. The event's
|
|
100
|
+
* selector is trusted only when it names the live session model — the fallback
|
|
101
|
+
* events are also emitted for advisor fallbacks, where only the advisor's model
|
|
102
|
+
* changes and the session model is untouched. Contained: a throw must not tear
|
|
103
|
+
* down the session or escape into omp's dispatch.
|
|
104
|
+
*/
|
|
105
|
+
function applyProviderDisplayFor(ctx: ExtContext, selector?: string): void {
|
|
106
|
+
try {
|
|
107
|
+
const live = ctx.models?.current();
|
|
108
|
+
const eventPin = typeof selector === "string" && liveModelIsSelector(ctx, selector) ? providerPinOfSelector(selector) : null;
|
|
109
|
+
applyProviderDisplay(live, eventPin ?? providerPinOf(live));
|
|
110
|
+
} catch {
|
|
111
|
+
// display-only; never let a failure escape into omp's dispatch
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* Append one usage-ledger row per assistant turn in the session's branch
|
|
117
|
+
* (spec #81). The branch is the session's own record — the same entries the
|
|
118
|
+
* session jsonl persists — so each row carries the model, the provider, and the
|
|
119
|
+
* usage block omp billed. Rows already in the ledger (same timestamp + model)
|
|
120
|
+
* are skipped, so firing this on several events cannot double-record a turn.
|
|
121
|
+
*
|
|
122
|
+
* The ledger follows the plugin's state location: a project session records into
|
|
123
|
+
* `<cwd>/.omp`, so a project's usage stays separate from the user-level profile
|
|
124
|
+
* (the same dir the updater writes config/state/history to, and the same dir the
|
|
125
|
+
* role map is read from).
|
|
126
|
+
*
|
|
127
|
+
* Best-effort: a recording failure never breaks a session (and `appendUsageRow`
|
|
128
|
+
* never throws either).
|
|
129
|
+
*/
|
|
130
|
+
function recordUsage(ctx: ExtContext): void {
|
|
131
|
+
try {
|
|
132
|
+
const branch = ctx.sessionManager?.getBranch();
|
|
133
|
+
if (branch === undefined) return;
|
|
134
|
+
const dir = resolveProjectDir() ?? agentDir();
|
|
135
|
+
const roleByModel = configuredRoles(dir);
|
|
136
|
+
const seen = new Set(readUsageRows(dir).map((row) => `${row.ts}|${row.model}`));
|
|
137
|
+
for (const entry of branch) {
|
|
138
|
+
const row = usageRowOf(entry, roleByModel);
|
|
139
|
+
if (row === null) continue;
|
|
140
|
+
const key = `${row.ts}|${row.model}`;
|
|
141
|
+
if (seen.has(key)) continue;
|
|
142
|
+
seen.add(key);
|
|
143
|
+
appendUsageRow(dir, row);
|
|
144
|
+
}
|
|
145
|
+
} catch {
|
|
146
|
+
// best-effort: the ledger never owns a session's fate
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* The config's role→selector map inverted to model slug → role: a turn's
|
|
152
|
+
* `message.model` names the selector's ranking id, and `currentRankingId`
|
|
153
|
+
* strips the provider prefix, the `@pin`, and the `:level` from the configured
|
|
154
|
+
* selector, so the two join. The first role wins when two share a model; an
|
|
155
|
+
* unreadable config maps nothing, so every row's role is "".
|
|
156
|
+
*/
|
|
157
|
+
function configuredRoles(dir: string): Map<string, string> {
|
|
158
|
+
const byModel = new Map<string, string>();
|
|
159
|
+
let text: string;
|
|
160
|
+
try {
|
|
161
|
+
text = readFileSync(join(dir, "config.yml"), "utf8");
|
|
162
|
+
} catch {
|
|
163
|
+
return byModel;
|
|
164
|
+
}
|
|
165
|
+
for (const [role, selector] of Object.entries(parseConfig(text).modelRoles)) {
|
|
166
|
+
const id = currentRankingId(selector);
|
|
167
|
+
if (id !== "" && !byModel.has(id)) byModel.set(id, role);
|
|
168
|
+
}
|
|
169
|
+
return byModel;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* The ledger row for one session branch entry: an assistant message carrying a
|
|
174
|
+
* usage block. Null for every other entry (user turns, tool results, custom
|
|
175
|
+
* entries) and for a message with no usage to record.
|
|
176
|
+
*/
|
|
177
|
+
function usageRowOf(entry: unknown, roleByModel: ReadonlyMap<string, string>): UsageRow | null {
|
|
178
|
+
if (!isRecord(entry) || entry.type !== "message") return null;
|
|
179
|
+
const message = entry.message;
|
|
180
|
+
if (!isRecord(message) || message.role !== "assistant" || typeof message.model !== "string") return null;
|
|
181
|
+
const usage = message.usage;
|
|
182
|
+
if (!isRecord(usage)) return null;
|
|
183
|
+
const cost = isRecord(usage.cost) ? usage.cost.total : 0;
|
|
184
|
+
return {
|
|
185
|
+
ts: typeof entry.timestamp === "string" ? entry.timestamp : new Date().toISOString(),
|
|
186
|
+
role: roleByModel.get(currentRankingId(message.model)) ?? "",
|
|
187
|
+
model: message.model,
|
|
188
|
+
provider: typeof message.provider === "string" ? message.provider : "",
|
|
189
|
+
input: usageNumber(usage.input),
|
|
190
|
+
output: usageNumber(usage.output),
|
|
191
|
+
cacheRead: usageNumber(usage.cacheRead),
|
|
192
|
+
cacheWrite: usageNumber(usage.cacheWrite),
|
|
193
|
+
cost: usageNumber(cost),
|
|
194
|
+
};
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/** A usage field as a finite number; an absent field records 0. */
|
|
198
|
+
function usageNumber(value: unknown): number {
|
|
199
|
+
return typeof value === "number" && Number.isFinite(value) ? value : 0;
|
|
85
200
|
}
|
|
86
201
|
|
|
87
202
|
/**
|
|
@@ -97,10 +212,8 @@ function makeApplySessionModel(pi: ExtensionAPI, ctx: ExtContext, notify: (lines
|
|
|
97
212
|
if (ctx.agent.kind !== "main") return;
|
|
98
213
|
if (!hasEmptyConversation(ctx)) return;
|
|
99
214
|
if (previous === null) return; // first run ever wrote a fresh default; the session predates no old role
|
|
100
|
-
if (!
|
|
101
|
-
const
|
|
102
|
-
const level = colon !== -1 && selector.slice(colon + 1) in THINKING_LEVELS ? selector.slice(colon + 1) : null;
|
|
103
|
-
const base = level === null ? selector : selector.slice(0, selector.length - level.length - 1);
|
|
215
|
+
if (!liveModelIsSelector(ctx, previous)) return;
|
|
216
|
+
const { base, level } = splitThinkingSuffix(selector);
|
|
104
217
|
const model = ctx.models?.resolve(base);
|
|
105
218
|
if (model === undefined || model === null) {
|
|
106
219
|
notify([`omp-llm-role: session model not updated — "${base}" matches no available model`]);
|
|
@@ -418,6 +531,34 @@ export default function (pi: ExtensionAPI) {
|
|
|
418
531
|
}
|
|
419
532
|
});
|
|
420
533
|
|
|
534
|
+
// Provider display (spec #77): keep the live model's name suffixed with its
|
|
535
|
+
// provider pin. Never pi.setModel — that would wipe omp's in-flight fallback
|
|
536
|
+
// state and clobber the thinking level; mutate the live model's name only.
|
|
537
|
+
// Registered after the updater so the session-start pin reflects the model
|
|
538
|
+
// the run just selected.
|
|
539
|
+
pi.on("session_start", async (_event, ctx: ExtContext) => {
|
|
540
|
+
applyProviderDisplayFor(ctx);
|
|
541
|
+
});
|
|
542
|
+
pi.on("retry_fallback_applied", async (event, ctx: ExtContext) => {
|
|
543
|
+
applyProviderDisplayFor(ctx, event.to);
|
|
544
|
+
});
|
|
545
|
+
pi.on("retry_fallback_succeeded", async (event, ctx: ExtContext) => {
|
|
546
|
+
applyProviderDisplayFor(ctx, event.model);
|
|
547
|
+
});
|
|
548
|
+
// Self-healing hook: omp restores the primary after a fallback cooldown with
|
|
549
|
+
// no session event, so re-derive the pin from the live model before each turn.
|
|
550
|
+
pi.on("turn_start", async (_event, ctx: ExtContext) => {
|
|
551
|
+
applyProviderDisplayFor(ctx);
|
|
552
|
+
});
|
|
553
|
+
|
|
554
|
+
// Usage ledger (spec #81): one row per assistant turn, read from the session
|
|
555
|
+
// branch. turn_end records the turn that just finished; session_shutdown is
|
|
556
|
+
// the backstop for a headless one-turn run (a short-lived `omp -p` may exit
|
|
557
|
+
// before another turn_end). The file-side dedupe makes the second hook
|
|
558
|
+
// harmless.
|
|
559
|
+
pi.on("turn_end", async (_event, ctx: ExtContext) => recordUsage(ctx));
|
|
560
|
+
pi.on("session_shutdown", async (_event, ctx: ExtContext) => recordUsage(ctx));
|
|
561
|
+
|
|
421
562
|
pi.registerCommand("refresh-roles", {
|
|
422
563
|
description: "Refresh model roles from today's llm-stats/OpenRouter rankings",
|
|
423
564
|
handler: async (_args, ctx: ExtContext) => {
|
|
@@ -465,6 +606,31 @@ export default function (pi: ExtensionAPI) {
|
|
|
465
606
|
? fetchKeyAvailability(token)
|
|
466
607
|
: { active: false, reason: "unavailable", allowed: new Set(), blocked: new Set(), publicCount: 0, keyedCount: 0, fetchedAt: new Date().toISOString() };
|
|
467
608
|
};
|
|
609
|
+
// Best-effort account-scoped inputs for the as-written pool: the key's
|
|
610
|
+
// tier gates the catalog, and the allowed-providers whitelist prunes the
|
|
611
|
+
// routes the account cannot call. The canary must be a CATALOG slug
|
|
612
|
+
// (`eligible[0].id`) — a ranking id 400s before the whitelist body, so
|
|
613
|
+
// the harvest would silently return nothing. Any failure (no key, an
|
|
614
|
+
// unreachable endpoint, no whitelist) degrades the pool to the raw
|
|
615
|
+
// dataset instead of blocking Explorer boot.
|
|
616
|
+
const loadPoolInputs = async (): Promise<{ tier: Tier; allowed: ReadonlySet<string> | null }> => {
|
|
617
|
+
const token = await ctx.modelRegistry.getApiKeyForProvider("openrouter").catch(() => undefined);
|
|
618
|
+
if (!token) return { tier: "billed", allowed: null };
|
|
619
|
+
let tier: Tier = "billed";
|
|
620
|
+
try {
|
|
621
|
+
tier = tierGate(await fetchKeyMeta(token));
|
|
622
|
+
} catch {
|
|
623
|
+
// an unreachable key endpoint leaves the ordinary account's tier
|
|
624
|
+
}
|
|
625
|
+
const canary = filterCatalog(catalog, tier)[0]?.id;
|
|
626
|
+
if (canary === undefined) return { tier, allowed: null };
|
|
627
|
+
try {
|
|
628
|
+
return { tier, allowed: await fetchAllowedProviders(token, canary) };
|
|
629
|
+
} catch {
|
|
630
|
+
return { tier, allowed: null };
|
|
631
|
+
}
|
|
632
|
+
};
|
|
633
|
+
const pool = await loadPoolInputs();
|
|
468
634
|
explorer = await startExplorer({
|
|
469
635
|
webDir: WEB_DIR,
|
|
470
636
|
cwd: ctx.cwd,
|
|
@@ -472,6 +638,13 @@ export default function (pi: ExtensionAPI) {
|
|
|
472
638
|
catalog,
|
|
473
639
|
availability: await loadAvailability(),
|
|
474
640
|
reloadAvailability: loadAvailability,
|
|
641
|
+
tier: pool.tier,
|
|
642
|
+
allowedProviders: pool.allowed,
|
|
643
|
+
reloadAllowedProviders: async () => (await loadPoolInputs()).allowed,
|
|
644
|
+
tokenProfile: resolveSettings(readPluginSettingsMap({ global: PLUGIN_SETTINGS_PATH, project: projectLockPath(ctx.cwd) })).settings.tokenProfile,
|
|
645
|
+
// The user-level ledger (the project scopes read their own
|
|
646
|
+
// `<root>/.omp` ledger, so the per-task column follows the scope).
|
|
647
|
+
usageDir: agentDir(),
|
|
475
648
|
reload: (refresh) => loadRankData({ refresh, roles: explorerRoles(), extraMetrics: Object.keys(KNOWN_METRICS) }),
|
|
476
649
|
port,
|
|
477
650
|
open,
|