omp-llm-role 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -5
- package/docs/dev/architecture.md +11 -8
- package/docs/dev/ci.md +9 -0
- package/docs/dev/current-state.md +47 -12
- package/docs/dev/explorer.md +106 -19
- package/docs/dev/releasing.md +33 -3
- package/docs/dev/scoring.md +55 -4
- package/docs/dev/testing.md +43 -2
- package/docs/dev/writes-and-state.md +30 -0
- package/docs/llm-role-rankings.md +115 -115
- package/package.json +11 -1
- package/src/agent-create.ts +4 -2
- package/src/engine.ts +33 -2
- package/src/explorer/boot.ts +43 -1
- package/src/explorer/explain.ts +90 -5
- package/src/explorer/levers.ts +100 -0
- package/src/explorer/pool.ts +46 -0
- package/src/explorer/server.ts +92 -21
- package/src/extension.ts +132 -2
- package/src/settings.ts +31 -0
- package/src/updater.ts +6 -1
- package/src/usage-ledger.ts +127 -0
- package/web/app.js +132 -0
- package/web/style.css +3 -0
package/src/explorer/boot.ts
CHANGED
|
@@ -13,10 +13,13 @@
|
|
|
13
13
|
|
|
14
14
|
import { execFile } from "node:child_process";
|
|
15
15
|
import type { Server } from "node:http";
|
|
16
|
-
import {
|
|
16
|
+
import { join } from "node:path";
|
|
17
|
+
import { enrichThinkingLevels, type CatalogEntry, type KeyAvailability, type Tier } from "../availability.ts";
|
|
17
18
|
import type { RankData } from "../engine.ts";
|
|
18
19
|
import { DEFAULT_ROLES, PLUGIN_SETTINGS_PATH } from "../settings.ts";
|
|
19
20
|
import { agentDir } from "../state.ts";
|
|
21
|
+
import { readUsageRows, type TokenProfile } from "../usage-ledger.ts";
|
|
22
|
+
import type { PoolInputs } from "./pool.ts";
|
|
20
23
|
import { readScopeRoles, resolveScope, resolveScopes, type Scope, type ScopeInputs } from "./scopes.ts";
|
|
21
24
|
import { createExplorerServer } from "./server.ts";
|
|
22
25
|
|
|
@@ -49,6 +52,25 @@ export type ExplorerBootOpts = {
|
|
|
49
52
|
reload(refresh: boolean): Promise<RankData>;
|
|
50
53
|
/** Re-derive the key-availability overlay for `POST /api/refresh` (best-effort). */
|
|
51
54
|
reloadAvailability?: () => Promise<KeyAvailability>;
|
|
55
|
+
/**
|
|
56
|
+
* The as-written pool's inputs (spec #81): the key's tier and the account's
|
|
57
|
+
* allowed-providers whitelist. Both are best-effort — an absent tier reads as
|
|
58
|
+
* `billed` (the ordinary account), and an absent/null whitelist leaves every
|
|
59
|
+
* route in place, so a failed harvest degrades to the raw pool instead of
|
|
60
|
+
* breaking the explorer.
|
|
61
|
+
*/
|
|
62
|
+
tier?: Tier;
|
|
63
|
+
allowedProviders?: ReadonlySet<string> | null;
|
|
64
|
+
/** Re-derive the whitelist for `POST /api/refresh` (best-effort: a throw or a
|
|
65
|
+
* null result degrades the as-written pool to no route pruning). */
|
|
66
|
+
reloadAllowedProviders?: () => Promise<ReadonlySet<string> | null>;
|
|
67
|
+
/** The user-set token profile the per-task price falls back to (the
|
|
68
|
+
* `tokenProfile` setting); absent = unset. */
|
|
69
|
+
tokenProfile?: TokenProfile | null;
|
|
70
|
+
/** Dir holding the user-level usage ledger the per-task price resolves its
|
|
71
|
+
* profile from (default: `agentDir()`, beside the plugin's other state). A
|
|
72
|
+
* project scope reads its own `<root>/.omp` ledger instead. */
|
|
73
|
+
usageDir?: string;
|
|
52
74
|
port?: number;
|
|
53
75
|
/** Open the URL in the default browser (macOS only). */
|
|
54
76
|
open?: boolean;
|
|
@@ -93,6 +115,11 @@ export async function startExplorer(opts: ExplorerBootOpts): Promise<ExplorerHan
|
|
|
93
115
|
const scopeInputs: ScopeInputs = { userLockPath, cwd, registryDir: opts.registryDir };
|
|
94
116
|
let rank = opts.rank;
|
|
95
117
|
let availability = opts.availability;
|
|
118
|
+
// The as-written pool's mutable half: the whitelist is re-derived on refresh,
|
|
119
|
+
// the catalog and tier are boot-time facts. `null` = no pruning.
|
|
120
|
+
let allowed = opts.allowedProviders ?? null;
|
|
121
|
+
const tier = opts.tier ?? "billed";
|
|
122
|
+
const usageDir = opts.usageDir ?? agentDir();
|
|
96
123
|
enrichThinkingLevels(rank.models, opts.catalog);
|
|
97
124
|
|
|
98
125
|
// Scope resolution is boot's job: the user-level scope, the registry's
|
|
@@ -117,7 +144,22 @@ export async function startExplorer(opts: ExplorerBootOpts): Promise<ExplorerHan
|
|
|
117
144
|
rank = await opts.reload(true);
|
|
118
145
|
enrichThinkingLevels(rank.models, opts.catalog);
|
|
119
146
|
if (opts.reloadAvailability) availability = await opts.reloadAvailability();
|
|
147
|
+
// Best-effort: the whitelist is account state, so a failed re-harvest must
|
|
148
|
+
// leave the explorer usable (routes unpruned) rather than fail the refresh.
|
|
149
|
+
if (opts.reloadAllowedProviders) {
|
|
150
|
+
try {
|
|
151
|
+
allowed = await opts.reloadAllowedProviders();
|
|
152
|
+
} catch {
|
|
153
|
+
allowed = null;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
120
156
|
},
|
|
157
|
+
getPoolInputs: (): PoolInputs => ({ catalog: opts.catalog, tier, allowed }),
|
|
158
|
+
// The ledger follows the plugin's state location: a project scope keeps its
|
|
159
|
+
// usage in its own `<root>/.omp`, so its per-task column never mixes with
|
|
160
|
+
// the user-level profile.
|
|
161
|
+
getUsageRows: (scope) => readUsageRows(scope.kind === "project" && scope.root !== undefined ? join(scope.root, ".omp") : usageDir),
|
|
162
|
+
tokenProfile: opts.tokenProfile ?? null,
|
|
121
163
|
});
|
|
122
164
|
// Optional detach (extension only): never hold the host process open on the
|
|
123
165
|
// server's account, so a short-lived `omp -p` run cannot hang.
|
package/src/explorer/explain.ts
CHANGED
|
@@ -15,8 +15,9 @@
|
|
|
15
15
|
import { assessFocusMetric, type FocusMetricAssessment } from "../agent-create.ts";
|
|
16
16
|
import { type KeyAvailability } from "../availability.ts";
|
|
17
17
|
import { cachedSourceInfo, sourceForMetric, type SourceDeclaration } from "../benchmark-sources.ts";
|
|
18
|
-
import { autoPinApplies, BENCHMARK_CHANCE, CAPABILITY_FILL, cardinalMetric, endpointFilteredModel, paretoFrontier, pinnedRoute, rankRole, roleLambda, roleMetricValue, roleRoute, routeEffectivePrice, thinkingAdjustedPrice, type Model, type Ranked, type RoleDef, type SuffixLevel } from "../engine.ts";
|
|
18
|
+
import { autoPinApplies, BENCHMARK_CHANCE, blendOutputPrice, blendRoutePool, CAPABILITY_FILL, cardinalMetric, endpointFilteredModel, paretoFrontier, pinnedRoute, rankRole, roleLambda, roleMetricValue, roleRoute, routeEffectivePrice, routeInputPrice, routeOutputPrice, routePassesEndpointFilters, thinkingAdjustedPrice, type Model, type OpenRouterEndpointRecord, type Ranked, type RoleDef, type SuffixLevel } from "../engine.ts";
|
|
19
19
|
import { KNOWN_METRICS } from "../settings.ts";
|
|
20
|
+
import { type TokenProfile } from "../usage-ledger.ts";
|
|
20
21
|
|
|
21
22
|
// ---------------------------------------------------------------------------
|
|
22
23
|
// Metric metadata (the UI's picklists and formula legends)
|
|
@@ -126,6 +127,79 @@ export function focusAssessments(
|
|
|
126
127
|
return out;
|
|
127
128
|
}
|
|
128
129
|
|
|
130
|
+
// ---------------------------------------------------------------------------
|
|
131
|
+
// Per-task price (display-only)
|
|
132
|
+
// ---------------------------------------------------------------------------
|
|
133
|
+
|
|
134
|
+
/** The per-task price of one model under a role: the role's priced route's input
|
|
135
|
+
* and output prices applied to the role's token profile, with the output term
|
|
136
|
+
* scaled by the role's thinking factor. Display-only — it never feeds the
|
|
137
|
+
* ranking's sort key, `value`, or the explanation's `cost` block. */
|
|
138
|
+
export type PerTaskPrice = {
|
|
139
|
+
inputTokens: number;
|
|
140
|
+
outputTokens: number;
|
|
141
|
+
/** the route's input price, $/M (cache-adjusted by the role's cache-hit rate) */
|
|
142
|
+
inputPrice: number;
|
|
143
|
+
/** the route's output price, $/M (derived from the billed blend and the input price) */
|
|
144
|
+
outputPrice: number;
|
|
145
|
+
/** the role's thinking factor (1 for a bare role) */
|
|
146
|
+
thinkingFactor: number;
|
|
147
|
+
/** dollars per task */
|
|
148
|
+
total: number;
|
|
149
|
+
};
|
|
150
|
+
|
|
151
|
+
/** The model's blended input price: the 1/weightPrice²-weighted mean of the
|
|
152
|
+
* routes' cache-adjusted input prices (`routeInputPrice`) — the same weighting
|
|
153
|
+
* `blendRoutePool` uses (the router's sort key), so price each route at its input
|
|
154
|
+
* price and read the blend back. The cache-hit rate must reach this term too, or
|
|
155
|
+
* `p_out` would mix a cache-adjusted billed blend with an unadjusted input price. */
|
|
156
|
+
function blendedInputPrice(routes: readonly OpenRouterEndpointRecord[], cacheHitRate: number): number | null {
|
|
157
|
+
const priced = routes.map((r) => ({ ...r, price: routeInputPrice(r, cacheHitRate) }));
|
|
158
|
+
return blendRoutePool(priced, 0).price;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/** The per-task price of `model` under `def` for a token profile: `p_in` is the
|
|
162
|
+
* role's priced route's input price (cache-adjusted), `p_out` its output price
|
|
163
|
+
* (the inverse 3:1 blend of the same route, `routeOutputPrice`), and the output
|
|
164
|
+
* term is scaled by the role's thinking factor. When the role prices on the
|
|
165
|
+
* 1/price² blend (`roleRoute` returns null) the model's billed blend and the input
|
|
166
|
+
* blend over the routes that survive the role's endpoint filters stand in — the
|
|
167
|
+
* same pool the ranking prices on, so `p_in` and the billed blend never mix two
|
|
168
|
+
* route pools. `null` when there is no usable route price (absent, never a silent
|
|
169
|
+
* zero). */
|
|
170
|
+
export function perTaskPrice(model: Model, def: RoleDef, profile: TokenProfile): PerTaskPrice | null {
|
|
171
|
+
const cacheHitRate = def.cacheHitRate ?? 0;
|
|
172
|
+
// A role with endpoint filters prices on the routes that survive them; a model
|
|
173
|
+
// whose every route fails is ineligible (the ranking drops it), so its per-task
|
|
174
|
+
// price is absent, not the unfiltered blend.
|
|
175
|
+
const em = endpointFilteredModel(model, def.filters, cacheHitRate);
|
|
176
|
+
if (em === null) return null;
|
|
177
|
+
const route = roleRoute(model, def);
|
|
178
|
+
let inputPrice: number | null;
|
|
179
|
+
let outputPrice: number | null;
|
|
180
|
+
if (route !== null) {
|
|
181
|
+
// The role prices this model on one route: that route's own input and output
|
|
182
|
+
// prices, the same pair the billed blend comes from.
|
|
183
|
+
inputPrice = routeInputPrice(route, cacheHitRate);
|
|
184
|
+
outputPrice = routeOutputPrice(route, cacheHitRate);
|
|
185
|
+
} else {
|
|
186
|
+
// The 1/price² blend case: the billed blend `em` is priced on and the input
|
|
187
|
+
// blend over exactly the routes it blended — the endpoint-filtered pool, never
|
|
188
|
+
// the model's full route list (a dropped route's input price belongs to no
|
|
189
|
+
// route's output price).
|
|
190
|
+
const pool = (model.routes ?? []).filter((r) => routePassesEndpointFilters(r, def.filters));
|
|
191
|
+
inputPrice = blendedInputPrice(pool, cacheHitRate);
|
|
192
|
+
outputPrice = inputPrice === null || em.price === null ? null : blendOutputPrice(em.price, inputPrice);
|
|
193
|
+
}
|
|
194
|
+
if (inputPrice === null || outputPrice === null) return null;
|
|
195
|
+
// The same gated factor the ranking applies: `thinkingAdjustedPrice(model, def, 1)`
|
|
196
|
+
// is 1 unless the model actually runs at the role's level, so the per-task price
|
|
197
|
+
// and the row's `priceEff` can never disagree for a non-thinking model.
|
|
198
|
+
const thinkingFactor = thinkingAdjustedPrice(model, def, 1);
|
|
199
|
+
const total = (profile.inputTokens * inputPrice + profile.outputTokens * outputPrice * thinkingFactor) / 1e6;
|
|
200
|
+
return { inputTokens: profile.inputTokens, outputTokens: profile.outputTokens, inputPrice, outputPrice, thinkingFactor, total };
|
|
201
|
+
}
|
|
202
|
+
|
|
129
203
|
// ---------------------------------------------------------------------------
|
|
130
204
|
// Rank rows (table data with baseline deltas)
|
|
131
205
|
// ---------------------------------------------------------------------------
|
|
@@ -156,6 +230,9 @@ export type RankRow = {
|
|
|
156
230
|
baselineRank: number | null;
|
|
157
231
|
/** baselineRank − rank; positive = moved up */
|
|
158
232
|
delta: number | null;
|
|
233
|
+
/** dollars per task under the role's token profile, or null when no profile
|
|
234
|
+
* was supplied or the model has no usable route price. Display-only. */
|
|
235
|
+
perTask: number | null;
|
|
159
236
|
};
|
|
160
237
|
|
|
161
238
|
/** The availability overlay verdict for one model id: `"usable"` when the keyed
|
|
@@ -172,8 +249,9 @@ function keyVerdict(availability: KeyAvailability | undefined, id: string): "usa
|
|
|
172
249
|
/** Rank `models` under `def`, annotating each row with its rank delta against
|
|
173
250
|
* `baseline` (the effective role's ranking, i.e. what the plugin does today) and
|
|
174
251
|
* its keyed-catalog availability overlay. Ranking order, deltas, and every other
|
|
175
|
-
* field are unaffected by `availability
|
|
176
|
-
|
|
252
|
+
* field are unaffected by `availability` or `profile`; `profile` only fills the
|
|
253
|
+
* display-only `perTask` column. */
|
|
254
|
+
export function rankRows(def: RoleDef, models: Model[], baseline: Ranked[], availability?: KeyAvailability, profile?: TokenProfile | null): RankRow[] {
|
|
177
255
|
const ranked = rankRole(def, models);
|
|
178
256
|
const frontier = paretoFrontier(ranked);
|
|
179
257
|
const baselineRank = new Map<string, number>();
|
|
@@ -200,6 +278,7 @@ export function rankRows(def: RoleDef, models: Model[], baseline: Ranked[], avai
|
|
|
200
278
|
rank,
|
|
201
279
|
baselineRank: b,
|
|
202
280
|
delta: b == null ? null : b - rank,
|
|
281
|
+
perTask: profile == null ? null : perTaskPrice(r.model, def, profile)?.total ?? null,
|
|
203
282
|
};
|
|
204
283
|
});
|
|
205
284
|
}
|
|
@@ -275,6 +354,10 @@ export type Explanation =
|
|
|
275
354
|
role: { name: string; lambda: number; derivedLambda: number; wPrice: number; qW: number; thinking: SuffixLevel | undefined; cacheHitRate: number };
|
|
276
355
|
contributions: Contribution[];
|
|
277
356
|
cost: { priceEff: number; billedPrice: number; lambda: number; penalty: number; q: number; value: number };
|
|
357
|
+
/** the per-task price breakdown under the role's token profile, or null
|
|
358
|
+
* when no profile was supplied or the model has no usable route price.
|
|
359
|
+
* Display-only — never part of `cost` or the ranking. */
|
|
360
|
+
perTask: PerTaskPrice | null;
|
|
278
361
|
gapAbove: number | null;
|
|
279
362
|
gapToTop: number;
|
|
280
363
|
above: { id: string; name: string; value: number } | null;
|
|
@@ -284,8 +367,9 @@ export type Explanation =
|
|
|
284
367
|
|
|
285
368
|
/** Full decomposition of one model's rank for a role, or the eligibility gates it
|
|
286
369
|
* failed (same gates `rankRole` applies, listed in the same order). The eligible
|
|
287
|
-
* branch also carries the keyed-catalog availability overlay (`key`/`keyReason`)
|
|
288
|
-
|
|
370
|
+
* branch also carries the keyed-catalog availability overlay (`key`/`keyReason`)
|
|
371
|
+
* and, when a token `profile` is supplied, the display-only per-task price. */
|
|
372
|
+
export function explainModel(def: RoleDef, models: Model[], modelId: string, roleName = "", availability?: KeyAvailability, profile?: TokenProfile | null): Explanation {
|
|
289
373
|
const model = models.find((m) => m.id === modelId);
|
|
290
374
|
if (!model) return { eligible: false, reasons: ["model not found in today's dataset"] };
|
|
291
375
|
|
|
@@ -424,6 +508,7 @@ export function explainModel(def: RoleDef, models: Model[], modelId: string, rol
|
|
|
424
508
|
role: { name: roleName, lambda, derivedLambda, wPrice, qW, thinking: def.thinking, cacheHitRate },
|
|
425
509
|
contributions,
|
|
426
510
|
cost: { priceEff: self.priceEff, billedPrice: route !== null ? routeEffectivePrice(route, cacheHitRate) ?? eff.price : eff.price, lambda, penalty: lambda * self.priceEff, q: self.q, value: self.value },
|
|
511
|
+
perTask: profile == null ? null : perTaskPrice(model, def, profile),
|
|
427
512
|
gapAbove,
|
|
428
513
|
gapToTop,
|
|
429
514
|
above,
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Price-lever readouts for the ranking explorer: the leader-flip threshold (the
|
|
3
|
+
* price weight at which a cheaper challenger overtakes the value leader) and the
|
|
4
|
+
* thinking level's effect on the leader.
|
|
5
|
+
*
|
|
6
|
+
* Both are pure and computed on the pool they are handed, so the caller decides
|
|
7
|
+
* raw vs as-written. The math is the engine's own — `roleLambda`, `rankRole`,
|
|
8
|
+
* `SUFFIX_LEVELS` and `thinkingPriceFactor` are reused, never re-derived.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import {
|
|
12
|
+
rankRole,
|
|
13
|
+
SUFFIX_LEVELS,
|
|
14
|
+
thinkingPriceFactor,
|
|
15
|
+
type Model,
|
|
16
|
+
type Ranked,
|
|
17
|
+
type RoleDef,
|
|
18
|
+
type SuffixLevel,
|
|
19
|
+
} from "../engine.ts";
|
|
20
|
+
|
|
21
|
+
/** The price weight at which a cheaper challenger overtakes the value leader. */
|
|
22
|
+
export type LeaderFlip = {
|
|
23
|
+
/** the role's current price weight */
|
|
24
|
+
wPrice: number;
|
|
25
|
+
/** the price weight at which a cheaper challenger overtakes */
|
|
26
|
+
flipW: number;
|
|
27
|
+
/** `flipW − wPrice` */
|
|
28
|
+
distance: number;
|
|
29
|
+
challenger: { id: string; name: string; value: number; priceEff: number };
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* The price weight at which a cheaper challenger overtakes the value leader, or
|
|
34
|
+
* `null` when no cheaper challenger exists (the readout is absent, never
|
|
35
|
+
* invented).
|
|
36
|
+
*
|
|
37
|
+
* `λ* = min over cheaper challengers X of (q_L − q_X)/(pEff_L − pEff_X)`, then
|
|
38
|
+
* `w* = 20·λ* ÷ (1 + 20·λ*)` — the inverse of `roleLambda`'s
|
|
39
|
+
* `λ = w/(1−w)/20`. The degenerate case — the value leader is also the quality
|
|
40
|
+
* leader, where the naive `(q_QL − q_VL)/(p_QL − p_VL)` is 0/0 — is handled by
|
|
41
|
+
* this same min formula; it is *undefined*, not zero.
|
|
42
|
+
*
|
|
43
|
+
* The threshold is exact when the role's price basis is weight-independent (the
|
|
44
|
+
* 1/price² blend, or a hard `providerPin`). With the soft `preferOwnProvider`
|
|
45
|
+
* basis the route choice itself moves with the price weight, so the threshold is
|
|
46
|
+
* a first-order estimate.
|
|
47
|
+
*/
|
|
48
|
+
export function leaderFlip(def: RoleDef, models: Model[]): LeaderFlip | null {
|
|
49
|
+
const ranked = rankRole(def, models);
|
|
50
|
+
const leader = ranked[0];
|
|
51
|
+
if (leader === undefined) return null;
|
|
52
|
+
let best: { lambda: number; challenger: Ranked } | null = null;
|
|
53
|
+
for (const x of ranked) {
|
|
54
|
+
if (!(x.priceEff < leader.priceEff)) continue; // cheaper challengers only
|
|
55
|
+
const lambda = (leader.q - x.q) / (leader.priceEff - x.priceEff);
|
|
56
|
+
if (best === null || lambda < best.lambda) best = { lambda, challenger: x };
|
|
57
|
+
}
|
|
58
|
+
if (best === null) return null;
|
|
59
|
+
const wPrice = def.weights.price ?? 0;
|
|
60
|
+
const flipW = (20 * best.lambda) / (1 + 20 * best.lambda);
|
|
61
|
+
return {
|
|
62
|
+
wPrice,
|
|
63
|
+
flipW,
|
|
64
|
+
distance: flipW - wPrice,
|
|
65
|
+
challenger: {
|
|
66
|
+
id: best.challenger.model.id,
|
|
67
|
+
name: best.challenger.model.name,
|
|
68
|
+
value: best.challenger.value,
|
|
69
|
+
priceEff: best.challenger.priceEff,
|
|
70
|
+
},
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** One thinking level's effect on the ranking: the level, its price factor, and
|
|
75
|
+
* the leader at that level (`null` when the level ranks nothing). */
|
|
76
|
+
export type ThinkingLevelRow = {
|
|
77
|
+
level: SuffixLevel;
|
|
78
|
+
factor: number;
|
|
79
|
+
leader: { id: string; name: string; value: number } | null;
|
|
80
|
+
};
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* The leader at each thinking level, with the level's price factor, or `null`
|
|
84
|
+
* for a bare role (`def.thinking === undefined` — a role with no thinking level
|
|
85
|
+
* is unaffected).
|
|
86
|
+
*
|
|
87
|
+
* The factor is a price-sensitivity amplifier, not a hidden cost: it multiplies
|
|
88
|
+
* `priceEff`, so it penalizes expensive models more in absolute λ-units.
|
|
89
|
+
*/
|
|
90
|
+
export function thinkingLevelEffect(def: RoleDef, models: Model[]): ThinkingLevelRow[] | null {
|
|
91
|
+
if (def.thinking === undefined) return null;
|
|
92
|
+
return (Object.keys(SUFFIX_LEVELS) as SuffixLevel[]).map((level) => {
|
|
93
|
+
const leader = rankRole({ ...def, thinking: level }, models)[0];
|
|
94
|
+
return {
|
|
95
|
+
level,
|
|
96
|
+
factor: thinkingPriceFactor(level),
|
|
97
|
+
leader: leader === undefined ? null : { id: leader.model.id, name: leader.model.name, value: leader.value },
|
|
98
|
+
};
|
|
99
|
+
});
|
|
100
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The as-written pool: the explorer's dataset filtered exactly as the updater
|
|
3
|
+
* filters it before ranking (spec #81). Two filters, both sourced from the
|
|
4
|
+
* availability primitives so the two pools cannot drift:
|
|
5
|
+
*
|
|
6
|
+
* 1. the catalog/tier gate — `filterCatalog(catalog, tier)`, then keep only
|
|
7
|
+
* models whose ranking id resolves to a tier-eligible entry
|
|
8
|
+
* (`resolveVariant`), mirroring `src/updater.ts:300` + `:376`;
|
|
9
|
+
* 2. the allowed-providers whitelist prune — each kept model's routes are
|
|
10
|
+
* filtered to allowed providers (`providerAllowed`), mirroring
|
|
11
|
+
* `src/updater.ts:349-357`. A null whitelist (no whitelist set / harvest
|
|
12
|
+
* failed) leaves the routes untouched.
|
|
13
|
+
*
|
|
14
|
+
* Pure: the input `RankData` and its models are never mutated. The fetch
|
|
15
|
+
* metadata (`fetchedAt`, `source`, `orMatched`, `orPriced`) describes the fetch,
|
|
16
|
+
* not the pool, and is carried through unchanged.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import { filterCatalog, providerAllowed, resolveVariant, type CatalogEntry, type Tier } from "../availability.ts";
|
|
20
|
+
import type { RankData } from "../engine.ts";
|
|
21
|
+
|
|
22
|
+
/** Which dataset the explorer ranks: the raw fetch, or the pool the updater writes from. */
|
|
23
|
+
export type PoolMode = "raw" | "as-written";
|
|
24
|
+
|
|
25
|
+
/** The inputs the as-written pool needs beyond the ranking data: the omp catalog,
|
|
26
|
+
* the key's tier, and the account's allowed-providers whitelist (`null` = no
|
|
27
|
+
* whitelist / harvest failed). */
|
|
28
|
+
export type PoolInputs = { catalog: CatalogEntry[]; tier: Tier; allowed: ReadonlySet<string> | null };
|
|
29
|
+
|
|
30
|
+
/** The as-written pool: `rank` with the catalog/tier gate and the
|
|
31
|
+
* allowed-providers prune applied. `tier: "none"` yields an empty pool. */
|
|
32
|
+
export function asWrittenPool(rank: RankData, inputs: PoolInputs): RankData {
|
|
33
|
+
const tier = inputs.tier;
|
|
34
|
+
const eligible = filterCatalog(inputs.catalog, tier);
|
|
35
|
+
const gated = tier === "none" ? [] : rank.models.filter((m) => resolveVariant(m.id, eligible, tier) !== null);
|
|
36
|
+
const allowed = inputs.allowed;
|
|
37
|
+
const models =
|
|
38
|
+
allowed === null
|
|
39
|
+
? gated
|
|
40
|
+
: gated.map((m) => {
|
|
41
|
+
const routes = m.routes;
|
|
42
|
+
if (routes === undefined) return m;
|
|
43
|
+
return { ...m, routes: routes.filter((r) => providerAllowed(allowed, r.providerSlug)) };
|
|
44
|
+
});
|
|
45
|
+
return { ...rank, models };
|
|
46
|
+
}
|
package/src/explorer/server.ts
CHANGED
|
@@ -10,9 +10,13 @@
|
|
|
10
10
|
* capability flags, and the feature registry
|
|
11
11
|
* POST /api/scope -> { scope } -> switch the active scope, return the
|
|
12
12
|
* new bootstrap payload
|
|
13
|
-
* POST /api/
|
|
14
|
-
*
|
|
15
|
-
* POST /api/
|
|
13
|
+
* POST /api/pool -> { mode } -> switch the ranked pool (raw vs
|
|
14
|
+
* as-written), return the new bootstrap payload
|
|
15
|
+
* POST /api/rank -> { role, def } -> rows with baseline deltas, a
|
|
16
|
+
* per-row `key` badge (usable/blocked/unknown), and
|
|
17
|
+
* the display-only per-task price
|
|
18
|
+
* POST /api/explain -> { role, def, modelId } -> full decomposition plus
|
|
19
|
+
* the two price-lever readouts
|
|
16
20
|
* POST /api/export -> { roles } -> validated, backed-up write of the
|
|
17
21
|
* active scope's lock file
|
|
18
22
|
* POST /api/refresh -> refetch the dataset, return the bootstrap payload
|
|
@@ -31,7 +35,10 @@ import { expandFeatures, FEATURES } from "../features.ts";
|
|
|
31
35
|
import { isRecord } from "../guards.ts";
|
|
32
36
|
import { validateRole, writeRoleSettings } from "../role-settings.ts";
|
|
33
37
|
import { KNOWN_METRICS, type UniverseEntry } from "../settings.ts";
|
|
38
|
+
import { resolveTokenProfile, type TokenProfile, type UsageRow } from "../usage-ledger.ts";
|
|
34
39
|
import { explainModel, focusAssessments, metricMetaFor, rankRows, weightableMetrics } from "./explain.ts";
|
|
40
|
+
import { leaderFlip, thinkingLevelEffect } from "./levers.ts";
|
|
41
|
+
import { asWrittenPool, type PoolInputs, type PoolMode } from "./pool.ts";
|
|
35
42
|
import type { Scope } from "./scopes.ts";
|
|
36
43
|
|
|
37
44
|
export type ExplorerOpts = {
|
|
@@ -63,10 +70,33 @@ export type ExplorerOpts = {
|
|
|
63
70
|
};
|
|
64
71
|
/** Re-runs loadRankData({ refresh: true }) and swaps the snapshot. */
|
|
65
72
|
refresh(): Promise<void>;
|
|
73
|
+
/**
|
|
74
|
+
* The inputs the `as-written` pool ranks with: the omp catalog, the key's
|
|
75
|
+
* tier, and the account's allowed-providers whitelist (`null` = no whitelist /
|
|
76
|
+
* harvest failed, which leaves every route in place). Re-derived per call, so
|
|
77
|
+
* a project's whitelist can never leak into another scope's view.
|
|
78
|
+
*/
|
|
79
|
+
getPoolInputs(): PoolInputs;
|
|
80
|
+
/**
|
|
81
|
+
* The recorded usage rows the per-task price resolves its token profile from,
|
|
82
|
+
* for the request's active scope — a project scope reads its own
|
|
83
|
+
* `<root>/.omp` ledger, so a project's usage never leaks into the user-level
|
|
84
|
+
* view. Optional: a host without a ledger supplies none, and the profile falls
|
|
85
|
+
* back to `tokenProfile` (or stays absent). Re-read per call, so a turn
|
|
86
|
+
* recorded while the explorer is open shows up without a restart.
|
|
87
|
+
*/
|
|
88
|
+
getUsageRows?(scope: Scope): readonly UsageRow[];
|
|
89
|
+
/**
|
|
90
|
+
* The user-set token profile — the fallback when a role has no recorded usage
|
|
91
|
+
* (the `tokenProfile` setting). `null` (the default) = unset.
|
|
92
|
+
*/
|
|
93
|
+
tokenProfile?: TokenProfile | null;
|
|
66
94
|
};
|
|
67
95
|
|
|
68
|
-
/** Per-server mutable state: the active scope id (null = the default scope)
|
|
69
|
-
|
|
96
|
+
/** Per-server mutable state: the active scope id (null = the default scope) and
|
|
97
|
+
* the ranked pool. Both are session state, so a scope switch and a refresh keep
|
|
98
|
+
* them (the pool inputs themselves are re-derived per call). */
|
|
99
|
+
type ExplorerSession = { activeScopeId: string | null; poolMode: PoolMode };
|
|
70
100
|
|
|
71
101
|
const MIME: Record<string, string> = {
|
|
72
102
|
".html": "text/html; charset=utf-8",
|
|
@@ -157,8 +187,18 @@ function activeScope(opts: ExplorerOpts, session: ExplorerSession): Scope {
|
|
|
157
187
|
return scope;
|
|
158
188
|
}
|
|
159
189
|
|
|
160
|
-
|
|
190
|
+
/** The dataset the session's pool mode ranks: the raw fetch, or the
|
|
191
|
+
* gated+pruned pool the updater writes from. Applied before ranking, never to
|
|
192
|
+
* the export path (which validates a pin against the raw dataset). */
|
|
193
|
+
function poolRank(opts: ExplorerOpts, session: ExplorerSession, rank: RankData): RankData {
|
|
194
|
+
return session.poolMode === "as-written" ? asWrittenPool(rank, opts.getPoolInputs()) : rank;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
function bootstrapPayload(opts: ExplorerOpts, session: ExplorerSession, scope: Scope): object {
|
|
161
198
|
const state = opts.getState(scope);
|
|
199
|
+
// Every count and assessment below describes the pool the user is looking at,
|
|
200
|
+
// so the header's model count matches the table's rows.
|
|
201
|
+
const rank = poolRank(opts, session, state.rank);
|
|
162
202
|
// The metric universe the UI may weight: the shipped keys plus any external
|
|
163
203
|
// metric a resolved role weights, with a derived metadata entry for each.
|
|
164
204
|
const external = new Set<string>();
|
|
@@ -170,7 +210,7 @@ function bootstrapPayload(opts: ExplorerOpts, scope: Scope): object {
|
|
|
170
210
|
// Five-axis assessment of each role's focus metrics, over the same resolved
|
|
171
211
|
// dataset the updater ranks on (so the explorer and the updater agree).
|
|
172
212
|
const focus = Object.fromEntries(
|
|
173
|
-
Object.entries(state.roles).map(([role, def]) => [role, focusAssessments(
|
|
213
|
+
Object.entries(state.roles).map(([role, def]) => [role, focusAssessments(rank.models, def, declared)]),
|
|
174
214
|
);
|
|
175
215
|
return {
|
|
176
216
|
roles: state.roles,
|
|
@@ -185,10 +225,14 @@ function bootstrapPayload(opts: ExplorerOpts, scope: Scope): object {
|
|
|
185
225
|
focusAssessments: focus,
|
|
186
226
|
levels: Object.keys(SUFFIX_LEVELS),
|
|
187
227
|
thinkingFactors: thinkingFactors(),
|
|
188
|
-
fetchedAt:
|
|
189
|
-
modelCount:
|
|
190
|
-
orMatched:
|
|
191
|
-
orPriced:
|
|
228
|
+
fetchedAt: rank.fetchedAt,
|
|
229
|
+
modelCount: rank.models.length,
|
|
230
|
+
orMatched: rank.orMatched,
|
|
231
|
+
orPriced: rank.orPriced,
|
|
232
|
+
// The ranked pool the counts above describe; the SPA's header toggle reads it.
|
|
233
|
+
poolMode: session.poolMode,
|
|
234
|
+
// The user-set token profile the per-task column falls back to (null = unset).
|
|
235
|
+
tokenProfile: opts.tokenProfile ?? null,
|
|
192
236
|
availability: {
|
|
193
237
|
active: state.availability.active,
|
|
194
238
|
reason: state.availability.reason,
|
|
@@ -210,7 +254,16 @@ async function handleScope(req: IncomingMessage, res: ServerResponse, opts: Expl
|
|
|
210
254
|
if (scope === undefined) throw new HttpError(404, `unknown scope: ${body.scope}`);
|
|
211
255
|
if (!scope.present) throw new HttpError(409, `scope unavailable: ${body.scope}`);
|
|
212
256
|
session.activeScopeId = scope.id;
|
|
213
|
-
sendJson(res, 200, bootstrapPayload(opts, scope));
|
|
257
|
+
sendJson(res, 200, bootstrapPayload(opts, session, scope));
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
async function handlePool(req: IncomingMessage, res: ServerResponse, opts: ExplorerOpts, session: ExplorerSession): Promise<void> {
|
|
261
|
+
const body = await readJson(req);
|
|
262
|
+
if (!isRecord(body) || (body.mode !== "raw" && body.mode !== "as-written")) {
|
|
263
|
+
throw new HttpError(400, 'expected { mode: "raw" | "as-written" }');
|
|
264
|
+
}
|
|
265
|
+
session.poolMode = body.mode;
|
|
266
|
+
sendJson(res, 200, bootstrapPayload(opts, session, activeScope(opts, session)));
|
|
214
267
|
}
|
|
215
268
|
|
|
216
269
|
async function handleRank(req: IncomingMessage, res: ServerResponse, opts: ExplorerOpts, session: ExplorerSession): Promise<void> {
|
|
@@ -220,7 +273,9 @@ async function handleRank(req: IncomingMessage, res: ServerResponse, opts: Explo
|
|
|
220
273
|
}
|
|
221
274
|
const role = body.role;
|
|
222
275
|
const authored = body.def as unknown as RoleDef;
|
|
223
|
-
const
|
|
276
|
+
const scope = activeScope(opts, session);
|
|
277
|
+
const state = opts.getState(scope);
|
|
278
|
+
const rank = poolRank(opts, session, state.rank);
|
|
224
279
|
// The posted def is AUTHORED (flags + explicit keys); expand it with the
|
|
225
280
|
// scope's global flags so the preview matches the resolved baseline
|
|
226
281
|
// (`state.roles[role]`) — otherwise every row's Δ would be a lie.
|
|
@@ -229,8 +284,8 @@ async function handleRank(req: IncomingMessage, res: ServerResponse, opts: Explo
|
|
|
229
284
|
// role against its effective def (shipped defaults merged with its overrides) so
|
|
230
285
|
// selecting it still shows deltas, not an empty baseline.
|
|
231
286
|
const baselineDef = state.roles[role] ?? state.universe[role]?.def ?? state.defaults[role];
|
|
232
|
-
const baseline = baselineDef ? rankRole(baselineDef,
|
|
233
|
-
const rows = rankRows(def,
|
|
287
|
+
const baseline = baselineDef ? rankRole(baselineDef, rank.models) : [];
|
|
288
|
+
const rows = rankRows(def, rank.models, baseline, state.availability, resolveTokenProfile(role, opts.getUsageRows?.(scope) ?? [], opts.tokenProfile ?? null));
|
|
234
289
|
sendJson(res, 200, {
|
|
235
290
|
eligible: rows.length,
|
|
236
291
|
lambda: roleLambda(def),
|
|
@@ -248,12 +303,22 @@ async function handleExplain(req: IncomingMessage, res: ServerResponse, opts: Ex
|
|
|
248
303
|
throw new HttpError(400, "expected { role: string, def: object, modelId: string }");
|
|
249
304
|
}
|
|
250
305
|
const authored = body.def as unknown as RoleDef;
|
|
251
|
-
const
|
|
306
|
+
const scope = activeScope(opts, session);
|
|
307
|
+
const state = opts.getState(scope);
|
|
308
|
+
const rank = poolRank(opts, session, state.rank);
|
|
252
309
|
// Same expansion as handleRank: the posted def is authored, the baseline is
|
|
253
310
|
// resolved, so the preview must apply the scope's global flags first.
|
|
254
311
|
const def = expandFeatures(authored, state.features ?? {});
|
|
255
|
-
const explanation = explainModel(def,
|
|
256
|
-
|
|
312
|
+
const explanation = explainModel(def, rank.models, body.modelId, body.role, state.availability, resolveTokenProfile(body.role, opts.getUsageRows?.(scope) ?? [], opts.tokenProfile ?? null));
|
|
313
|
+
// The two price-lever readouts are role-level, so they ride beside the
|
|
314
|
+
// per-model decomposition — computed on the active pool, so the readout and
|
|
315
|
+
// the ranking above it agree.
|
|
316
|
+
sendJson(res, 200, {
|
|
317
|
+
...explanation,
|
|
318
|
+
leaderFlip: leaderFlip(def, rank.models),
|
|
319
|
+
thinkingLevels: thinkingLevelEffect(def, rank.models),
|
|
320
|
+
errors: validateRole(body.role, authored),
|
|
321
|
+
});
|
|
257
322
|
}
|
|
258
323
|
|
|
259
324
|
async function handleExport(req: IncomingMessage, res: ServerResponse, opts: ExplorerOpts, session: ExplorerSession): Promise<void> {
|
|
@@ -286,12 +351,16 @@ async function handle(req: IncomingMessage, res: ServerResponse, opts: ExplorerO
|
|
|
286
351
|
if (path.startsWith("/api/")) {
|
|
287
352
|
if (path === "/api/bootstrap") {
|
|
288
353
|
if (method !== "GET") return sendText(res, 405, "method not allowed");
|
|
289
|
-
return sendJson(res, 200, bootstrapPayload(opts, activeScope(opts, session)));
|
|
354
|
+
return sendJson(res, 200, bootstrapPayload(opts, session, activeScope(opts, session)));
|
|
290
355
|
}
|
|
291
356
|
if (path === "/api/scope") {
|
|
292
357
|
if (method !== "POST") return sendText(res, 405, "method not allowed");
|
|
293
358
|
return handleScope(req, res, opts, session);
|
|
294
359
|
}
|
|
360
|
+
if (path === "/api/pool") {
|
|
361
|
+
if (method !== "POST") return sendText(res, 405, "method not allowed");
|
|
362
|
+
return handlePool(req, res, opts, session);
|
|
363
|
+
}
|
|
295
364
|
if (path === "/api/rank") {
|
|
296
365
|
if (method !== "POST") return sendText(res, 405, "method not allowed");
|
|
297
366
|
return handleRank(req, res, opts, session);
|
|
@@ -307,7 +376,7 @@ async function handle(req: IncomingMessage, res: ServerResponse, opts: ExplorerO
|
|
|
307
376
|
if (path === "/api/refresh") {
|
|
308
377
|
if (method !== "POST") return sendText(res, 405, "method not allowed");
|
|
309
378
|
await opts.refresh();
|
|
310
|
-
return sendJson(res, 200, bootstrapPayload(opts, activeScope(opts, session)));
|
|
379
|
+
return sendJson(res, 200, bootstrapPayload(opts, session, activeScope(opts, session)));
|
|
311
380
|
}
|
|
312
381
|
return sendText(res, 404, "not found");
|
|
313
382
|
}
|
|
@@ -317,7 +386,9 @@ async function handle(req: IncomingMessage, res: ServerResponse, opts: ExplorerO
|
|
|
317
386
|
}
|
|
318
387
|
|
|
319
388
|
export function createExplorerServer(opts: ExplorerOpts): Server {
|
|
320
|
-
|
|
389
|
+
// `raw` is the historical behaviour, so a fresh server ranks exactly what it
|
|
390
|
+
// ranked before the pool toggle existed.
|
|
391
|
+
const session: ExplorerSession = { activeScopeId: null, poolMode: "raw" };
|
|
321
392
|
return createServer((req, res) => {
|
|
322
393
|
handle(req, res, opts, session).catch((err: unknown) => {
|
|
323
394
|
if (res.headersSent) {
|