omp-llm-role 1.1.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +75 -13
- package/docs/dev/architecture.md +35 -11
- package/docs/dev/ci.md +21 -1
- package/docs/dev/current-state.md +99 -27
- package/docs/dev/data-sources.md +33 -4
- package/docs/dev/explorer.md +126 -21
- package/docs/dev/quirks.md +32 -0
- package/docs/dev/releasing.md +63 -32
- package/docs/dev/scoring.md +153 -39
- package/docs/dev/testing.md +112 -24
- package/docs/dev/writes-and-state.md +86 -0
- package/docs/llm-role-rankings.md +126 -126
- package/package.json +92 -22
- package/src/agent-create.ts +4 -2
- package/src/availability.ts +87 -9
- package/src/cli/create-role.ts +1 -1
- package/src/engine.ts +190 -78
- package/src/explorer/boot.ts +43 -1
- package/src/explorer/explain.ts +113 -10
- package/src/explorer/levers.ts +100 -0
- package/src/explorer/pool.ts +46 -0
- package/src/explorer/server.ts +107 -23
- package/src/extension.ts +189 -16
- package/src/features.ts +9 -7
- package/src/provider-display.ts +67 -0
- package/src/settings.ts +39 -2
- package/src/updater.ts +97 -14
- package/src/usage-ledger.ts +127 -0
- package/web/app.js +156 -1
- package/web/style.css +3 -0
package/src/engine.ts
CHANGED
|
@@ -181,6 +181,9 @@ export type RoleDef = {
|
|
|
181
181
|
minOutputTokens?: number;
|
|
182
182
|
/** drop the model when its role-priced blend exceeds this ($/M); 0 = off */
|
|
183
183
|
maxPriceUsdPerM?: number;
|
|
184
|
+
/** soft throughput floor (tok/s); 0/absent = off. A route at or above the
|
|
185
|
+
* floor sorts before a route below it, but is never dropped. */
|
|
186
|
+
preferredMinThroughput?: number;
|
|
184
187
|
};
|
|
185
188
|
/** explicit λ override ($ per quality point); default derives from the price weight */
|
|
186
189
|
lambda?: number;
|
|
@@ -205,9 +208,9 @@ export type RoleDef = {
|
|
|
205
208
|
/** Opt-in capability flags (`roles.<role>.features.<id>`); expanded by
|
|
206
209
|
* `expandFeatures` before ranking. `true`/`false` overrides the global flag. */
|
|
207
210
|
features?: Record<string, boolean>;
|
|
208
|
-
/** Soft provider preference (issue #40): price a model on its
|
|
209
|
-
* when the role's `providerPinning` feature
|
|
210
|
-
* Unlike `providerPin` this NEVER drops a model. */
|
|
211
|
+
/** Soft provider preference (issue #40, re-based by #58): price a model on its
|
|
212
|
+
* best route by the role's own value when the role's `providerPinning` feature
|
|
213
|
+
* is on, else the 1/price² blend. Unlike `providerPin` this NEVER drops a model. */
|
|
211
214
|
preferOwnProvider?: boolean;
|
|
212
215
|
/** Locked role: the plugin still ranks it (so it stays in the universe and its
|
|
213
216
|
* sources are fetched) but never rewrites its `modelRoles` selector or fallback
|
|
@@ -310,6 +313,19 @@ export type OpenRouterEndpointPages = Record<string, OpenRouterEndpointRecord[]>
|
|
|
310
313
|
|
|
311
314
|
export type OrEnrichment = { tput: number; latency: number | null; price: number | null; thinking: boolean; routes: OpenRouterEndpointRecord[] };
|
|
312
315
|
|
|
316
|
+
/** The route's input (prompt) $/M under an assumed cache-hit rate: the input term
|
|
317
|
+
* `routeEffectivePrice` blends into the billed price. A hit rate `h` blends the
|
|
318
|
+
* cache-read input in, `h·cacheRead + (1−h)·input`; with `h = 0` or no cache-read
|
|
319
|
+
* price this is the listed input price, and a route with no input price has none —
|
|
320
|
+
* missing data keeps the full input price, never a silent discount. The single
|
|
321
|
+
* accessor for the input term: the billed blend, the output-price inverse
|
|
322
|
+
* (`routeOutputPrice`) and the explorer's per-task price all read it. */
|
|
323
|
+
export function routeInputPrice(route: OpenRouterEndpointRecord, cacheHitRate = 0): number | null {
|
|
324
|
+
if (route.weightPrice == null) return null;
|
|
325
|
+
if (cacheHitRate <= 0 || route.cacheReadPrice == null) return route.weightPrice;
|
|
326
|
+
return cacheHitRate * route.cacheReadPrice + (1 - cacheHitRate) * route.weightPrice;
|
|
327
|
+
}
|
|
328
|
+
|
|
313
329
|
/** The effective billed $/M of one route under an assumed cache-hit rate. The route's
|
|
314
330
|
* billed 3:1 price is `price = (3·input + output)/4` on the listed input price; a hit
|
|
315
331
|
* rate `h` blends the cache-read input into the input term,
|
|
@@ -317,15 +333,33 @@ export type OrEnrichment = { tput: number; latency: number | null; price: number
|
|
|
317
333
|
* (the output term is unchanged). With `h = 0`, no cache-read price, or no input price
|
|
318
334
|
* this is the route's listed `price` — missing data keeps the full input price, never a
|
|
319
335
|
* silent discount. Shared by the blend (`blendRoutePool`), the pin path (`rankRole`) and
|
|
320
|
-
* `roleMetricValue`. */
|
|
336
|
+
* `roleMetricValue`; the input term itself is `routeInputPrice`. */
|
|
321
337
|
export function routeEffectivePrice(route: OpenRouterEndpointRecord, cacheHitRate = 0): number | null {
|
|
322
338
|
if (route.price == null) return null;
|
|
323
339
|
if (cacheHitRate <= 0 || route.cacheReadPrice == null || route.weightPrice == null) return route.price;
|
|
324
340
|
const input = route.weightPrice;
|
|
325
|
-
const effInput =
|
|
341
|
+
const effInput = routeInputPrice(route, cacheHitRate) as number;
|
|
326
342
|
return route.price + (3 * (effInput - input)) / 4;
|
|
327
343
|
}
|
|
328
344
|
|
|
345
|
+
/** The output (completion) $/M implied by a billed 3:1 blend and the input price
|
|
346
|
+
* behind it: the inverse of `billed = IO_PRICE_RATIO·(3·input + output)`, i.e.
|
|
347
|
+
* `output = billed/IO_PRICE_RATIO − (1−IO_PRICE_RATIO)/IO_PRICE_RATIO · input`.
|
|
348
|
+
* The 3:1 ratio lives here (and in `IO_PRICE_RATIO`), not in the callers; dividing
|
|
349
|
+
* by `IO_PRICE_RATIO` (2⁻²) is exact, so this is the plain `4·billed − 3·input`. */
|
|
350
|
+
export function blendOutputPrice(billed: number, input: number): number {
|
|
351
|
+
return billed / IO_PRICE_RATIO - ((1 - IO_PRICE_RATIO) / IO_PRICE_RATIO) * input;
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
/** The route's output (completion) $/M: the inverse blend of its own billed and
|
|
355
|
+
* input prices (see `blendOutputPrice`). Null when either of them is missing. */
|
|
356
|
+
export function routeOutputPrice(route: OpenRouterEndpointRecord, cacheHitRate = 0): number | null {
|
|
357
|
+
const billed = routeEffectivePrice(route, cacheHitRate);
|
|
358
|
+
const input = routeInputPrice(route, cacheHitRate);
|
|
359
|
+
if (billed === null || input === null) return null;
|
|
360
|
+
return blendOutputPrice(billed, input);
|
|
361
|
+
}
|
|
362
|
+
|
|
329
363
|
/** Expected price/throughput/latency of one request over a route pool under the
|
|
330
364
|
* router's default price-based load balancing: P(route i) ∝ 1/input_price_i² — the
|
|
331
365
|
* router sorts on the input (prompt) price alone, so the weight basis is
|
|
@@ -1060,39 +1094,144 @@ export function pinnedRoute(model: Model, providerPin: string | undefined): Open
|
|
|
1060
1094
|
return (model.routes ?? []).find((r) => r.providerSlug === providerPin) ?? null;
|
|
1061
1095
|
}
|
|
1062
1096
|
|
|
1063
|
-
/** The
|
|
1064
|
-
*
|
|
1065
|
-
*
|
|
1066
|
-
*
|
|
1067
|
-
*
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
const
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1097
|
+
/** The role's thinking-adjusted billed price for a model at `basePrice`: the
|
|
1098
|
+
* thinking factor applies only when the model actually runs the role's level
|
|
1099
|
+
* (catalog `thinkingLevels` wins; without it the `supports_reasoning` flag
|
|
1100
|
+
* gates). Mirrors the factor `rankRole` applies, so the route basis and the
|
|
1101
|
+
* ranking can never disagree. */
|
|
1102
|
+
export function thinkingAdjustedPrice(model: Model, def: RoleDef, basePrice: number): number {
|
|
1103
|
+
const levelFactor = def.thinking === undefined ? 1 : thinkingPriceFactor(def.thinking);
|
|
1104
|
+
if (levelFactor === 1) return basePrice;
|
|
1105
|
+
const levels = model.thinkingLevels;
|
|
1106
|
+
const runsAtLevel = levels
|
|
1107
|
+
? levels.length > 0 && (def.thinking === undefined || META_LEVELS[def.thinking] === true || levels.includes(def.thinking))
|
|
1108
|
+
: model.thinking;
|
|
1109
|
+
return runsAtLevel ? basePrice * levelFactor : basePrice;
|
|
1110
|
+
}
|
|
1111
|
+
|
|
1112
|
+
/** The role's value of a model priced on `route` (null = the 1/price² blend):
|
|
1113
|
+
* the price-free quality composite q over the model's metrics with the route's
|
|
1114
|
+
* p50 throughput and cache-adjusted billed price, minus λ·priceEff. Returns null
|
|
1115
|
+
* when the model has no usable billed price. */
|
|
1116
|
+
function roleValueOnRoute(
|
|
1117
|
+
em: Model,
|
|
1118
|
+
def: RoleDef,
|
|
1119
|
+
route: OpenRouterEndpointRecord | null,
|
|
1120
|
+
cacheHitRate: number,
|
|
1121
|
+
lambda: number,
|
|
1122
|
+
levelFactor: number,
|
|
1123
|
+
qW: number,
|
|
1124
|
+
): { q: number; priceEff: number; value: number; parts: Record<string, number> } | null {
|
|
1125
|
+
const basePrice = route !== null ? routeEffectivePrice(route, cacheHitRate) : em.price;
|
|
1126
|
+
if (basePrice == null) return null;
|
|
1127
|
+
const priceEff = levelFactor === 1 ? basePrice : thinkingAdjustedPrice(em, def, basePrice);
|
|
1128
|
+
let q = 0;
|
|
1129
|
+
const parts: Record<string, number> = {};
|
|
1130
|
+
for (const [metric, w] of Object.entries(def.weights)) {
|
|
1131
|
+
if (metric === "price") continue; // cost enters as the λ·$ penalty, not the blend
|
|
1132
|
+
const raw = roleMetricValue(em, metric, route, cacheHitRate);
|
|
1133
|
+
if (raw == null) {
|
|
1134
|
+
// Sparse capability metrics are capability-filled (below-median, not 0)
|
|
1135
|
+
// so absence is not a coverage penalty; every other metric contributes
|
|
1136
|
+
// nothing when missing.
|
|
1137
|
+
const fill = CAPABILITY_FILL[metric];
|
|
1138
|
+
if (fill === undefined) continue;
|
|
1139
|
+
const contrib = (w / qW) * fill;
|
|
1140
|
+
parts[metric] = contrib;
|
|
1141
|
+
q += contrib;
|
|
1142
|
+
continue;
|
|
1143
|
+
}
|
|
1144
|
+
const contrib = (w / qW) * cardinalMetric(metric, raw);
|
|
1145
|
+
parts[metric] = contrib;
|
|
1146
|
+
q += contrib;
|
|
1147
|
+
}
|
|
1148
|
+
return { q, priceEff, value: q - lambda * priceEff, parts };
|
|
1149
|
+
}
|
|
1150
|
+
|
|
1151
|
+
/** The model's routes that are candidates for the role's soft best-route basis,
|
|
1152
|
+
* sorted by the value-for-money score desc (ties: providerSlug asc, id asc). A
|
|
1153
|
+
* candidate is non-degraded (status === 0), passes
|
|
1154
|
+
* `routePassesEndpointFilters(route, def.filters)`, has a usable billed price,
|
|
1155
|
+
* and (when `filters.maxPriceUsdPerM > 0`) a thinking-adjusted price at or below
|
|
1156
|
+
* the cap.
|
|
1157
|
+
*
|
|
1158
|
+
* The score is `vfm = wT·tNorm − wP·pNorm`, where `tNorm`/`pNorm` are the
|
|
1159
|
+
* route's p50 throughput and cache-adjusted billed price min/max-normalized over
|
|
1160
|
+
* the candidate set, and `wT = wTput/(wPrice+wTput)`, `wP = wPrice/(wPrice+wTput)`.
|
|
1161
|
+
* A route with no p50 uses the model's blended throughput (an absent p50 is
|
|
1162
|
+
* unknown, not slow). `filters.preferredMinThroughput` is a SOFT floor: routes at
|
|
1163
|
+
* or above it sort before routes below it, but are never dropped. */
|
|
1164
|
+
export function rankedRoutes(model: Model, def: RoleDef): OpenRouterEndpointRecord[] {
|
|
1165
|
+
const cacheHitRate = def.cacheHitRate ?? 0;
|
|
1166
|
+
const cap = def.filters?.maxPriceUsdPerM ?? 0;
|
|
1167
|
+
const floor = def.filters?.preferredMinThroughput ?? 0;
|
|
1168
|
+
const out = (model.routes ?? []).filter((r) => {
|
|
1169
|
+
if (r.status !== 0) return false;
|
|
1170
|
+
if (!routePassesEndpointFilters(r, def.filters)) return false;
|
|
1171
|
+
const p = routeEffectivePrice(r, cacheHitRate);
|
|
1172
|
+
if (p === null || p <= 0) return false;
|
|
1173
|
+
if (cap > 0 && thinkingAdjustedPrice(model, def, p) > cap) return false;
|
|
1174
|
+
return true;
|
|
1175
|
+
});
|
|
1176
|
+
if (out.length === 0) return [];
|
|
1177
|
+
// A route with no p50 falls back to the model's blended throughput over the
|
|
1178
|
+
// endpoint-filtered pool — the same basis `roleMetricValue` used before.
|
|
1179
|
+
const em = endpointFilteredModel(model, def.filters, cacheHitRate) ?? model;
|
|
1180
|
+
const wTput = def.weights.throughput ?? 0;
|
|
1181
|
+
const wPrice = def.weights.price ?? 0;
|
|
1182
|
+
const wSum = wTput + wPrice;
|
|
1183
|
+
const wT = wSum > 0 ? wTput / wSum : 0;
|
|
1184
|
+
const wP = wSum > 0 ? wPrice / wSum : 0;
|
|
1185
|
+
// Decorate-sort-undecorate: the throughput/price lookups are not free, so
|
|
1186
|
+
// compute each route's terms once.
|
|
1187
|
+
const decorated = out.map((route) => ({
|
|
1188
|
+
route,
|
|
1189
|
+
tput: roleMetricValue(em, "throughput", route, cacheHitRate) ?? 0,
|
|
1190
|
+
priceEff: routeEffectivePrice(route, cacheHitRate) as number,
|
|
1191
|
+
}));
|
|
1192
|
+
const tMin = Math.min(...decorated.map((d) => d.tput));
|
|
1193
|
+
const tMax = Math.max(...decorated.map((d) => d.tput));
|
|
1194
|
+
const pMin = Math.min(...decorated.map((d) => d.priceEff));
|
|
1195
|
+
const pMax = Math.max(...decorated.map((d) => d.priceEff));
|
|
1196
|
+
const tSpan = tMax - tMin;
|
|
1197
|
+
const pSpan = pMax - pMin;
|
|
1198
|
+
return decorated
|
|
1199
|
+
.map((d) => {
|
|
1200
|
+
const tNorm = tSpan > 0 ? (d.tput - tMin) / tSpan : 0;
|
|
1201
|
+
const pNorm = pSpan > 0 ? (d.priceEff - pMin) / pSpan : 0;
|
|
1202
|
+
return { route: d.route, score: wT * tNorm - wP * pNorm, aboveFloor: floor <= 0 || d.tput >= floor };
|
|
1203
|
+
})
|
|
1204
|
+
.sort((a, b) => {
|
|
1205
|
+
if (a.aboveFloor !== b.aboveFloor) return a.aboveFloor ? -1 : 1;
|
|
1206
|
+
if (a.score !== b.score) return b.score - a.score;
|
|
1207
|
+
if (a.route.providerSlug !== b.route.providerSlug) return a.route.providerSlug < b.route.providerSlug ? -1 : 1;
|
|
1208
|
+
return a.route.id < b.route.id ? -1 : a.route.id > b.route.id ? 1 : 0;
|
|
1209
|
+
})
|
|
1210
|
+
.map((d) => d.route);
|
|
1211
|
+
}
|
|
1212
|
+
|
|
1213
|
+
/** `rankedRoutes(model, def)[0]` or null. The auto-pin slug and the soft basis. */
|
|
1214
|
+
export function bestRoute(model: Model, def: RoleDef): OpenRouterEndpointRecord | null {
|
|
1215
|
+
return rankedRoutes(model, def)[0] ?? null;
|
|
1216
|
+
}
|
|
1217
|
+
|
|
1218
|
+
/** True when the role's soft best-route basis applies: the `preferOwnProvider`
|
|
1219
|
+
* capability is on and no manual `providerPin` overrides it. The soft basis the
|
|
1220
|
+
* engine prices on and the auto-pin the updater writes share this predicate. */
|
|
1221
|
+
export function autoPinApplies(def: RoleDef): boolean {
|
|
1222
|
+
return def.providerPin === undefined && def.preferOwnProvider === true;
|
|
1082
1223
|
}
|
|
1083
1224
|
|
|
1084
1225
|
/** The route a role prices a model on: the hard pin's route when the role is
|
|
1085
|
-
* pinned, else the model's
|
|
1086
|
-
* preference is on
|
|
1087
|
-
*
|
|
1088
|
-
*
|
|
1226
|
+
* pinned, else the model's best route by the role's own value when the soft
|
|
1227
|
+
* `preferOwnProvider` preference is on, else null (the 1/price² blend). A
|
|
1228
|
+
* preference never drops a model — a model with no candidate route keeps the
|
|
1229
|
+
* blend. */
|
|
1089
1230
|
export function roleRoute(model: Model, def: RoleDef): OpenRouterEndpointRecord | null {
|
|
1090
1231
|
const pin = pinnedRoute(model, def.providerPin);
|
|
1091
1232
|
if (pin !== null) return pin;
|
|
1092
|
-
if (def
|
|
1093
|
-
|
|
1094
|
-
if (own === null) return null;
|
|
1095
|
-
return routeEffectivePrice(own, def.cacheHitRate ?? 0) !== null ? own : null;
|
|
1233
|
+
if (!autoPinApplies(def)) return null;
|
|
1234
|
+
return bestRoute(model, def);
|
|
1096
1235
|
}
|
|
1097
1236
|
|
|
1098
1237
|
/** The metric value a role sees for a model. A pinned role's throughput is its
|
|
@@ -1142,12 +1281,11 @@ export function rankRole(def: RoleDef, models: Model[]): Ranked[] {
|
|
|
1142
1281
|
// the pin and is ineligible (recorded by `providerPinDrops`).
|
|
1143
1282
|
const pinRoute = pinnedRoute(m, def.providerPin);
|
|
1144
1283
|
if (def.providerPin !== undefined && pinRoute === null) continue;
|
|
1145
|
-
// Soft
|
|
1146
|
-
//
|
|
1147
|
-
//
|
|
1148
|
-
//
|
|
1149
|
-
//
|
|
1150
|
-
// count is unchanged. `providerPinDrops`/`pinUnmatched` key on
|
|
1284
|
+
// Soft best-route preference (issue #40, re-based by #58): with no hard pin,
|
|
1285
|
+
// a role that prefers a bound route is priced on the model's best route by
|
|
1286
|
+
// the role's own value, else on the 1/price² blend. This is a preference,
|
|
1287
|
+
// never a gate: a model with no candidate route keeps the blend, so the
|
|
1288
|
+
// eligible count is unchanged. `providerPinDrops`/`pinUnmatched` key on
|
|
1151
1289
|
// `def.providerPin` only, so the preference never drops a model.
|
|
1152
1290
|
const route = roleRoute(m, def);
|
|
1153
1291
|
// Endpoint capability gate: a role that declares endpoint filters is priced on
|
|
@@ -1160,49 +1298,23 @@ export function rankRole(def: RoleDef, models: Model[]): Ranked[] {
|
|
|
1160
1298
|
const em = endpointFilteredModel(m, def.filters, cacheHitRate);
|
|
1161
1299
|
if (em === null) continue;
|
|
1162
1300
|
// The required-metric gate keys on the hard pin only (null when unpinned), so
|
|
1163
|
-
// the soft
|
|
1301
|
+
// the soft best-route preference can never change eligibility — it only
|
|
1164
1302
|
// re-prices a model that already passes every gate.
|
|
1165
1303
|
if (def.required.some((k) => roleMetricValue(em, k, pinRoute, cacheHitRate) == null)) continue;
|
|
1166
|
-
// The
|
|
1167
|
-
// price
|
|
1168
|
-
const
|
|
1169
|
-
if (
|
|
1170
|
-
|
|
1171
|
-
//
|
|
1172
|
-
//
|
|
1173
|
-
//
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
? levels.length > 0 && (def.thinking === undefined || META_LEVELS[def.thinking] === true || levels.includes(def.thinking))
|
|
1179
|
-
: m.thinking;
|
|
1180
|
-
const priceEff = levelFactor === 1 || !runsAtLevel ? basePrice : basePrice * levelFactor;
|
|
1181
|
-
// maxPriceUsdPerM caps the role-priced price (the thinking-adjusted price the
|
|
1182
|
-
// role actually pays — the pinned route's price when pinned); 0 = off.
|
|
1183
|
-
if ((def.filters?.maxPriceUsdPerM ?? 0) > 0 && priceEff > (def.filters?.maxPriceUsdPerM as number)) continue;
|
|
1184
|
-
|
|
1185
|
-
let q = 0;
|
|
1186
|
-
const parts: Record<string, number> = {};
|
|
1187
|
-
for (const [metric, w] of Object.entries(def.weights)) {
|
|
1188
|
-
if (metric === "price") continue; // cost enters as the λ·$ penalty, not the blend
|
|
1189
|
-
const raw = roleMetricValue(em, metric, route, cacheHitRate);
|
|
1190
|
-
if (raw == null) {
|
|
1191
|
-
// Sparse capability metrics are capability-filled (below-median, not 0)
|
|
1192
|
-
// so absence is not a coverage penalty; every other metric contributes
|
|
1193
|
-
// nothing when missing.
|
|
1194
|
-
const fill = CAPABILITY_FILL[metric];
|
|
1195
|
-
if (fill === undefined) continue;
|
|
1196
|
-
const contrib = (w / qW) * fill;
|
|
1197
|
-
parts[metric] = contrib;
|
|
1198
|
-
q += contrib;
|
|
1199
|
-
continue;
|
|
1200
|
-
}
|
|
1201
|
-
const contrib = (w / qW) * cardinalMetric(metric, raw);
|
|
1202
|
-
parts[metric] = contrib;
|
|
1203
|
-
q += contrib;
|
|
1304
|
+
// The role's value on its route basis (the pin's route, the soft best route,
|
|
1305
|
+
// or the 1/price² blend). Null when the model has no usable billed price.
|
|
1306
|
+
const rv = roleValueOnRoute(em, def, route, cacheHitRate, lambda, levelFactor, qW);
|
|
1307
|
+
if (rv === null) continue;
|
|
1308
|
+
// maxPriceUsdPerM caps the model's own price basis, never the soft best-route
|
|
1309
|
+
// basis: with no hard pin the gate uses the model's blend price (thinking-
|
|
1310
|
+
// adjusted), so enabling `preferOwnProvider` cannot change eligibility. A hard
|
|
1311
|
+
// pin keeps gating its route price — the price the request actually pays. 0 = off.
|
|
1312
|
+
const cap = def.filters?.maxPriceUsdPerM ?? 0;
|
|
1313
|
+
if (cap > 0) {
|
|
1314
|
+
const capBase = pinRoute !== null ? routeEffectivePrice(pinRoute, cacheHitRate) : em.price;
|
|
1315
|
+
if (capBase != null && thinkingAdjustedPrice(em, def, capBase) > cap) continue;
|
|
1204
1316
|
}
|
|
1205
|
-
ranked.push({ model: em, value:
|
|
1317
|
+
ranked.push({ model: em, value: rv.value, q: rv.q, priceEff: rv.priceEff, parts: rv.parts });
|
|
1206
1318
|
}
|
|
1207
1319
|
ranked.sort((a, b) => b.value - a.value);
|
|
1208
1320
|
return ranked;
|
package/src/explorer/boot.ts
CHANGED
|
@@ -13,10 +13,13 @@
|
|
|
13
13
|
|
|
14
14
|
import { execFile } from "node:child_process";
|
|
15
15
|
import type { Server } from "node:http";
|
|
16
|
-
import {
|
|
16
|
+
import { join } from "node:path";
|
|
17
|
+
import { enrichThinkingLevels, type CatalogEntry, type KeyAvailability, type Tier } from "../availability.ts";
|
|
17
18
|
import type { RankData } from "../engine.ts";
|
|
18
19
|
import { DEFAULT_ROLES, PLUGIN_SETTINGS_PATH } from "../settings.ts";
|
|
19
20
|
import { agentDir } from "../state.ts";
|
|
21
|
+
import { readUsageRows, type TokenProfile } from "../usage-ledger.ts";
|
|
22
|
+
import type { PoolInputs } from "./pool.ts";
|
|
20
23
|
import { readScopeRoles, resolveScope, resolveScopes, type Scope, type ScopeInputs } from "./scopes.ts";
|
|
21
24
|
import { createExplorerServer } from "./server.ts";
|
|
22
25
|
|
|
@@ -49,6 +52,25 @@ export type ExplorerBootOpts = {
|
|
|
49
52
|
reload(refresh: boolean): Promise<RankData>;
|
|
50
53
|
/** Re-derive the key-availability overlay for `POST /api/refresh` (best-effort). */
|
|
51
54
|
reloadAvailability?: () => Promise<KeyAvailability>;
|
|
55
|
+
/**
|
|
56
|
+
* The as-written pool's inputs (spec #81): the key's tier and the account's
|
|
57
|
+
* allowed-providers whitelist. Both are best-effort — an absent tier reads as
|
|
58
|
+
* `billed` (the ordinary account), and an absent/null whitelist leaves every
|
|
59
|
+
* route in place, so a failed harvest degrades to the raw pool instead of
|
|
60
|
+
* breaking the explorer.
|
|
61
|
+
*/
|
|
62
|
+
tier?: Tier;
|
|
63
|
+
allowedProviders?: ReadonlySet<string> | null;
|
|
64
|
+
/** Re-derive the whitelist for `POST /api/refresh` (best-effort: a throw or a
|
|
65
|
+
* null result degrades the as-written pool to no route pruning). */
|
|
66
|
+
reloadAllowedProviders?: () => Promise<ReadonlySet<string> | null>;
|
|
67
|
+
/** The user-set token profile the per-task price falls back to (the
|
|
68
|
+
* `tokenProfile` setting); absent = unset. */
|
|
69
|
+
tokenProfile?: TokenProfile | null;
|
|
70
|
+
/** Dir holding the user-level usage ledger the per-task price resolves its
|
|
71
|
+
* profile from (default: `agentDir()`, beside the plugin's other state). A
|
|
72
|
+
* project scope reads its own `<root>/.omp` ledger instead. */
|
|
73
|
+
usageDir?: string;
|
|
52
74
|
port?: number;
|
|
53
75
|
/** Open the URL in the default browser (macOS only). */
|
|
54
76
|
open?: boolean;
|
|
@@ -93,6 +115,11 @@ export async function startExplorer(opts: ExplorerBootOpts): Promise<ExplorerHan
|
|
|
93
115
|
const scopeInputs: ScopeInputs = { userLockPath, cwd, registryDir: opts.registryDir };
|
|
94
116
|
let rank = opts.rank;
|
|
95
117
|
let availability = opts.availability;
|
|
118
|
+
// The as-written pool's mutable half: the whitelist is re-derived on refresh,
|
|
119
|
+
// the catalog and tier are boot-time facts. `null` = no pruning.
|
|
120
|
+
let allowed = opts.allowedProviders ?? null;
|
|
121
|
+
const tier = opts.tier ?? "billed";
|
|
122
|
+
const usageDir = opts.usageDir ?? agentDir();
|
|
96
123
|
enrichThinkingLevels(rank.models, opts.catalog);
|
|
97
124
|
|
|
98
125
|
// Scope resolution is boot's job: the user-level scope, the registry's
|
|
@@ -117,7 +144,22 @@ export async function startExplorer(opts: ExplorerBootOpts): Promise<ExplorerHan
|
|
|
117
144
|
rank = await opts.reload(true);
|
|
118
145
|
enrichThinkingLevels(rank.models, opts.catalog);
|
|
119
146
|
if (opts.reloadAvailability) availability = await opts.reloadAvailability();
|
|
147
|
+
// Best-effort: the whitelist is account state, so a failed re-harvest must
|
|
148
|
+
// leave the explorer usable (routes unpruned) rather than fail the refresh.
|
|
149
|
+
if (opts.reloadAllowedProviders) {
|
|
150
|
+
try {
|
|
151
|
+
allowed = await opts.reloadAllowedProviders();
|
|
152
|
+
} catch {
|
|
153
|
+
allowed = null;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
120
156
|
},
|
|
157
|
+
getPoolInputs: (): PoolInputs => ({ catalog: opts.catalog, tier, allowed }),
|
|
158
|
+
// The ledger follows the plugin's state location: a project scope keeps its
|
|
159
|
+
// usage in its own `<root>/.omp`, so its per-task column never mixes with
|
|
160
|
+
// the user-level profile.
|
|
161
|
+
getUsageRows: (scope) => readUsageRows(scope.kind === "project" && scope.root !== undefined ? join(scope.root, ".omp") : usageDir),
|
|
162
|
+
tokenProfile: opts.tokenProfile ?? null,
|
|
121
163
|
});
|
|
122
164
|
// Optional detach (extension only): never hold the host process open on the
|
|
123
165
|
// server's account, so a short-lived `omp -p` run cannot hang.
|
package/src/explorer/explain.ts
CHANGED
|
@@ -15,8 +15,9 @@
|
|
|
15
15
|
import { assessFocusMetric, type FocusMetricAssessment } from "../agent-create.ts";
|
|
16
16
|
import { type KeyAvailability } from "../availability.ts";
|
|
17
17
|
import { cachedSourceInfo, sourceForMetric, type SourceDeclaration } from "../benchmark-sources.ts";
|
|
18
|
-
import { BENCHMARK_CHANCE, CAPABILITY_FILL, cardinalMetric, endpointFilteredModel, paretoFrontier, pinnedRoute, rankRole, roleLambda, roleMetricValue, roleRoute, routeEffectivePrice, type Model, type Ranked, type RoleDef, type SuffixLevel } from "../engine.ts";
|
|
18
|
+
import { autoPinApplies, BENCHMARK_CHANCE, blendOutputPrice, blendRoutePool, CAPABILITY_FILL, cardinalMetric, endpointFilteredModel, paretoFrontier, pinnedRoute, rankRole, roleLambda, roleMetricValue, roleRoute, routeEffectivePrice, routeInputPrice, routeOutputPrice, routePassesEndpointFilters, thinkingAdjustedPrice, type Model, type OpenRouterEndpointRecord, type Ranked, type RoleDef, type SuffixLevel } from "../engine.ts";
|
|
19
19
|
import { KNOWN_METRICS } from "../settings.ts";
|
|
20
|
+
import { type TokenProfile } from "../usage-ledger.ts";
|
|
20
21
|
|
|
21
22
|
// ---------------------------------------------------------------------------
|
|
22
23
|
// Metric metadata (the UI's picklists and formula legends)
|
|
@@ -126,6 +127,79 @@ export function focusAssessments(
|
|
|
126
127
|
return out;
|
|
127
128
|
}
|
|
128
129
|
|
|
130
|
+
// ---------------------------------------------------------------------------
|
|
131
|
+
// Per-task price (display-only)
|
|
132
|
+
// ---------------------------------------------------------------------------
|
|
133
|
+
|
|
134
|
+
/** The per-task price of one model under a role: the role's priced route's input
|
|
135
|
+
* and output prices applied to the role's token profile, with the output term
|
|
136
|
+
* scaled by the role's thinking factor. Display-only — it never feeds the
|
|
137
|
+
* ranking's sort key, `value`, or the explanation's `cost` block. */
|
|
138
|
+
export type PerTaskPrice = {
|
|
139
|
+
inputTokens: number;
|
|
140
|
+
outputTokens: number;
|
|
141
|
+
/** the route's input price, $/M (cache-adjusted by the role's cache-hit rate) */
|
|
142
|
+
inputPrice: number;
|
|
143
|
+
/** the route's output price, $/M (derived from the billed blend and the input price) */
|
|
144
|
+
outputPrice: number;
|
|
145
|
+
/** the role's thinking factor (1 for a bare role) */
|
|
146
|
+
thinkingFactor: number;
|
|
147
|
+
/** dollars per task */
|
|
148
|
+
total: number;
|
|
149
|
+
};
|
|
150
|
+
|
|
151
|
+
/** The model's blended input price: the 1/weightPrice²-weighted mean of the
|
|
152
|
+
* routes' cache-adjusted input prices (`routeInputPrice`) — the same weighting
|
|
153
|
+
* `blendRoutePool` uses (the router's sort key), so price each route at its input
|
|
154
|
+
* price and read the blend back. The cache-hit rate must reach this term too, or
|
|
155
|
+
* `p_out` would mix a cache-adjusted billed blend with an unadjusted input price. */
|
|
156
|
+
function blendedInputPrice(routes: readonly OpenRouterEndpointRecord[], cacheHitRate: number): number | null {
|
|
157
|
+
const priced = routes.map((r) => ({ ...r, price: routeInputPrice(r, cacheHitRate) }));
|
|
158
|
+
return blendRoutePool(priced, 0).price;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/** The per-task price of `model` under `def` for a token profile: `p_in` is the
|
|
162
|
+
* role's priced route's input price (cache-adjusted), `p_out` its output price
|
|
163
|
+
* (the inverse 3:1 blend of the same route, `routeOutputPrice`), and the output
|
|
164
|
+
* term is scaled by the role's thinking factor. When the role prices on the
|
|
165
|
+
* 1/price² blend (`roleRoute` returns null) the model's billed blend and the input
|
|
166
|
+
* blend over the routes that survive the role's endpoint filters stand in — the
|
|
167
|
+
* same pool the ranking prices on, so `p_in` and the billed blend never mix two
|
|
168
|
+
* route pools. `null` when there is no usable route price (absent, never a silent
|
|
169
|
+
* zero). */
|
|
170
|
+
export function perTaskPrice(model: Model, def: RoleDef, profile: TokenProfile): PerTaskPrice | null {
|
|
171
|
+
const cacheHitRate = def.cacheHitRate ?? 0;
|
|
172
|
+
// A role with endpoint filters prices on the routes that survive them; a model
|
|
173
|
+
// whose every route fails is ineligible (the ranking drops it), so its per-task
|
|
174
|
+
// price is absent, not the unfiltered blend.
|
|
175
|
+
const em = endpointFilteredModel(model, def.filters, cacheHitRate);
|
|
176
|
+
if (em === null) return null;
|
|
177
|
+
const route = roleRoute(model, def);
|
|
178
|
+
let inputPrice: number | null;
|
|
179
|
+
let outputPrice: number | null;
|
|
180
|
+
if (route !== null) {
|
|
181
|
+
// The role prices this model on one route: that route's own input and output
|
|
182
|
+
// prices, the same pair the billed blend comes from.
|
|
183
|
+
inputPrice = routeInputPrice(route, cacheHitRate);
|
|
184
|
+
outputPrice = routeOutputPrice(route, cacheHitRate);
|
|
185
|
+
} else {
|
|
186
|
+
// The 1/price² blend case: the billed blend `em` is priced on and the input
|
|
187
|
+
// blend over exactly the routes it blended — the endpoint-filtered pool, never
|
|
188
|
+
// the model's full route list (a dropped route's input price belongs to no
|
|
189
|
+
// route's output price).
|
|
190
|
+
const pool = (model.routes ?? []).filter((r) => routePassesEndpointFilters(r, def.filters));
|
|
191
|
+
inputPrice = blendedInputPrice(pool, cacheHitRate);
|
|
192
|
+
outputPrice = inputPrice === null || em.price === null ? null : blendOutputPrice(em.price, inputPrice);
|
|
193
|
+
}
|
|
194
|
+
if (inputPrice === null || outputPrice === null) return null;
|
|
195
|
+
// The same gated factor the ranking applies: `thinkingAdjustedPrice(model, def, 1)`
|
|
196
|
+
// is 1 unless the model actually runs at the role's level, so the per-task price
|
|
197
|
+
// and the row's `priceEff` can never disagree for a non-thinking model.
|
|
198
|
+
const thinkingFactor = thinkingAdjustedPrice(model, def, 1);
|
|
199
|
+
const total = (profile.inputTokens * inputPrice + profile.outputTokens * outputPrice * thinkingFactor) / 1e6;
|
|
200
|
+
return { inputTokens: profile.inputTokens, outputTokens: profile.outputTokens, inputPrice, outputPrice, thinkingFactor, total };
|
|
201
|
+
}
|
|
202
|
+
|
|
129
203
|
// ---------------------------------------------------------------------------
|
|
130
204
|
// Rank rows (table data with baseline deltas)
|
|
131
205
|
// ---------------------------------------------------------------------------
|
|
@@ -156,6 +230,9 @@ export type RankRow = {
|
|
|
156
230
|
baselineRank: number | null;
|
|
157
231
|
/** baselineRank − rank; positive = moved up */
|
|
158
232
|
delta: number | null;
|
|
233
|
+
/** dollars per task under the role's token profile, or null when no profile
|
|
234
|
+
* was supplied or the model has no usable route price. Display-only. */
|
|
235
|
+
perTask: number | null;
|
|
159
236
|
};
|
|
160
237
|
|
|
161
238
|
/** The availability overlay verdict for one model id: `"usable"` when the keyed
|
|
@@ -172,8 +249,9 @@ function keyVerdict(availability: KeyAvailability | undefined, id: string): "usa
|
|
|
172
249
|
/** Rank `models` under `def`, annotating each row with its rank delta against
|
|
173
250
|
* `baseline` (the effective role's ranking, i.e. what the plugin does today) and
|
|
174
251
|
* its keyed-catalog availability overlay. Ranking order, deltas, and every other
|
|
175
|
-
* field are unaffected by `availability
|
|
176
|
-
|
|
252
|
+
* field are unaffected by `availability` or `profile`; `profile` only fills the
|
|
253
|
+
* display-only `perTask` column. */
|
|
254
|
+
export function rankRows(def: RoleDef, models: Model[], baseline: Ranked[], availability?: KeyAvailability, profile?: TokenProfile | null): RankRow[] {
|
|
177
255
|
const ranked = rankRole(def, models);
|
|
178
256
|
const frontier = paretoFrontier(ranked);
|
|
179
257
|
const baselineRank = new Map<string, number>();
|
|
@@ -200,6 +278,7 @@ export function rankRows(def: RoleDef, models: Model[], baseline: Ranked[], avai
|
|
|
200
278
|
rank,
|
|
201
279
|
baselineRank: b,
|
|
202
280
|
delta: b == null ? null : b - rank,
|
|
281
|
+
perTask: profile == null ? null : perTaskPrice(r.model, def, profile)?.total ?? null,
|
|
203
282
|
};
|
|
204
283
|
});
|
|
205
284
|
}
|
|
@@ -264,13 +343,21 @@ export type Explanation =
|
|
|
264
343
|
value: number;
|
|
265
344
|
q: number;
|
|
266
345
|
priceEff: number;
|
|
267
|
-
/** true when the model was priced on
|
|
346
|
+
/** true when the model was priced on the soft best route (the
|
|
268
347
|
* `preferOwnProvider` preference, no hard pin) rather than the 1/price²
|
|
269
348
|
* blend — the panel shows WHY the price is what it is. */
|
|
270
349
|
ownProvider: boolean;
|
|
350
|
+
/** the automatic pin: the model's best route's provider slug when the role
|
|
351
|
+
* prefers a bound route and carries no manual pin, else null. Read-only in
|
|
352
|
+
* the UI; a manual `providerPin` overrides it. */
|
|
353
|
+
autoPin: string | null;
|
|
271
354
|
role: { name: string; lambda: number; derivedLambda: number; wPrice: number; qW: number; thinking: SuffixLevel | undefined; cacheHitRate: number };
|
|
272
355
|
contributions: Contribution[];
|
|
273
356
|
cost: { priceEff: number; billedPrice: number; lambda: number; penalty: number; q: number; value: number };
|
|
357
|
+
/** the per-task price breakdown under the role's token profile, or null
|
|
358
|
+
* when no profile was supplied or the model has no usable route price.
|
|
359
|
+
* Display-only — never part of `cost` or the ranking. */
|
|
360
|
+
perTask: PerTaskPrice | null;
|
|
274
361
|
gapAbove: number | null;
|
|
275
362
|
gapToTop: number;
|
|
276
363
|
above: { id: string; name: string; value: number } | null;
|
|
@@ -280,8 +367,9 @@ export type Explanation =
|
|
|
280
367
|
|
|
281
368
|
/** Full decomposition of one model's rank for a role, or the eligibility gates it
|
|
282
369
|
* failed (same gates `rankRole` applies, listed in the same order). The eligible
|
|
283
|
-
* branch also carries the keyed-catalog availability overlay (`key`/`keyReason`)
|
|
284
|
-
|
|
370
|
+
* branch also carries the keyed-catalog availability overlay (`key`/`keyReason`)
|
|
371
|
+
* and, when a token `profile` is supplied, the display-only per-task price. */
|
|
372
|
+
export function explainModel(def: RoleDef, models: Model[], modelId: string, roleName = "", availability?: KeyAvailability, profile?: TokenProfile | null): Explanation {
|
|
285
373
|
const model = models.find((m) => m.id === modelId);
|
|
286
374
|
if (!model) return { eligible: false, reasons: ["model not found in today's dataset"] };
|
|
287
375
|
|
|
@@ -294,18 +382,31 @@ export function explainModel(def: RoleDef, models: Model[], modelId: string, rol
|
|
|
294
382
|
const cacheHitRate = def.cacheHitRate ?? 0;
|
|
295
383
|
const em = endpointFilteredModel(model, def.filters, cacheHitRate);
|
|
296
384
|
const eff = em ?? model;
|
|
297
|
-
// Soft
|
|
298
|
-
// prefers
|
|
299
|
-
// else on the blend. Never a gate — a model with no
|
|
385
|
+
// Soft best-route preference (mirrors rankRole): with no hard pin, a role that
|
|
386
|
+
// prefers a bound route is priced on the model's best route by the role's own
|
|
387
|
+
// value, else on the blend. Never a gate — a model with no candidate route keeps
|
|
388
|
+
// the blend.
|
|
300
389
|
const route = roleRoute(model, def);
|
|
301
|
-
// The model is priced on
|
|
390
|
+
// The model is priced on the soft best route (not a hard pin, not the blend).
|
|
302
391
|
const ownProvider = pinRoute === null && route !== null;
|
|
392
|
+
// The automatic pin the updater would write: the model's best route's slug,
|
|
393
|
+
// only when the role prefers a bound route and carries no manual pin. Reuses the
|
|
394
|
+
// already-computed `route` (the same best route) instead of recomputing it.
|
|
395
|
+
const autoPin = autoPinApplies(def) ? route?.providerSlug ?? null : null;
|
|
303
396
|
const missingRequired = def.required.filter((k) => roleMetricValue(eff, k, pinRoute, cacheHitRate) == null);
|
|
304
397
|
if (missingRequired.length > 0) reasons.push(`missing required metric(s): ${missingRequired.join(", ")}`);
|
|
305
398
|
if (def.filters?.image && !model.multimodal) reasons.push("role requires image input; model is text-only");
|
|
306
399
|
if (def.providerPin !== undefined && pinRoute === null) reasons.push("no route matches the role's provider pin");
|
|
307
400
|
if (em === null) reasons.push("no standard-tier route satisfies the role's endpoint filters");
|
|
308
401
|
if (eff.price == null) reasons.push("no billed OpenRouter route");
|
|
402
|
+
// maxPriceUsdPerM caps the model's own price basis, never the soft best-route
|
|
403
|
+
// basis (mirrors rankRole): with no hard pin the gate uses the blend price, so
|
|
404
|
+
// the soft basis cannot change eligibility; a hard pin gates its route price.
|
|
405
|
+
const cap = def.filters?.maxPriceUsdPerM ?? 0;
|
|
406
|
+
if (cap > 0) {
|
|
407
|
+
const capBase = pinRoute !== null ? routeEffectivePrice(pinRoute, cacheHitRate) : eff.price;
|
|
408
|
+
if (capBase != null && thinkingAdjustedPrice(eff, def, capBase) > cap) reasons.push(`price exceeds the role's maxPriceUsdPerM cap (${cap})`);
|
|
409
|
+
}
|
|
309
410
|
if (reasons.length > 0) return { eligible: false, reasons };
|
|
310
411
|
|
|
311
412
|
const ranked = rankRole(def, models);
|
|
@@ -403,9 +504,11 @@ export function explainModel(def: RoleDef, models: Model[], modelId: string, rol
|
|
|
403
504
|
q: self.q,
|
|
404
505
|
priceEff: self.priceEff,
|
|
405
506
|
ownProvider,
|
|
507
|
+
autoPin,
|
|
406
508
|
role: { name: roleName, lambda, derivedLambda, wPrice, qW, thinking: def.thinking, cacheHitRate },
|
|
407
509
|
contributions,
|
|
408
510
|
cost: { priceEff: self.priceEff, billedPrice: route !== null ? routeEffectivePrice(route, cacheHitRate) ?? eff.price : eff.price, lambda, penalty: lambda * self.priceEff, q: self.q, value: self.value },
|
|
511
|
+
perTask: profile == null ? null : perTaskPrice(model, def, profile),
|
|
409
512
|
gapAbove,
|
|
410
513
|
gapToTop,
|
|
411
514
|
above,
|