omp-llm-role 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -8
- package/docs/dev/architecture.md +25 -4
- package/docs/dev/ci.md +12 -1
- package/docs/dev/current-state.md +63 -26
- package/docs/dev/data-sources.md +33 -4
- package/docs/dev/explorer.md +22 -4
- package/docs/dev/quirks.md +32 -0
- package/docs/dev/releasing.md +31 -30
- package/docs/dev/scoring.md +98 -35
- package/docs/dev/testing.md +70 -23
- package/docs/dev/writes-and-state.md +56 -0
- package/docs/llm-role-rankings.md +127 -127
- package/package.json +82 -22
- package/src/availability.ts +87 -9
- package/src/cli/create-role.ts +1 -1
- package/src/engine.ts +157 -76
- package/src/explorer/explain.ts +24 -6
- package/src/explorer/server.ts +15 -2
- package/src/extension.ts +58 -15
- package/src/features.ts +9 -7
- package/src/provider-display.ts +67 -0
- package/src/settings.ts +8 -2
- package/src/updater.ts +91 -13
- package/web/app.js +24 -1
package/src/engine.ts
CHANGED
|
@@ -181,6 +181,9 @@ export type RoleDef = {
|
|
|
181
181
|
minOutputTokens?: number;
|
|
182
182
|
/** drop the model when its role-priced blend exceeds this ($/M); 0 = off */
|
|
183
183
|
maxPriceUsdPerM?: number;
|
|
184
|
+
/** soft throughput floor (tok/s); 0/absent = off. A route at or above the
|
|
185
|
+
* floor sorts before a route below it, but is never dropped. */
|
|
186
|
+
preferredMinThroughput?: number;
|
|
184
187
|
};
|
|
185
188
|
/** explicit λ override ($ per quality point); default derives from the price weight */
|
|
186
189
|
lambda?: number;
|
|
@@ -205,9 +208,9 @@ export type RoleDef = {
|
|
|
205
208
|
/** Opt-in capability flags (`roles.<role>.features.<id>`); expanded by
|
|
206
209
|
* `expandFeatures` before ranking. `true`/`false` overrides the global flag. */
|
|
207
210
|
features?: Record<string, boolean>;
|
|
208
|
-
/** Soft provider preference (issue #40): price a model on its
|
|
209
|
-
* when the role's `providerPinning` feature
|
|
210
|
-
* Unlike `providerPin` this NEVER drops a model. */
|
|
211
|
+
/** Soft provider preference (issue #40, re-based by #58): price a model on its
|
|
212
|
+
* best route by the role's own value when the role's `providerPinning` feature
|
|
213
|
+
* is on, else the 1/price² blend. Unlike `providerPin` this NEVER drops a model. */
|
|
211
214
|
preferOwnProvider?: boolean;
|
|
212
215
|
/** Locked role: the plugin still ranks it (so it stays in the universe and its
|
|
213
216
|
* sources are fetched) but never rewrites its `modelRoles` selector or fallback
|
|
@@ -1060,39 +1063,144 @@ export function pinnedRoute(model: Model, providerPin: string | undefined): Open
|
|
|
1060
1063
|
return (model.routes ?? []).find((r) => r.providerSlug === providerPin) ?? null;
|
|
1061
1064
|
}
|
|
1062
1065
|
|
|
1063
|
-
/** The
|
|
1064
|
-
*
|
|
1065
|
-
*
|
|
1066
|
-
*
|
|
1067
|
-
*
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
const
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1066
|
+
/** The role's thinking-adjusted billed price for a model at `basePrice`: the
|
|
1067
|
+
* thinking factor applies only when the model actually runs the role's level
|
|
1068
|
+
* (catalog `thinkingLevels` wins; without it the `supports_reasoning` flag
|
|
1069
|
+
* gates). Mirrors the factor `rankRole` applies, so the route basis and the
|
|
1070
|
+
* ranking can never disagree. */
|
|
1071
|
+
export function thinkingAdjustedPrice(model: Model, def: RoleDef, basePrice: number): number {
|
|
1072
|
+
const levelFactor = def.thinking === undefined ? 1 : thinkingPriceFactor(def.thinking);
|
|
1073
|
+
if (levelFactor === 1) return basePrice;
|
|
1074
|
+
const levels = model.thinkingLevels;
|
|
1075
|
+
const runsAtLevel = levels
|
|
1076
|
+
? levels.length > 0 && (def.thinking === undefined || META_LEVELS[def.thinking] === true || levels.includes(def.thinking))
|
|
1077
|
+
: model.thinking;
|
|
1078
|
+
return runsAtLevel ? basePrice * levelFactor : basePrice;
|
|
1079
|
+
}
|
|
1080
|
+
|
|
1081
|
+
/** The role's value of a model priced on `route` (null = the 1/price² blend):
|
|
1082
|
+
* the price-free quality composite q over the model's metrics with the route's
|
|
1083
|
+
* p50 throughput and cache-adjusted billed price, minus λ·priceEff. Returns null
|
|
1084
|
+
* when the model has no usable billed price. */
|
|
1085
|
+
function roleValueOnRoute(
|
|
1086
|
+
em: Model,
|
|
1087
|
+
def: RoleDef,
|
|
1088
|
+
route: OpenRouterEndpointRecord | null,
|
|
1089
|
+
cacheHitRate: number,
|
|
1090
|
+
lambda: number,
|
|
1091
|
+
levelFactor: number,
|
|
1092
|
+
qW: number,
|
|
1093
|
+
): { q: number; priceEff: number; value: number; parts: Record<string, number> } | null {
|
|
1094
|
+
const basePrice = route !== null ? routeEffectivePrice(route, cacheHitRate) : em.price;
|
|
1095
|
+
if (basePrice == null) return null;
|
|
1096
|
+
const priceEff = levelFactor === 1 ? basePrice : thinkingAdjustedPrice(em, def, basePrice);
|
|
1097
|
+
let q = 0;
|
|
1098
|
+
const parts: Record<string, number> = {};
|
|
1099
|
+
for (const [metric, w] of Object.entries(def.weights)) {
|
|
1100
|
+
if (metric === "price") continue; // cost enters as the λ·$ penalty, not the blend
|
|
1101
|
+
const raw = roleMetricValue(em, metric, route, cacheHitRate);
|
|
1102
|
+
if (raw == null) {
|
|
1103
|
+
// Sparse capability metrics are capability-filled (below-median, not 0)
|
|
1104
|
+
// so absence is not a coverage penalty; every other metric contributes
|
|
1105
|
+
// nothing when missing.
|
|
1106
|
+
const fill = CAPABILITY_FILL[metric];
|
|
1107
|
+
if (fill === undefined) continue;
|
|
1108
|
+
const contrib = (w / qW) * fill;
|
|
1109
|
+
parts[metric] = contrib;
|
|
1110
|
+
q += contrib;
|
|
1111
|
+
continue;
|
|
1112
|
+
}
|
|
1113
|
+
const contrib = (w / qW) * cardinalMetric(metric, raw);
|
|
1114
|
+
parts[metric] = contrib;
|
|
1115
|
+
q += contrib;
|
|
1116
|
+
}
|
|
1117
|
+
return { q, priceEff, value: q - lambda * priceEff, parts };
|
|
1118
|
+
}
|
|
1119
|
+
|
|
1120
|
+
/** The model's routes that are candidates for the role's soft best-route basis,
|
|
1121
|
+
* sorted by the value-for-money score desc (ties: providerSlug asc, id asc). A
|
|
1122
|
+
* candidate is non-degraded (status === 0), passes
|
|
1123
|
+
* `routePassesEndpointFilters(route, def.filters)`, has a usable billed price,
|
|
1124
|
+
* and (when `filters.maxPriceUsdPerM > 0`) a thinking-adjusted price at or below
|
|
1125
|
+
* the cap.
|
|
1126
|
+
*
|
|
1127
|
+
* The score is `vfm = wT·tNorm − wP·pNorm`, where `tNorm`/`pNorm` are the
|
|
1128
|
+
* route's p50 throughput and cache-adjusted billed price min/max-normalized over
|
|
1129
|
+
* the candidate set, and `wT = wTput/(wPrice+wTput)`, `wP = wPrice/(wPrice+wTput)`.
|
|
1130
|
+
* A route with no p50 uses the model's blended throughput (an absent p50 is
|
|
1131
|
+
* unknown, not slow). `filters.preferredMinThroughput` is a SOFT floor: routes at
|
|
1132
|
+
* or above it sort before routes below it, but are never dropped. */
|
|
1133
|
+
export function rankedRoutes(model: Model, def: RoleDef): OpenRouterEndpointRecord[] {
|
|
1134
|
+
const cacheHitRate = def.cacheHitRate ?? 0;
|
|
1135
|
+
const cap = def.filters?.maxPriceUsdPerM ?? 0;
|
|
1136
|
+
const floor = def.filters?.preferredMinThroughput ?? 0;
|
|
1137
|
+
const out = (model.routes ?? []).filter((r) => {
|
|
1138
|
+
if (r.status !== 0) return false;
|
|
1139
|
+
if (!routePassesEndpointFilters(r, def.filters)) return false;
|
|
1140
|
+
const p = routeEffectivePrice(r, cacheHitRate);
|
|
1141
|
+
if (p === null || p <= 0) return false;
|
|
1142
|
+
if (cap > 0 && thinkingAdjustedPrice(model, def, p) > cap) return false;
|
|
1143
|
+
return true;
|
|
1144
|
+
});
|
|
1145
|
+
if (out.length === 0) return [];
|
|
1146
|
+
// A route with no p50 falls back to the model's blended throughput over the
|
|
1147
|
+
// endpoint-filtered pool — the same basis `roleMetricValue` used before.
|
|
1148
|
+
const em = endpointFilteredModel(model, def.filters, cacheHitRate) ?? model;
|
|
1149
|
+
const wTput = def.weights.throughput ?? 0;
|
|
1150
|
+
const wPrice = def.weights.price ?? 0;
|
|
1151
|
+
const wSum = wTput + wPrice;
|
|
1152
|
+
const wT = wSum > 0 ? wTput / wSum : 0;
|
|
1153
|
+
const wP = wSum > 0 ? wPrice / wSum : 0;
|
|
1154
|
+
// Decorate-sort-undecorate: the throughput/price lookups are not free, so
|
|
1155
|
+
// compute each route's terms once.
|
|
1156
|
+
const decorated = out.map((route) => ({
|
|
1157
|
+
route,
|
|
1158
|
+
tput: roleMetricValue(em, "throughput", route, cacheHitRate) ?? 0,
|
|
1159
|
+
priceEff: routeEffectivePrice(route, cacheHitRate) as number,
|
|
1160
|
+
}));
|
|
1161
|
+
const tMin = Math.min(...decorated.map((d) => d.tput));
|
|
1162
|
+
const tMax = Math.max(...decorated.map((d) => d.tput));
|
|
1163
|
+
const pMin = Math.min(...decorated.map((d) => d.priceEff));
|
|
1164
|
+
const pMax = Math.max(...decorated.map((d) => d.priceEff));
|
|
1165
|
+
const tSpan = tMax - tMin;
|
|
1166
|
+
const pSpan = pMax - pMin;
|
|
1167
|
+
return decorated
|
|
1168
|
+
.map((d) => {
|
|
1169
|
+
const tNorm = tSpan > 0 ? (d.tput - tMin) / tSpan : 0;
|
|
1170
|
+
const pNorm = pSpan > 0 ? (d.priceEff - pMin) / pSpan : 0;
|
|
1171
|
+
return { route: d.route, score: wT * tNorm - wP * pNorm, aboveFloor: floor <= 0 || d.tput >= floor };
|
|
1172
|
+
})
|
|
1173
|
+
.sort((a, b) => {
|
|
1174
|
+
if (a.aboveFloor !== b.aboveFloor) return a.aboveFloor ? -1 : 1;
|
|
1175
|
+
if (a.score !== b.score) return b.score - a.score;
|
|
1176
|
+
if (a.route.providerSlug !== b.route.providerSlug) return a.route.providerSlug < b.route.providerSlug ? -1 : 1;
|
|
1177
|
+
return a.route.id < b.route.id ? -1 : a.route.id > b.route.id ? 1 : 0;
|
|
1178
|
+
})
|
|
1179
|
+
.map((d) => d.route);
|
|
1180
|
+
}
|
|
1181
|
+
|
|
1182
|
+
/** `rankedRoutes(model, def)[0]` or null. The auto-pin slug and the soft basis. */
|
|
1183
|
+
export function bestRoute(model: Model, def: RoleDef): OpenRouterEndpointRecord | null {
|
|
1184
|
+
return rankedRoutes(model, def)[0] ?? null;
|
|
1185
|
+
}
|
|
1186
|
+
|
|
1187
|
+
/** True when the role's soft best-route basis applies: the `preferOwnProvider`
|
|
1188
|
+
* capability is on and no manual `providerPin` overrides it. The soft basis the
|
|
1189
|
+
* engine prices on and the auto-pin the updater writes share this predicate. */
|
|
1190
|
+
export function autoPinApplies(def: RoleDef): boolean {
|
|
1191
|
+
return def.providerPin === undefined && def.preferOwnProvider === true;
|
|
1082
1192
|
}
|
|
1083
1193
|
|
|
1084
1194
|
/** The route a role prices a model on: the hard pin's route when the role is
|
|
1085
|
-
* pinned, else the model's
|
|
1086
|
-
* preference is on
|
|
1087
|
-
*
|
|
1088
|
-
*
|
|
1195
|
+
* pinned, else the model's best route by the role's own value when the soft
|
|
1196
|
+
* `preferOwnProvider` preference is on, else null (the 1/price² blend). A
|
|
1197
|
+
* preference never drops a model — a model with no candidate route keeps the
|
|
1198
|
+
* blend. */
|
|
1089
1199
|
export function roleRoute(model: Model, def: RoleDef): OpenRouterEndpointRecord | null {
|
|
1090
1200
|
const pin = pinnedRoute(model, def.providerPin);
|
|
1091
1201
|
if (pin !== null) return pin;
|
|
1092
|
-
if (def
|
|
1093
|
-
|
|
1094
|
-
if (own === null) return null;
|
|
1095
|
-
return routeEffectivePrice(own, def.cacheHitRate ?? 0) !== null ? own : null;
|
|
1202
|
+
if (!autoPinApplies(def)) return null;
|
|
1203
|
+
return bestRoute(model, def);
|
|
1096
1204
|
}
|
|
1097
1205
|
|
|
1098
1206
|
/** The metric value a role sees for a model. A pinned role's throughput is its
|
|
@@ -1142,12 +1250,11 @@ export function rankRole(def: RoleDef, models: Model[]): Ranked[] {
|
|
|
1142
1250
|
// the pin and is ineligible (recorded by `providerPinDrops`).
|
|
1143
1251
|
const pinRoute = pinnedRoute(m, def.providerPin);
|
|
1144
1252
|
if (def.providerPin !== undefined && pinRoute === null) continue;
|
|
1145
|
-
// Soft
|
|
1146
|
-
//
|
|
1147
|
-
//
|
|
1148
|
-
//
|
|
1149
|
-
//
|
|
1150
|
-
// count is unchanged. `providerPinDrops`/`pinUnmatched` key on
|
|
1253
|
+
// Soft best-route preference (issue #40, re-based by #58): with no hard pin,
|
|
1254
|
+
// a role that prefers a bound route is priced on the model's best route by
|
|
1255
|
+
// the role's own value, else on the 1/price² blend. This is a preference,
|
|
1256
|
+
// never a gate: a model with no candidate route keeps the blend, so the
|
|
1257
|
+
// eligible count is unchanged. `providerPinDrops`/`pinUnmatched` key on
|
|
1151
1258
|
// `def.providerPin` only, so the preference never drops a model.
|
|
1152
1259
|
const route = roleRoute(m, def);
|
|
1153
1260
|
// Endpoint capability gate: a role that declares endpoint filters is priced on
|
|
@@ -1160,49 +1267,23 @@ export function rankRole(def: RoleDef, models: Model[]): Ranked[] {
|
|
|
1160
1267
|
const em = endpointFilteredModel(m, def.filters, cacheHitRate);
|
|
1161
1268
|
if (em === null) continue;
|
|
1162
1269
|
// The required-metric gate keys on the hard pin only (null when unpinned), so
|
|
1163
|
-
// the soft
|
|
1270
|
+
// the soft best-route preference can never change eligibility — it only
|
|
1164
1271
|
// re-prices a model that already passes every gate.
|
|
1165
1272
|
if (def.required.some((k) => roleMetricValue(em, k, pinRoute, cacheHitRate) == null)) continue;
|
|
1166
|
-
// The
|
|
1167
|
-
// price
|
|
1168
|
-
const
|
|
1169
|
-
if (
|
|
1170
|
-
|
|
1171
|
-
//
|
|
1172
|
-
//
|
|
1173
|
-
//
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
? levels.length > 0 && (def.thinking === undefined || META_LEVELS[def.thinking] === true || levels.includes(def.thinking))
|
|
1179
|
-
: m.thinking;
|
|
1180
|
-
const priceEff = levelFactor === 1 || !runsAtLevel ? basePrice : basePrice * levelFactor;
|
|
1181
|
-
// maxPriceUsdPerM caps the role-priced price (the thinking-adjusted price the
|
|
1182
|
-
// role actually pays — the pinned route's price when pinned); 0 = off.
|
|
1183
|
-
if ((def.filters?.maxPriceUsdPerM ?? 0) > 0 && priceEff > (def.filters?.maxPriceUsdPerM as number)) continue;
|
|
1184
|
-
|
|
1185
|
-
let q = 0;
|
|
1186
|
-
const parts: Record<string, number> = {};
|
|
1187
|
-
for (const [metric, w] of Object.entries(def.weights)) {
|
|
1188
|
-
if (metric === "price") continue; // cost enters as the λ·$ penalty, not the blend
|
|
1189
|
-
const raw = roleMetricValue(em, metric, route, cacheHitRate);
|
|
1190
|
-
if (raw == null) {
|
|
1191
|
-
// Sparse capability metrics are capability-filled (below-median, not 0)
|
|
1192
|
-
// so absence is not a coverage penalty; every other metric contributes
|
|
1193
|
-
// nothing when missing.
|
|
1194
|
-
const fill = CAPABILITY_FILL[metric];
|
|
1195
|
-
if (fill === undefined) continue;
|
|
1196
|
-
const contrib = (w / qW) * fill;
|
|
1197
|
-
parts[metric] = contrib;
|
|
1198
|
-
q += contrib;
|
|
1199
|
-
continue;
|
|
1200
|
-
}
|
|
1201
|
-
const contrib = (w / qW) * cardinalMetric(metric, raw);
|
|
1202
|
-
parts[metric] = contrib;
|
|
1203
|
-
q += contrib;
|
|
1273
|
+
// The role's value on its route basis (the pin's route, the soft best route,
|
|
1274
|
+
// or the 1/price² blend). Null when the model has no usable billed price.
|
|
1275
|
+
const rv = roleValueOnRoute(em, def, route, cacheHitRate, lambda, levelFactor, qW);
|
|
1276
|
+
if (rv === null) continue;
|
|
1277
|
+
// maxPriceUsdPerM caps the model's own price basis, never the soft best-route
|
|
1278
|
+
// basis: with no hard pin the gate uses the model's blend price (thinking-
|
|
1279
|
+
// adjusted), so enabling `preferOwnProvider` cannot change eligibility. A hard
|
|
1280
|
+
// pin keeps gating its route price — the price the request actually pays. 0 = off.
|
|
1281
|
+
const cap = def.filters?.maxPriceUsdPerM ?? 0;
|
|
1282
|
+
if (cap > 0) {
|
|
1283
|
+
const capBase = pinRoute !== null ? routeEffectivePrice(pinRoute, cacheHitRate) : em.price;
|
|
1284
|
+
if (capBase != null && thinkingAdjustedPrice(em, def, capBase) > cap) continue;
|
|
1204
1285
|
}
|
|
1205
|
-
ranked.push({ model: em, value:
|
|
1286
|
+
ranked.push({ model: em, value: rv.value, q: rv.q, priceEff: rv.priceEff, parts: rv.parts });
|
|
1206
1287
|
}
|
|
1207
1288
|
ranked.sort((a, b) => b.value - a.value);
|
|
1208
1289
|
return ranked;
|
package/src/explorer/explain.ts
CHANGED
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
import { assessFocusMetric, type FocusMetricAssessment } from "../agent-create.ts";
|
|
16
16
|
import { type KeyAvailability } from "../availability.ts";
|
|
17
17
|
import { cachedSourceInfo, sourceForMetric, type SourceDeclaration } from "../benchmark-sources.ts";
|
|
18
|
-
import { BENCHMARK_CHANCE, CAPABILITY_FILL, cardinalMetric, endpointFilteredModel, paretoFrontier, pinnedRoute, rankRole, roleLambda, roleMetricValue, roleRoute, routeEffectivePrice, type Model, type Ranked, type RoleDef, type SuffixLevel } from "../engine.ts";
|
|
18
|
+
import { autoPinApplies, BENCHMARK_CHANCE, CAPABILITY_FILL, cardinalMetric, endpointFilteredModel, paretoFrontier, pinnedRoute, rankRole, roleLambda, roleMetricValue, roleRoute, routeEffectivePrice, thinkingAdjustedPrice, type Model, type Ranked, type RoleDef, type SuffixLevel } from "../engine.ts";
|
|
19
19
|
import { KNOWN_METRICS } from "../settings.ts";
|
|
20
20
|
|
|
21
21
|
// ---------------------------------------------------------------------------
|
|
@@ -264,10 +264,14 @@ export type Explanation =
|
|
|
264
264
|
value: number;
|
|
265
265
|
q: number;
|
|
266
266
|
priceEff: number;
|
|
267
|
-
/** true when the model was priced on
|
|
267
|
+
/** true when the model was priced on the soft best route (the
|
|
268
268
|
* `preferOwnProvider` preference, no hard pin) rather than the 1/price²
|
|
269
269
|
* blend — the panel shows WHY the price is what it is. */
|
|
270
270
|
ownProvider: boolean;
|
|
271
|
+
/** the automatic pin: the model's best route's provider slug when the role
|
|
272
|
+
* prefers a bound route and carries no manual pin, else null. Read-only in
|
|
273
|
+
* the UI; a manual `providerPin` overrides it. */
|
|
274
|
+
autoPin: string | null;
|
|
271
275
|
role: { name: string; lambda: number; derivedLambda: number; wPrice: number; qW: number; thinking: SuffixLevel | undefined; cacheHitRate: number };
|
|
272
276
|
contributions: Contribution[];
|
|
273
277
|
cost: { priceEff: number; billedPrice: number; lambda: number; penalty: number; q: number; value: number };
|
|
@@ -294,18 +298,31 @@ export function explainModel(def: RoleDef, models: Model[], modelId: string, rol
|
|
|
294
298
|
const cacheHitRate = def.cacheHitRate ?? 0;
|
|
295
299
|
const em = endpointFilteredModel(model, def.filters, cacheHitRate);
|
|
296
300
|
const eff = em ?? model;
|
|
297
|
-
// Soft
|
|
298
|
-
// prefers
|
|
299
|
-
// else on the blend. Never a gate — a model with no
|
|
301
|
+
// Soft best-route preference (mirrors rankRole): with no hard pin, a role that
|
|
302
|
+
// prefers a bound route is priced on the model's best route by the role's own
|
|
303
|
+
// value, else on the blend. Never a gate — a model with no candidate route keeps
|
|
304
|
+
// the blend.
|
|
300
305
|
const route = roleRoute(model, def);
|
|
301
|
-
// The model is priced on
|
|
306
|
+
// The model is priced on the soft best route (not a hard pin, not the blend).
|
|
302
307
|
const ownProvider = pinRoute === null && route !== null;
|
|
308
|
+
// The automatic pin the updater would write: the model's best route's slug,
|
|
309
|
+
// only when the role prefers a bound route and carries no manual pin. Reuses the
|
|
310
|
+
// already-computed `route` (the same best route) instead of recomputing it.
|
|
311
|
+
const autoPin = autoPinApplies(def) ? route?.providerSlug ?? null : null;
|
|
303
312
|
const missingRequired = def.required.filter((k) => roleMetricValue(eff, k, pinRoute, cacheHitRate) == null);
|
|
304
313
|
if (missingRequired.length > 0) reasons.push(`missing required metric(s): ${missingRequired.join(", ")}`);
|
|
305
314
|
if (def.filters?.image && !model.multimodal) reasons.push("role requires image input; model is text-only");
|
|
306
315
|
if (def.providerPin !== undefined && pinRoute === null) reasons.push("no route matches the role's provider pin");
|
|
307
316
|
if (em === null) reasons.push("no standard-tier route satisfies the role's endpoint filters");
|
|
308
317
|
if (eff.price == null) reasons.push("no billed OpenRouter route");
|
|
318
|
+
// maxPriceUsdPerM caps the model's own price basis, never the soft best-route
|
|
319
|
+
// basis (mirrors rankRole): with no hard pin the gate uses the blend price, so
|
|
320
|
+
// the soft basis cannot change eligibility; a hard pin gates its route price.
|
|
321
|
+
const cap = def.filters?.maxPriceUsdPerM ?? 0;
|
|
322
|
+
if (cap > 0) {
|
|
323
|
+
const capBase = pinRoute !== null ? routeEffectivePrice(pinRoute, cacheHitRate) : eff.price;
|
|
324
|
+
if (capBase != null && thinkingAdjustedPrice(eff, def, capBase) > cap) reasons.push(`price exceeds the role's maxPriceUsdPerM cap (${cap})`);
|
|
325
|
+
}
|
|
309
326
|
if (reasons.length > 0) return { eligible: false, reasons };
|
|
310
327
|
|
|
311
328
|
const ranked = rankRole(def, models);
|
|
@@ -403,6 +420,7 @@ export function explainModel(def: RoleDef, models: Model[], modelId: string, rol
|
|
|
403
420
|
q: self.q,
|
|
404
421
|
priceEff: self.priceEff,
|
|
405
422
|
ownProvider,
|
|
423
|
+
autoPin,
|
|
406
424
|
role: { name: roleName, lambda, derivedLambda, wPrice, qW, thinking: def.thinking, cacheHitRate },
|
|
407
425
|
contributions,
|
|
408
426
|
cost: { priceEff: self.priceEff, billedPrice: route !== null ? routeEffectivePrice(route, cacheHitRate) ?? eff.price : eff.price, lambda, penalty: lambda * self.priceEff, q: self.q, value: self.value },
|
package/src/explorer/server.ts
CHANGED
|
@@ -26,7 +26,7 @@ import { createServer, type IncomingMessage, type Server, type ServerResponse }
|
|
|
26
26
|
import { extname, resolve, sep } from "node:path";
|
|
27
27
|
import type { KeyAvailability } from "../availability.ts";
|
|
28
28
|
import { loadDeclaredSources } from "../benchmark-sources.ts";
|
|
29
|
-
import { SUFFIX_LEVELS, rankRole, roleLambda, thinkingPriceFactor, type RankData, type RoleDef, type SuffixLevel } from "../engine.ts";
|
|
29
|
+
import { pinnedRoute, SUFFIX_LEVELS, rankRole, roleLambda, thinkingPriceFactor, type RankData, type RoleDef, type SuffixLevel } from "../engine.ts";
|
|
30
30
|
import { expandFeatures, FEATURES } from "../features.ts";
|
|
31
31
|
import { isRecord } from "../guards.ts";
|
|
32
32
|
import { validateRole, writeRoleSettings } from "../role-settings.ts";
|
|
@@ -260,7 +260,20 @@ async function handleExport(req: IncomingMessage, res: ServerResponse, opts: Exp
|
|
|
260
260
|
const body = await readJson(req);
|
|
261
261
|
if (!isRecord(body) || !isRecord(body.roles)) throw new HttpError(400, "expected { roles: object }");
|
|
262
262
|
const scope = activeScope(opts, session);
|
|
263
|
-
const
|
|
263
|
+
const dirty = body.roles as unknown as Record<string, RoleDef>;
|
|
264
|
+
// A manual provider pin must name a route the loaded dataset actually has, or
|
|
265
|
+
// the exported role would be unroutable. Validate before any write; the
|
|
266
|
+
// plugin's own validator (empty/@/:) still runs, so both error sets surface.
|
|
267
|
+
const state = opts.getState(scope);
|
|
268
|
+
const errors: string[] = [];
|
|
269
|
+
for (const [name, def] of Object.entries(dirty)) {
|
|
270
|
+
errors.push(...validateRole(name, def));
|
|
271
|
+
if (def.providerPin === undefined) continue;
|
|
272
|
+
const known = state.rank.models.some((m) => pinnedRoute(m, def.providerPin) !== null);
|
|
273
|
+
if (!known) errors.push(`role "${name}": provider pin "${def.providerPin}" matches no route in the loaded dataset`);
|
|
274
|
+
}
|
|
275
|
+
if (errors.length > 0) return sendJson(res, 200, { ok: false, errors });
|
|
276
|
+
const result = writeRoleSettings(scope.lockPath, dirty);
|
|
264
277
|
if (!result.ok) return sendJson(res, 200, { ok: false, errors: result.errors });
|
|
265
278
|
sendJson(res, 200, { ok: true, backupPath: result.backupPath, roles: result.roles, lockPath: scope.lockPath });
|
|
266
279
|
}
|
package/src/extension.ts
CHANGED
|
@@ -20,13 +20,14 @@ import { generateAgentSpec, generateProjectProfile } from "./agent-architect.ts"
|
|
|
20
20
|
import { authorBenchmarkSource } from "./benchmark-author.ts";
|
|
21
21
|
import { applyBenchmarkScores, BENCHMARK_ENTRY_CAP, cachedSourceInfo, declarationToSource, declaredSourceForLink, dryRunDeclaration, fetchJson, loadBenchmarkCatalog, loadBenchmarkScores, loadDeclaredSources, resolveBenchmarkSource, saveDeclaredSource, sourceForMetric, validateDeclaration, type BenchmarkCatalogEntry, type BenchmarkPayloadMeta, type BenchmarkSource, type SourceDeclaration } from "./benchmark-sources.ts";
|
|
22
22
|
import { judgeBenchmarkRelevance } from "./benchmark-discovery.ts";
|
|
23
|
-
import {
|
|
23
|
+
import { catalogFromOmpModelsJson, fetchKeyAvailability, splitThinkingSuffix, type KeyAvailability } from "./availability.ts";
|
|
24
24
|
import { loadRankData, rankRole, type Model } from "./engine.ts";
|
|
25
25
|
import { defaultErrorReportDeps, reportUnexpectedError, type ErrorReportInput } from "./error-report.ts";
|
|
26
26
|
import { formatFeatures } from "./features.ts";
|
|
27
27
|
import { startExplorer, type ExplorerHandle } from "./explorer/boot.ts";
|
|
28
28
|
import { resolveScopes, unionRoles } from "./explorer/scopes.ts";
|
|
29
29
|
import { isRecord } from "./guards.ts";
|
|
30
|
+
import { applyProviderDisplay, providerPinOf, providerPinOfSelector } from "./provider-display.ts";
|
|
30
31
|
import { applyProfileOverrides, formatProjectRolesReport, parseProjectRolesArgs, PROJECT_ROLES_USAGE, setupProject, type ProjectProfile } from "./project-setup.ts";
|
|
31
32
|
import { DEFAULT_ROLES, findProjectAnchor, KNOWN_METRICS, PLUGIN_SETTINGS_PATH, projectLockPath, readPluginSettingsMap, resolveSettings } from "./settings.ts";
|
|
32
33
|
import { runUpdater, type Deps } from "./updater.ts";
|
|
@@ -71,17 +72,41 @@ function hasEmptyConversation(ctx: ExtContext): boolean {
|
|
|
71
72
|
return true;
|
|
72
73
|
}
|
|
73
74
|
|
|
74
|
-
/**
|
|
75
|
-
|
|
75
|
+
/**
|
|
76
|
+
* True when `selector` names the live session model: the selector's base
|
|
77
|
+
* (thinking suffix stripped) resolves to a model with the same `provider` and
|
|
78
|
+
* `id` as `ctx.models.current()`. Any thinking level or provider pin is
|
|
79
|
+
* ignored. Used both to detect a session booted on the old default role and to
|
|
80
|
+
* decide whether a fallback event's selector is authoritative for the session
|
|
81
|
+
* (advisor fallbacks emit the same events with the advisor's selector).
|
|
82
|
+
*/
|
|
83
|
+
function liveModelIsSelector(ctx: ExtContext, selector: string): boolean {
|
|
76
84
|
const current = ctx.models?.current();
|
|
77
|
-
const
|
|
78
|
-
const
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
if (
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
+
const { base } = splitThinkingSuffix(selector);
|
|
86
|
+
const target = ctx.models?.resolve(base);
|
|
87
|
+
if (current === undefined || current === null || target === undefined || target === null) return false;
|
|
88
|
+
if (typeof current !== "object" || typeof target !== "object") return false;
|
|
89
|
+
if (!("provider" in current) || !("id" in current) || !("provider" in target) || !("id" in target)) return false;
|
|
90
|
+
return current.provider === target.provider && current.id === target.id;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Apply the provider display suffix to the live session model (spec #77).
|
|
95
|
+
* `selector` is the event's authoritative selector string when the event
|
|
96
|
+
* carries one; the live model's routing block is the fallback. The event's
|
|
97
|
+
* selector is trusted only when it names the live session model — the fallback
|
|
98
|
+
* events are also emitted for advisor fallbacks, where only the advisor's model
|
|
99
|
+
* changes and the session model is untouched. Contained: a throw must not tear
|
|
100
|
+
* down the session or escape into omp's dispatch.
|
|
101
|
+
*/
|
|
102
|
+
function applyProviderDisplayFor(ctx: ExtContext, selector?: string): void {
|
|
103
|
+
try {
|
|
104
|
+
const live = ctx.models?.current();
|
|
105
|
+
const eventPin = typeof selector === "string" && liveModelIsSelector(ctx, selector) ? providerPinOfSelector(selector) : null;
|
|
106
|
+
applyProviderDisplay(live, eventPin ?? providerPinOf(live));
|
|
107
|
+
} catch {
|
|
108
|
+
// display-only; never let a failure escape into omp's dispatch
|
|
109
|
+
}
|
|
85
110
|
}
|
|
86
111
|
|
|
87
112
|
/**
|
|
@@ -97,10 +122,8 @@ function makeApplySessionModel(pi: ExtensionAPI, ctx: ExtContext, notify: (lines
|
|
|
97
122
|
if (ctx.agent.kind !== "main") return;
|
|
98
123
|
if (!hasEmptyConversation(ctx)) return;
|
|
99
124
|
if (previous === null) return; // first run ever wrote a fresh default; the session predates no old role
|
|
100
|
-
if (!
|
|
101
|
-
const
|
|
102
|
-
const level = colon !== -1 && selector.slice(colon + 1) in THINKING_LEVELS ? selector.slice(colon + 1) : null;
|
|
103
|
-
const base = level === null ? selector : selector.slice(0, selector.length - level.length - 1);
|
|
125
|
+
if (!liveModelIsSelector(ctx, previous)) return;
|
|
126
|
+
const { base, level } = splitThinkingSuffix(selector);
|
|
104
127
|
const model = ctx.models?.resolve(base);
|
|
105
128
|
if (model === undefined || model === null) {
|
|
106
129
|
notify([`omp-llm-role: session model not updated — "${base}" matches no available model`]);
|
|
@@ -418,6 +441,26 @@ export default function (pi: ExtensionAPI) {
|
|
|
418
441
|
}
|
|
419
442
|
});
|
|
420
443
|
|
|
444
|
+
// Provider display (spec #77): keep the live model's name suffixed with its
|
|
445
|
+
// provider pin. Never pi.setModel — that would wipe omp's in-flight fallback
|
|
446
|
+
// state and clobber the thinking level; mutate the live model's name only.
|
|
447
|
+
// Registered after the updater so the session-start pin reflects the model
|
|
448
|
+
// the run just selected.
|
|
449
|
+
pi.on("session_start", async (_event, ctx: ExtContext) => {
|
|
450
|
+
applyProviderDisplayFor(ctx);
|
|
451
|
+
});
|
|
452
|
+
pi.on("retry_fallback_applied", async (event, ctx: ExtContext) => {
|
|
453
|
+
applyProviderDisplayFor(ctx, event.to);
|
|
454
|
+
});
|
|
455
|
+
pi.on("retry_fallback_succeeded", async (event, ctx: ExtContext) => {
|
|
456
|
+
applyProviderDisplayFor(ctx, event.model);
|
|
457
|
+
});
|
|
458
|
+
// Self-healing hook: omp restores the primary after a fallback cooldown with
|
|
459
|
+
// no session event, so re-derive the pin from the live model before each turn.
|
|
460
|
+
pi.on("turn_start", async (_event, ctx: ExtContext) => {
|
|
461
|
+
applyProviderDisplayFor(ctx);
|
|
462
|
+
});
|
|
463
|
+
|
|
421
464
|
pi.registerCommand("refresh-roles", {
|
|
422
465
|
description: "Refresh model roles from today's llm-stats/OpenRouter rankings",
|
|
423
466
|
handler: async (_args, ctx: ExtContext) => {
|
package/src/features.ts
CHANGED
|
@@ -19,7 +19,7 @@ export type FeatureRow = {
|
|
|
19
19
|
description: string;
|
|
20
20
|
/** The recommended settings the flag applies. A bundle value is either a
|
|
21
21
|
* scalar knob or a per-model policy field (`preferOwnProvider`). */
|
|
22
|
-
recommended: { filters?: { tools?: boolean; minOutputTokens?: number; maxPriceUsdPerM?: number }; cacheHitRate?: number; preferOwnProvider?: true };
|
|
22
|
+
recommended: { filters?: { tools?: boolean; minOutputTokens?: number; maxPriceUsdPerM?: number; preferredMinThroughput?: number }; cacheHitRate?: number; preferOwnProvider?: true };
|
|
23
23
|
/** What the flag buys, for `--list-features` and the docs. */
|
|
24
24
|
buys: string;
|
|
25
25
|
};
|
|
@@ -41,14 +41,14 @@ export const FEATURES: readonly FeatureRow[] = [
|
|
|
41
41
|
},
|
|
42
42
|
{
|
|
43
43
|
id: "providerPinning",
|
|
44
|
-
label: "
|
|
44
|
+
label: "Best-route provider pinning",
|
|
45
45
|
// Per-model policy, not a scalar slug: the sensible pin differs per model,
|
|
46
46
|
// so the bundle carries `preferOwnProvider` (evaluated against the model's
|
|
47
|
-
//
|
|
48
|
-
//
|
|
49
|
-
description: "
|
|
50
|
-
recommended: { preferOwnProvider: true },
|
|
51
|
-
buys: "
|
|
47
|
+
// best route at rank time). It changes no eligibility — a model with no
|
|
48
|
+
// candidate route keeps the default 1/price² blend.
|
|
49
|
+
description: "Bind each request to the model's best provider route (per-model policy; never drops a model) and prefer routes at or above 50 tok/s",
|
|
50
|
+
recommended: { preferOwnProvider: true, filters: { preferredMinThroughput: 50 } },
|
|
51
|
+
buys: "each request is bound to the model's best provider route — the selector gains @<slug> — and the fallback chain carries the same model on its next-best providers plus each fallback model on its best providers, so a down route fails over within the model's own routes; a soft 50 tok/s floor deprioritizes a pathologically slow route without dropping it",
|
|
52
52
|
},
|
|
53
53
|
{
|
|
54
54
|
id: "costCap",
|
|
@@ -69,6 +69,7 @@ function formatBundle(rec: FeatureRow["recommended"]): string {
|
|
|
69
69
|
if (rec.filters?.tools !== undefined) parts.push(`filters.tools=${rec.filters.tools}`);
|
|
70
70
|
if (rec.filters?.minOutputTokens !== undefined) parts.push(`filters.minOutputTokens=${rec.filters.minOutputTokens}`);
|
|
71
71
|
if (rec.filters?.maxPriceUsdPerM !== undefined) parts.push(`filters.maxPriceUsdPerM=${rec.filters.maxPriceUsdPerM}`);
|
|
72
|
+
if (rec.filters?.preferredMinThroughput !== undefined) parts.push(`filters.preferredMinThroughput=${rec.filters.preferredMinThroughput}`);
|
|
72
73
|
if (rec.cacheHitRate !== undefined) parts.push(`cacheHitRate=${rec.cacheHitRate}`);
|
|
73
74
|
if (rec.preferOwnProvider !== undefined) parts.push(`preferOwnProvider=${rec.preferOwnProvider}`);
|
|
74
75
|
return parts.join(", ");
|
|
@@ -95,6 +96,7 @@ export function expandFeatures(authored: RoleDef, globalFlags: Record<string, bo
|
|
|
95
96
|
if (rec.filters.tools !== undefined && f.tools === undefined) f.tools = rec.filters.tools;
|
|
96
97
|
if (rec.filters.minOutputTokens !== undefined && f.minOutputTokens === undefined) f.minOutputTokens = rec.filters.minOutputTokens;
|
|
97
98
|
if (rec.filters.maxPriceUsdPerM !== undefined && f.maxPriceUsdPerM === undefined) f.maxPriceUsdPerM = rec.filters.maxPriceUsdPerM;
|
|
99
|
+
if (rec.filters.preferredMinThroughput !== undefined && f.preferredMinThroughput === undefined) f.preferredMinThroughput = rec.filters.preferredMinThroughput;
|
|
98
100
|
}
|
|
99
101
|
if (rec.cacheHitRate !== undefined && out.cacheHitRate === undefined) out.cacheHitRate = rec.cacheHitRate;
|
|
100
102
|
if (rec.preferOwnProvider !== undefined && out.preferOwnProvider === undefined) out.preferOwnProvider = rec.preferOwnProvider;
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Provider display: keep the live session model's `name` suffixed with its
|
|
3
|
+
* upstream provider pin, so omp's model segment renders e.g.
|
|
4
|
+
* `DeepSeek V4.1 Flash @coreweave/fp8`.
|
|
5
|
+
*
|
|
6
|
+
* omp's built-in model segment renders `state.model.name || state.model.id`
|
|
7
|
+
* only and exposes no extension hook to add to it; the only lever is the live
|
|
8
|
+
* model object's `name` (the exact object `ctx.models.current()` returns). This
|
|
9
|
+
* module is pure and dual-runtime (no `@oh-my-pi/*` import) — the extension
|
|
10
|
+
* wiring only calls it.
|
|
11
|
+
*/
|
|
12
|
+
import { splitThinkingSuffix } from "./availability.ts";
|
|
13
|
+
import { isRecord } from "./guards.ts";
|
|
14
|
+
|
|
15
|
+
/** The single upstream route of a compat routing block, or null. Mirrors omp's
|
|
16
|
+
* `getSingleRoutingOnly`: exactly one entry, a non-empty string. */
|
|
17
|
+
function singleOnly(routing: unknown): string | null {
|
|
18
|
+
if (!isRecord(routing) || !Array.isArray(routing.only) || routing.only.length !== 1) return null;
|
|
19
|
+
const upstream = routing.only[0];
|
|
20
|
+
return typeof upstream === "string" && upstream ? upstream : null;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* The provider pin carried by a model's compat routing block:
|
|
25
|
+
* `compat.openRouterRouting.only[0]`, else `compat.vercelGatewayRouting.only[0]`.
|
|
26
|
+
* Null when absent, when `only` is not exactly one entry, or when the entry is
|
|
27
|
+
* not a non-empty string.
|
|
28
|
+
*/
|
|
29
|
+
export function providerPinOf(model: unknown): string | null {
|
|
30
|
+
if (!isRecord(model) || !isRecord(model.compat)) return null;
|
|
31
|
+
const compat = model.compat;
|
|
32
|
+
return singleOnly(compat.openRouterRouting) ?? singleOnly(compat.vercelGatewayRouting);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* The trailing `@<slug>` of a selector (`openrouter/<id>@<slug>:<level>`),
|
|
37
|
+
* after stripping a trailing thinking suffix (`:high`, `:auto`, …). Null when
|
|
38
|
+
* there is no `@` or the slug is empty. Strip order mirrors `currentRankingId`
|
|
39
|
+
* in `src/availability.ts`: level first, then `@`.
|
|
40
|
+
*/
|
|
41
|
+
export function providerPinOfSelector(selector: string): string | null {
|
|
42
|
+
const { base } = splitThinkingSuffix(selector);
|
|
43
|
+
const at = base.lastIndexOf("@");
|
|
44
|
+
if (at === -1) return null;
|
|
45
|
+
const slug = base.slice(at + 1);
|
|
46
|
+
return slug || null;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Strip any previous ` @<slug>` suffix from `name`, then append ` @<pin>` when
|
|
51
|
+
* `pin` is non-null. Idempotent; a null pin returns the base name (suffix
|
|
52
|
+
* stripped). Plain text only — no theme tokens, no ANSI.
|
|
53
|
+
*/
|
|
54
|
+
export function withProviderSuffix(name: string, pin: string | null): string {
|
|
55
|
+
const base = name.replace(/\s@\S+$/, "");
|
|
56
|
+
return pin === null ? base : `${base} @${pin}`;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Set `model.name` to `withProviderSuffix(model.name, pin)` when `model` is a
|
|
61
|
+
* record with a string `name`; a no-op otherwise.
|
|
62
|
+
*/
|
|
63
|
+
export function applyProviderDisplay(model: unknown, pin: string | null): void {
|
|
64
|
+
if (isRecord(model) && typeof model.name === "string") {
|
|
65
|
+
model.name = withProviderSuffix(model.name, pin);
|
|
66
|
+
}
|
|
67
|
+
}
|