omp-llm-role 1.1.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +75 -13
- package/docs/dev/architecture.md +35 -11
- package/docs/dev/ci.md +21 -1
- package/docs/dev/current-state.md +99 -27
- package/docs/dev/data-sources.md +33 -4
- package/docs/dev/explorer.md +126 -21
- package/docs/dev/quirks.md +32 -0
- package/docs/dev/releasing.md +63 -32
- package/docs/dev/scoring.md +153 -39
- package/docs/dev/testing.md +112 -24
- package/docs/dev/writes-and-state.md +86 -0
- package/docs/llm-role-rankings.md +126 -126
- package/package.json +92 -22
- package/src/agent-create.ts +4 -2
- package/src/availability.ts +87 -9
- package/src/cli/create-role.ts +1 -1
- package/src/engine.ts +190 -78
- package/src/explorer/boot.ts +43 -1
- package/src/explorer/explain.ts +113 -10
- package/src/explorer/levers.ts +100 -0
- package/src/explorer/pool.ts +46 -0
- package/src/explorer/server.ts +107 -23
- package/src/extension.ts +189 -16
- package/src/features.ts +9 -7
- package/src/provider-display.ts +67 -0
- package/src/settings.ts +39 -2
- package/src/updater.ts +97 -14
- package/src/usage-ledger.ts +127 -0
- package/web/app.js +156 -1
- package/web/style.css +3 -0
package/src/features.ts
CHANGED
|
@@ -19,7 +19,7 @@ export type FeatureRow = {
|
|
|
19
19
|
description: string;
|
|
20
20
|
/** The recommended settings the flag applies. A bundle value is either a
|
|
21
21
|
* scalar knob or a per-model policy field (`preferOwnProvider`). */
|
|
22
|
-
recommended: { filters?: { tools?: boolean; minOutputTokens?: number; maxPriceUsdPerM?: number }; cacheHitRate?: number; preferOwnProvider?: true };
|
|
22
|
+
recommended: { filters?: { tools?: boolean; minOutputTokens?: number; maxPriceUsdPerM?: number; preferredMinThroughput?: number }; cacheHitRate?: number; preferOwnProvider?: true };
|
|
23
23
|
/** What the flag buys, for `--list-features` and the docs. */
|
|
24
24
|
buys: string;
|
|
25
25
|
};
|
|
@@ -41,14 +41,14 @@ export const FEATURES: readonly FeatureRow[] = [
|
|
|
41
41
|
},
|
|
42
42
|
{
|
|
43
43
|
id: "providerPinning",
|
|
44
|
-
label: "
|
|
44
|
+
label: "Best-route provider pinning",
|
|
45
45
|
// Per-model policy, not a scalar slug: the sensible pin differs per model,
|
|
46
46
|
// so the bundle carries `preferOwnProvider` (evaluated against the model's
|
|
47
|
-
//
|
|
48
|
-
//
|
|
49
|
-
description: "
|
|
50
|
-
recommended: { preferOwnProvider: true },
|
|
51
|
-
buys: "
|
|
47
|
+
// best route at rank time). It changes no eligibility — a model with no
|
|
48
|
+
// candidate route keeps the default 1/price² blend.
|
|
49
|
+
description: "Bind each request to the model's best provider route (per-model policy; never drops a model) and prefer routes at or above 50 tok/s",
|
|
50
|
+
recommended: { preferOwnProvider: true, filters: { preferredMinThroughput: 50 } },
|
|
51
|
+
buys: "each request is bound to the model's best provider route — the selector gains @<slug> — and the fallback chain carries the same model on its next-best providers plus each fallback model on its best providers, so a down route fails over within the model's own routes; a soft 50 tok/s floor deprioritizes a pathologically slow route without dropping it",
|
|
52
52
|
},
|
|
53
53
|
{
|
|
54
54
|
id: "costCap",
|
|
@@ -69,6 +69,7 @@ function formatBundle(rec: FeatureRow["recommended"]): string {
|
|
|
69
69
|
if (rec.filters?.tools !== undefined) parts.push(`filters.tools=${rec.filters.tools}`);
|
|
70
70
|
if (rec.filters?.minOutputTokens !== undefined) parts.push(`filters.minOutputTokens=${rec.filters.minOutputTokens}`);
|
|
71
71
|
if (rec.filters?.maxPriceUsdPerM !== undefined) parts.push(`filters.maxPriceUsdPerM=${rec.filters.maxPriceUsdPerM}`);
|
|
72
|
+
if (rec.filters?.preferredMinThroughput !== undefined) parts.push(`filters.preferredMinThroughput=${rec.filters.preferredMinThroughput}`);
|
|
72
73
|
if (rec.cacheHitRate !== undefined) parts.push(`cacheHitRate=${rec.cacheHitRate}`);
|
|
73
74
|
if (rec.preferOwnProvider !== undefined) parts.push(`preferOwnProvider=${rec.preferOwnProvider}`);
|
|
74
75
|
return parts.join(", ");
|
|
@@ -95,6 +96,7 @@ export function expandFeatures(authored: RoleDef, globalFlags: Record<string, bo
|
|
|
95
96
|
if (rec.filters.tools !== undefined && f.tools === undefined) f.tools = rec.filters.tools;
|
|
96
97
|
if (rec.filters.minOutputTokens !== undefined && f.minOutputTokens === undefined) f.minOutputTokens = rec.filters.minOutputTokens;
|
|
97
98
|
if (rec.filters.maxPriceUsdPerM !== undefined && f.maxPriceUsdPerM === undefined) f.maxPriceUsdPerM = rec.filters.maxPriceUsdPerM;
|
|
99
|
+
if (rec.filters.preferredMinThroughput !== undefined && f.preferredMinThroughput === undefined) f.preferredMinThroughput = rec.filters.preferredMinThroughput;
|
|
98
100
|
}
|
|
99
101
|
if (rec.cacheHitRate !== undefined && out.cacheHitRate === undefined) out.cacheHitRate = rec.cacheHitRate;
|
|
100
102
|
if (rec.preferOwnProvider !== undefined && out.preferOwnProvider === undefined) out.preferOwnProvider = rec.preferOwnProvider;
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Provider display: keep the live session model's `name` suffixed with its
|
|
3
|
+
* upstream provider pin, so omp's model segment renders e.g.
|
|
4
|
+
* `DeepSeek V4.1 Flash @coreweave/fp8`.
|
|
5
|
+
*
|
|
6
|
+
* omp's built-in model segment renders `state.model.name || state.model.id`
|
|
7
|
+
* only and exposes no extension hook to add to it; the only lever is the live
|
|
8
|
+
* model object's `name` (the exact object `ctx.models.current()` returns). This
|
|
9
|
+
* module is pure and dual-runtime (no `@oh-my-pi/*` import) — the extension
|
|
10
|
+
* wiring only calls it.
|
|
11
|
+
*/
|
|
12
|
+
import { splitThinkingSuffix } from "./availability.ts";
|
|
13
|
+
import { isRecord } from "./guards.ts";
|
|
14
|
+
|
|
15
|
+
/** The single upstream route of a compat routing block, or null. Mirrors omp's
|
|
16
|
+
* `getSingleRoutingOnly`: exactly one entry, a non-empty string. */
|
|
17
|
+
function singleOnly(routing: unknown): string | null {
|
|
18
|
+
if (!isRecord(routing) || !Array.isArray(routing.only) || routing.only.length !== 1) return null;
|
|
19
|
+
const upstream = routing.only[0];
|
|
20
|
+
return typeof upstream === "string" && upstream ? upstream : null;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* The provider pin carried by a model's compat routing block:
|
|
25
|
+
* `compat.openRouterRouting.only[0]`, else `compat.vercelGatewayRouting.only[0]`.
|
|
26
|
+
* Null when absent, when `only` is not exactly one entry, or when the entry is
|
|
27
|
+
* not a non-empty string.
|
|
28
|
+
*/
|
|
29
|
+
export function providerPinOf(model: unknown): string | null {
|
|
30
|
+
if (!isRecord(model) || !isRecord(model.compat)) return null;
|
|
31
|
+
const compat = model.compat;
|
|
32
|
+
return singleOnly(compat.openRouterRouting) ?? singleOnly(compat.vercelGatewayRouting);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* The trailing `@<slug>` of a selector (`openrouter/<id>@<slug>:<level>`),
|
|
37
|
+
* after stripping a trailing thinking suffix (`:high`, `:auto`, …). Null when
|
|
38
|
+
* there is no `@` or the slug is empty. Strip order mirrors `currentRankingId`
|
|
39
|
+
* in `src/availability.ts`: level first, then `@`.
|
|
40
|
+
*/
|
|
41
|
+
export function providerPinOfSelector(selector: string): string | null {
|
|
42
|
+
const { base } = splitThinkingSuffix(selector);
|
|
43
|
+
const at = base.lastIndexOf("@");
|
|
44
|
+
if (at === -1) return null;
|
|
45
|
+
const slug = base.slice(at + 1);
|
|
46
|
+
return slug || null;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Strip any previous ` @<slug>` suffix from `name`, then append ` @<pin>` when
|
|
51
|
+
* `pin` is non-null. Idempotent; a null pin returns the base name (suffix
|
|
52
|
+
* stripped). Plain text only — no theme tokens, no ANSI.
|
|
53
|
+
*/
|
|
54
|
+
export function withProviderSuffix(name: string, pin: string | null): string {
|
|
55
|
+
const base = name.replace(/\s@\S+$/, "");
|
|
56
|
+
return pin === null ? base : `${base} @${pin}`;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Set `model.name` to `withProviderSuffix(model.name, pin)` when `model` is a
|
|
61
|
+
* record with a string `name`; a no-op otherwise.
|
|
62
|
+
*/
|
|
63
|
+
export function applyProviderDisplay(model: unknown, pin: string | null): void {
|
|
64
|
+
if (isRecord(model) && typeof model.name === "string") {
|
|
65
|
+
model.name = withProviderSuffix(model.name, pin);
|
|
66
|
+
}
|
|
67
|
+
}
|
package/src/settings.ts
CHANGED
|
@@ -16,6 +16,7 @@ import { dirname, join } from "node:path";
|
|
|
16
16
|
import { SUFFIX_LEVELS, type RoleDef } from "./engine.ts";
|
|
17
17
|
import { FEATURES, expandFeatures, featureById } from "./features.ts";
|
|
18
18
|
import { isRecord } from "./guards.ts";
|
|
19
|
+
import type { TokenProfile } from "./usage-ledger.ts";
|
|
19
20
|
|
|
20
21
|
/** User-level plugin settings lock file (the explorer's read/export target and
|
|
21
22
|
* the default source for `readPluginSettingsMap`). */
|
|
@@ -244,6 +245,9 @@ export type ResolvedSettings = {
|
|
|
244
245
|
/** Global capability flags (`features.<id>`); a role's `features.<id>` overrides
|
|
245
246
|
* the global in either direction. Expanded per role by `resolveSettings`. */
|
|
246
247
|
features: Record<string, boolean>;
|
|
248
|
+
/** Per-task token mix for the per-task price, used when the role has no
|
|
249
|
+
* recorded usage. `null` = no profile (the per-task price is absent). */
|
|
250
|
+
tokenProfile: TokenProfile | null;
|
|
247
251
|
};
|
|
248
252
|
|
|
249
253
|
export const DEFAULT_SETTINGS: ResolvedSettings = {
|
|
@@ -255,6 +259,7 @@ export const DEFAULT_SETTINGS: ResolvedSettings = {
|
|
|
255
259
|
activateDefaultOnEmptySession: true,
|
|
256
260
|
errorReporting: "off",
|
|
257
261
|
features: {},
|
|
262
|
+
tokenProfile: null,
|
|
258
263
|
};
|
|
259
264
|
|
|
260
265
|
/** One row of the plugin's settings schema (omp's `PluginSettingSchema`): the
|
|
@@ -312,6 +317,16 @@ export function deriveSettingsSchema(): Record<string, PluginSettingSchema> {
|
|
|
312
317
|
default: DEFAULT_SETTINGS.errorReporting,
|
|
313
318
|
values: ["off", "ask", "auto"],
|
|
314
319
|
},
|
|
320
|
+
"tokenProfile.inputTokens": {
|
|
321
|
+
type: "number",
|
|
322
|
+
description: "Input tokens per task for the per-task price (used when the role has no recorded usage)",
|
|
323
|
+
min: 0,
|
|
324
|
+
},
|
|
325
|
+
"tokenProfile.outputTokens": {
|
|
326
|
+
type: "number",
|
|
327
|
+
description: "Output tokens per task for the per-task price (used when the role has no recorded usage)",
|
|
328
|
+
min: 0,
|
|
329
|
+
},
|
|
315
330
|
};
|
|
316
331
|
for (const row of FEATURES) {
|
|
317
332
|
schema[`features.${row.id}`] = {
|
|
@@ -359,7 +374,7 @@ export function deriveSettingsSchema(): Record<string, PluginSettingSchema> {
|
|
|
359
374
|
}
|
|
360
375
|
schema[`${p}.preferOwnProvider`] = {
|
|
361
376
|
type: "boolean",
|
|
362
|
-
description: `
|
|
377
|
+
description: `Bind ${name} requests to each model's best provider route (soft preference; never drops a model)`,
|
|
363
378
|
default: false,
|
|
364
379
|
};
|
|
365
380
|
schema[`${p}.required`] = {
|
|
@@ -395,6 +410,12 @@ export function deriveSettingsSchema(): Record<string, PluginSettingSchema> {
|
|
|
395
410
|
default: def.filters?.maxPriceUsdPerM ?? 0,
|
|
396
411
|
min: 0,
|
|
397
412
|
};
|
|
413
|
+
schema[`${p}.filters.preferredMinThroughput`] = {
|
|
414
|
+
type: "number",
|
|
415
|
+
description: `Soft throughput floor for ${name} routes (tok/s; 0 = off) — a below-floor route sorts last but is never dropped`,
|
|
416
|
+
default: def.filters?.preferredMinThroughput ?? 0,
|
|
417
|
+
min: 0,
|
|
418
|
+
};
|
|
398
419
|
schema[`${p}.lambda`] = {
|
|
399
420
|
type: "number",
|
|
400
421
|
description: `Explicit λ ($ per quality point) override for ${name}`,
|
|
@@ -567,6 +588,22 @@ export function resolveSettings(raw: Record<string, unknown>): { settings: Resol
|
|
|
567
588
|
errors.push(`errorReporting: must be one of off, ask, auto, got ${JSON.stringify(merged.errorReporting)}`);
|
|
568
589
|
}
|
|
569
590
|
|
|
591
|
+
// The per-task token profile: null (no profile), or a record whose two counts
|
|
592
|
+
// are finite numbers >= 0. One error string per violation.
|
|
593
|
+
const tokenProfile: unknown = merged.tokenProfile;
|
|
594
|
+
if (tokenProfile !== null) {
|
|
595
|
+
if (!isRecord(tokenProfile)) {
|
|
596
|
+
errors.push(`tokenProfile: must be null or an object with inputTokens/outputTokens, got ${JSON.stringify(tokenProfile)}`);
|
|
597
|
+
} else {
|
|
598
|
+
for (const key of ["inputTokens", "outputTokens"] as const) {
|
|
599
|
+
const v = tokenProfile[key];
|
|
600
|
+
if (typeof v !== "number" || !Number.isFinite(v) || v < 0) {
|
|
601
|
+
errors.push(`tokenProfile.${key}: must be a number >= 0, got ${JSON.stringify(v)}`);
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
}
|
|
605
|
+
}
|
|
606
|
+
|
|
570
607
|
// Global capability flags (`features.<id>`): an unknown id or a non-boolean
|
|
571
608
|
// value aborts the run (a typo must never silently do nothing). A non-record
|
|
572
609
|
// `features` is reset so the per-role expansion below always gets a map.
|
|
@@ -647,7 +684,7 @@ export function resolveSettings(raw: Record<string, unknown>): { settings: Resol
|
|
|
647
684
|
if (rdef.filters.tools !== undefined && typeof rdef.filters.tools !== "boolean") {
|
|
648
685
|
errors.push(`role ${name}: filters.tools must be a boolean`);
|
|
649
686
|
}
|
|
650
|
-
for (const key of ["minContextTokens", "minOutputTokens", "maxPriceUsdPerM"] as const) {
|
|
687
|
+
for (const key of ["minContextTokens", "minOutputTokens", "maxPriceUsdPerM", "preferredMinThroughput"] as const) {
|
|
651
688
|
const v = rdef.filters[key];
|
|
652
689
|
if (v !== undefined && (typeof v !== "number" || !Number.isFinite(v) || v < 0)) {
|
|
653
690
|
errors.push(`role ${name}: filters.${key} must be a number ≥ 0`);
|
package/src/updater.ts
CHANGED
|
@@ -15,8 +15,8 @@
|
|
|
15
15
|
|
|
16
16
|
import { readFileSync, statSync } from "node:fs";
|
|
17
17
|
import { dirname, join } from "node:path";
|
|
18
|
-
import { computeRankings, endpointFilterDrops, loadRankData, META_LEVELS, providerPinDrops, type Ranked, type RankData } from "./engine.ts";
|
|
19
|
-
import { currentRankingId, enrichThinkingLevels, fetchKeyAvailability, fetchKeyMeta, filterCatalog, probeModel, resolveVariant, THINKING_LEVELS, tierGate, type CatalogEntry, type KeyAvailability, type KeyMeta, type ProbeVerdict } from "./availability.ts";
|
|
18
|
+
import { autoPinApplies, bestRoute, computeRankings, endpointFilterDrops, loadRankData, META_LEVELS, providerPinDrops, rankedRoutes, type Ranked, type RankData, type RoleDef } from "./engine.ts";
|
|
19
|
+
import { currentRankingId, enrichThinkingLevels, fetchAllowedProviders, fetchKeyAvailability, fetchKeyMeta, filterCatalog, probeModel, providerAllowed, resolveVariant, THINKING_LEVELS, tierGate, type CatalogEntry, type KeyAvailability, type KeyMeta, type ProbeVerdict } from "./availability.ts";
|
|
20
20
|
import { ConfigEditError, parseConfig, patchConfig, writeConfigAtomic, type ConfigPatch } from "./config-edit.ts";
|
|
21
21
|
import { PLUGIN_SETTINGS_PATH, projectLockPath, readPluginSettingsMap, resolveSettings, roleUniverse, type ResolvedSettings } from "./settings.ts";
|
|
22
22
|
import { discoverAgentPins } from "./agent-pins.ts";
|
|
@@ -68,6 +68,11 @@ export type Trigger = "session-start" | "manual";
|
|
|
68
68
|
/** A ranked model resolved to a concrete catalog id. */
|
|
69
69
|
type Candidate = { ranked: Ranked; catalogId: string };
|
|
70
70
|
|
|
71
|
+
/** Per managed role: its bare chain key, role suffix, provider pin, the pool it
|
|
72
|
+
* was chosen from, and the inputs the auto-pin chain builder needs (the expanded
|
|
73
|
+
* def for `rankedRoutes`, and whether the pin was auto-derived). */
|
|
74
|
+
type ChainPlan = { key: string; suffix: string | undefined; pin: string; pool: Candidate[]; chosenIdx: number; def: RoleDef; autoPin: boolean };
|
|
75
|
+
|
|
71
76
|
export type Deps = {
|
|
72
77
|
getToken(): Promise<string>;
|
|
73
78
|
getCatalog(): Promise<CatalogEntry[]>;
|
|
@@ -80,6 +85,13 @@ export type Deps = {
|
|
|
80
85
|
probeModel?(token: string, catalogId: string): Promise<ProbeVerdict>;
|
|
81
86
|
/** DI seam like `getKeyMeta`/`probeModel`; production defaults to `fetchKeyAvailability` (never throws). */
|
|
82
87
|
getKeyAvailability?(token: string): Promise<KeyAvailability>;
|
|
88
|
+
/**
|
|
89
|
+
* DI seam like `getKeyAvailability`; production defaults to
|
|
90
|
+
* `fetchAllowedProviders` (never throws). Returns the account's
|
|
91
|
+
* allowed-providers privacy whitelist, or null when there is none (or the
|
|
92
|
+
* harvest failed) — a null result leaves every route pool untouched.
|
|
93
|
+
*/
|
|
94
|
+
getAllowedProviders?(token: string, modelId: string): Promise<Set<string> | null>;
|
|
83
95
|
/**
|
|
84
96
|
* Host coupling (extension only): apply a concrete selector to the live
|
|
85
97
|
* session model. Receives the final `default` selector (with thinking
|
|
@@ -200,8 +212,13 @@ function writePatch(configPath: string, dir: string, patch: ConfigPatch): { stat
|
|
|
200
212
|
* behavior applies. omp resolves the project dir as `<cwd>/.omp` with no
|
|
201
213
|
* walk-up, so the plugin's project-scope read must use the same dir (not the
|
|
202
214
|
* walk-up anchor `findProjectAnchor` returns).
|
|
215
|
+
*
|
|
216
|
+
* Exported as the one home for "where this session's plugin state lives": the
|
|
217
|
+
* updater scopes config/state/history/lock to it, and the extension records the
|
|
218
|
+
* usage ledger there too, so a project's usage is separate from the user-level
|
|
219
|
+
* profile.
|
|
203
220
|
*/
|
|
204
|
-
function resolveProjectDir(): string | null {
|
|
221
|
+
export function resolveProjectDir(): string | null {
|
|
205
222
|
const projectDir = join(process.cwd(), ".omp");
|
|
206
223
|
const raw = readPluginSettingsMap({ global: join(projectDir, "plugins", "omp-plugins.lock.json"), project: null });
|
|
207
224
|
return Object.keys(raw).length > 0 ? projectDir : null;
|
|
@@ -319,9 +336,38 @@ export async function runUpdater(trigger: Trigger, deps: Deps, opts?: { force?:
|
|
|
319
336
|
`omp-llm-role: OpenRouter keyed catalog ${availability.reason === "no-filter" ? "not filtering" : "unavailable"} — enable OpenRouter → Settings → "Filter the model catalog for API keys" to skip probing key-blocked models; the probe walk is still in use`,
|
|
320
337
|
);
|
|
321
338
|
}
|
|
339
|
+
|
|
340
|
+
// Allowed-providers privacy whitelist (spec #65): harvested once per run
|
|
341
|
+
// with a valid catalog model id (the canary 404s, so it bills no tokens).
|
|
342
|
+
// The whitelist is invisible to /api/v1/key and the catalog endpoints, so a
|
|
343
|
+
// route on a provider outside it would be written into config.yml and 404
|
|
344
|
+
// at request time. Pruning every model's route pool to allowed providers
|
|
345
|
+
// before ranking makes the soft basis, the auto-pin and the fallback chains
|
|
346
|
+
// whitelist-aware through the single `m.routes` seam. A null result (no
|
|
347
|
+
// whitelist set / harvest failed) leaves the pools untouched.
|
|
348
|
+
const canaryModelId = eligible[0]?.id;
|
|
349
|
+
const allowed =
|
|
350
|
+
canaryModelId === undefined
|
|
351
|
+
? null
|
|
352
|
+
: await (deps.getAllowedProviders ? deps.getAllowedProviders(token, canaryModelId) : fetchAllowedProviders(token, canaryModelId));
|
|
353
|
+
if (allowed !== null) {
|
|
354
|
+
let prunedCount = 0;
|
|
355
|
+
const pruned = rank.models.map((m) => {
|
|
356
|
+
const routes = m.routes;
|
|
357
|
+
if (routes === undefined) return m;
|
|
358
|
+
const kept = routes.filter((r) => providerAllowed(allowed, r.providerSlug));
|
|
359
|
+
prunedCount += routes.length - kept.length;
|
|
360
|
+
return { ...m, routes: kept };
|
|
361
|
+
});
|
|
362
|
+
rank = { ...rank, models: pruned };
|
|
363
|
+
notes.push(
|
|
364
|
+
`omp-llm-role: OpenRouter allowed-providers whitelist active — ${allowed.size} providers; ${prunedCount} routes pruned`,
|
|
365
|
+
);
|
|
366
|
+
} else {
|
|
367
|
+
notes.push(`omp-llm-role: OpenRouter allowed-providers whitelist: no whitelist`);
|
|
368
|
+
}
|
|
322
369
|
const poolByRole: Record<string, Candidate[]> = {};
|
|
323
|
-
|
|
324
|
-
const chainPlanByRole: Record<string, { key: string; suffix: string | undefined; pin: string; pool: Candidate[]; chosenIdx: number }> = {};
|
|
370
|
+
const chainPlanByRole: Record<string, ChainPlan> = {};
|
|
325
371
|
/** Roles whose `providerPin` matched no route: left unchanged, their current chain preserved. */
|
|
326
372
|
const pinUnmatched: string[] = [];
|
|
327
373
|
const probe = deps.probeModel ?? probeModel;
|
|
@@ -436,7 +482,16 @@ export async function runUpdater(trigger: Trigger, deps: Deps, opts?: { force?:
|
|
|
436
482
|
// thinking level (`openrouter/<id>@<slug>:<level>`) — omp's
|
|
437
483
|
// `splitUpstreamRouting` parses the trailing `@<slug>` and applies
|
|
438
484
|
// `compat.openRouterRouting = { only: [slug] }`.
|
|
439
|
-
|
|
485
|
+
//
|
|
486
|
+
// Auto-pin (spec #58): with the soft `preferOwnProvider` capability on and
|
|
487
|
+
// no manual pin, the chosen model's best route by the role's own value
|
|
488
|
+
// rides as that suffix, so the price the ranking assumed is the price the
|
|
489
|
+
// request pays. A manual `providerPin` is a hard gate and always wins; a
|
|
490
|
+
// model with no candidate route keeps default routing (bare selector).
|
|
491
|
+
const manualPin = def.providerPin;
|
|
492
|
+
const autoRoute = autoPinApplies(def) ? bestRoute(chosen.ranked.model, def) : null;
|
|
493
|
+
const pinSlug = manualPin ?? autoRoute?.providerSlug;
|
|
494
|
+
const pin = pinSlug !== undefined ? `@${pinSlug}` : "";
|
|
440
495
|
const chosenRow = rowById.get(chosen.catalogId);
|
|
441
496
|
// Append only a level the model's catalog thinking[] actually supports —
|
|
442
497
|
// omp clamps unsupported levels, so an unsupported pin would run at a
|
|
@@ -463,7 +518,7 @@ export async function runUpdater(trigger: Trigger, deps: Deps, opts?: { force?:
|
|
|
463
518
|
// came from. Values are built after the loop, once every key claim is known.
|
|
464
519
|
if (settings.writeFallbackChains) {
|
|
465
520
|
const key = suffix !== undefined && finalSelector.endsWith(`:${suffix}`) ? finalSelector.slice(0, -(suffix.length + 1)) : finalSelector;
|
|
466
|
-
chainPlanByRole[role] = { key, suffix, pin, pool, chosenIdx: pool.indexOf(chosen) };
|
|
521
|
+
chainPlanByRole[role] = { key, suffix, pin, pool, chosenIdx: pool.indexOf(chosen), def, autoPin: autoRoute !== null };
|
|
467
522
|
}
|
|
468
523
|
}
|
|
469
524
|
|
|
@@ -492,13 +547,41 @@ export async function runUpdater(trigger: Trigger, deps: Deps, opts?: { force?:
|
|
|
492
547
|
const cheaper = after.filter((p) => p.ranked.priceEff <= chosenPrice);
|
|
493
548
|
const pricier = after.filter((p) => p.ranked.priceEff > chosenPrice);
|
|
494
549
|
const chainPool = [...cheaper, ...pricier].slice(0, settings.fallbackChainDepth);
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
550
|
+
// A chain value's thinking level is appended only when the target's
|
|
551
|
+
// catalog entry advertises support (or the level is a meta level).
|
|
552
|
+
const levelFor = (catalogId: string): string => {
|
|
553
|
+
const row = rowById.get(catalogId);
|
|
554
|
+
return suffix !== undefined && row !== undefined && (META_LEVELS[suffix] === true || row.thinking.includes(suffix)) ? `:${suffix}` : "";
|
|
555
|
+
};
|
|
556
|
+
const values: string[] = [];
|
|
557
|
+
if (plan.autoPin) {
|
|
558
|
+
// Auto-pin chain (spec #58, expanded by #69): every gate-passing route
|
|
559
|
+
// of the primary model (all but the pinned best), then every
|
|
560
|
+
// gate-passing route of each fallback model, in the role's
|
|
561
|
+
// value-for-money score order (`rankedRoutes`, spec #71) — every entry
|
|
562
|
+
// bound to its route's provider, so a down primary
|
|
563
|
+
// provider falls back to the same model on another provider rather
|
|
564
|
+
// than a different model. `rankedRoutes` already applies every gate
|
|
565
|
+
// (non-degraded, endpoint filters, usable billed price, the
|
|
566
|
+
// maxPriceUsdPerM cap, the allowed-providers whitelist), so the chain
|
|
567
|
+
// carries exactly the routes the role can use. The dedup below
|
|
568
|
+
// collapses two routes that share a provider slug (same chain value),
|
|
569
|
+
// so the chain length is (R_primary − 1) + Σ R_fallback over distinct
|
|
570
|
+
// gate-passing provider slugs.
|
|
571
|
+
const chosenCandidate = plan.pool[plan.chosenIdx];
|
|
572
|
+
for (const r of rankedRoutes(chosenCandidate.ranked.model, plan.def).slice(1)) {
|
|
573
|
+
values.push(`openrouter/${chosenCandidate.catalogId}@${r.providerSlug}${levelFor(chosenCandidate.catalogId)}`);
|
|
574
|
+
}
|
|
575
|
+
for (const p of chainPool) {
|
|
576
|
+
for (const r of rankedRoutes(p.ranked.model, plan.def)) {
|
|
577
|
+
values.push(`openrouter/${p.catalogId}@${r.providerSlug}${levelFor(p.catalogId)}`);
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
} else {
|
|
581
|
+
// Manual pin (every value carries it) or no pin (bare values).
|
|
582
|
+
for (const p of chainPool) values.push(`openrouter/${p.catalogId}${plan.pin}${levelFor(p.catalogId)}`);
|
|
583
|
+
}
|
|
584
|
+
chainUpserts[plan.key] = [...new Set(values)].filter((v) => v !== plan.key);
|
|
502
585
|
referenced.add(plan.key);
|
|
503
586
|
}
|
|
504
587
|
// Locked roles keep their current chain: the plugin neither rewrites nor
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The per-role usage ledger — an append-only jsonl file beside the plugin's
|
|
3
|
+
* state, one row per recorded assistant turn, and the token profile derived
|
|
4
|
+
* from it.
|
|
5
|
+
*
|
|
6
|
+
* - `llm-role-usage.jsonl` — one row per turn (role, model, provider, token
|
|
7
|
+
* counts, cost). Bounded to `USAGE_LEDGER_CAP` rows: the normal path is a
|
|
8
|
+
* single append; an over-cap file is compacted to its newest rows via a temp
|
|
9
|
+
* file + atomic rename, so a crash mid-compaction leaves the original intact.
|
|
10
|
+
* - Recording is best-effort: a failure never breaks a session.
|
|
11
|
+
*
|
|
12
|
+
* The per-task price resolves its token profile from the recorded usage first
|
|
13
|
+
* (the role's median per-turn counts), then the user-set `tokenProfile` setting.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { appendFileSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
|
17
|
+
import { join } from "node:path";
|
|
18
|
+
import { quantile } from "./agent-create.ts";
|
|
19
|
+
import { isRecord } from "./guards.ts";
|
|
20
|
+
|
|
21
|
+
export type UsageRow = {
|
|
22
|
+
ts: string;
|
|
23
|
+
role: string;
|
|
24
|
+
model: string;
|
|
25
|
+
provider: string;
|
|
26
|
+
input: number;
|
|
27
|
+
output: number;
|
|
28
|
+
cacheRead: number;
|
|
29
|
+
cacheWrite: number;
|
|
30
|
+
cost: number;
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
export type TokenProfile = { inputTokens: number; outputTokens: number };
|
|
34
|
+
|
|
35
|
+
export const USAGE_LEDGER_FILE = "llm-role-usage.jsonl";
|
|
36
|
+
export const USAGE_LEDGER_CAP = 2000;
|
|
37
|
+
|
|
38
|
+
const STRING_FIELDS = ["ts", "role", "model", "provider"] as const;
|
|
39
|
+
const NUMBER_FIELDS = ["input", "output", "cacheRead", "cacheWrite", "cost"] as const;
|
|
40
|
+
|
|
41
|
+
/** A parsed line is a row only when every field is present with the right type;
|
|
42
|
+
* anything else is a malformed line the reader skips. */
|
|
43
|
+
function toUsageRow(v: unknown): UsageRow | null {
|
|
44
|
+
if (!isRecord(v)) return null;
|
|
45
|
+
for (const k of STRING_FIELDS) if (typeof v[k] !== "string") return null;
|
|
46
|
+
for (const k of NUMBER_FIELDS) if (typeof v[k] !== "number" || !Number.isFinite(v[k])) return null;
|
|
47
|
+
return {
|
|
48
|
+
ts: v.ts as string,
|
|
49
|
+
role: v.role as string,
|
|
50
|
+
model: v.model as string,
|
|
51
|
+
provider: v.provider as string,
|
|
52
|
+
input: v.input as number,
|
|
53
|
+
output: v.output as number,
|
|
54
|
+
cacheRead: v.cacheRead as number,
|
|
55
|
+
cacheWrite: v.cacheWrite as number,
|
|
56
|
+
cost: v.cost as number,
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Keep the newest `USAGE_LEDGER_CAP` rows: temp file + atomic rename, so a
|
|
61
|
+
* crash mid-compaction leaves the original file intact. */
|
|
62
|
+
function compactIfOverCap(dir: string, path: string): void {
|
|
63
|
+
const lines = readFileSync(path, "utf8")
|
|
64
|
+
.split("\n")
|
|
65
|
+
.filter((l) => l.length > 0);
|
|
66
|
+
if (lines.length <= USAGE_LEDGER_CAP) return;
|
|
67
|
+
const tmp = join(dir, `${USAGE_LEDGER_FILE}.tmp`);
|
|
68
|
+
writeFileSync(tmp, `${lines.slice(-USAGE_LEDGER_CAP).join("\n")}\n`);
|
|
69
|
+
renameSync(tmp, path);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** Append one row; best-effort — a recording failure never throws. */
|
|
73
|
+
export function appendUsageRow(dir: string, row: UsageRow): void {
|
|
74
|
+
try {
|
|
75
|
+
mkdirSync(dir, { recursive: true });
|
|
76
|
+
const path = join(dir, USAGE_LEDGER_FILE);
|
|
77
|
+
appendFileSync(path, `${JSON.stringify(row)}\n`);
|
|
78
|
+
compactIfOverCap(dir, path);
|
|
79
|
+
} catch {
|
|
80
|
+
// best-effort: never break a session over a ledger write
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** Tolerant read: a missing, unreadable, or malformed file degrades to []; an
|
|
85
|
+
* unparseable or wrongly-shaped line is skipped. */
|
|
86
|
+
export function readUsageRows(dir: string): UsageRow[] {
|
|
87
|
+
let text: string;
|
|
88
|
+
try {
|
|
89
|
+
text = readFileSync(join(dir, USAGE_LEDGER_FILE), "utf8");
|
|
90
|
+
} catch {
|
|
91
|
+
return [];
|
|
92
|
+
}
|
|
93
|
+
const rows: UsageRow[] = [];
|
|
94
|
+
for (const line of text.split("\n")) {
|
|
95
|
+
if (line.length === 0) continue;
|
|
96
|
+
let parsed: unknown;
|
|
97
|
+
try {
|
|
98
|
+
parsed = JSON.parse(line);
|
|
99
|
+
} catch {
|
|
100
|
+
continue;
|
|
101
|
+
}
|
|
102
|
+
const row = toUsageRow(parsed);
|
|
103
|
+
if (row) rows.push(row);
|
|
104
|
+
}
|
|
105
|
+
return rows;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/** The role's median per-turn input/output counts, or null when it has no rows.
|
|
109
|
+
* The median is `agent-create.ts`'s `quantile(values, 0.5)` — the repo's one
|
|
110
|
+
* quantile convention (linear interpolation), not a second implementation. */
|
|
111
|
+
export function roleTokenProfile(rows: readonly UsageRow[], role: string): TokenProfile | null {
|
|
112
|
+
const mine = rows.filter((r) => r.role === role);
|
|
113
|
+
if (mine.length === 0) return null;
|
|
114
|
+
return {
|
|
115
|
+
inputTokens: quantile(mine.map((r) => r.input), 0.5),
|
|
116
|
+
outputTokens: quantile(mine.map((r) => r.output), 0.5),
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** Precedence: the recorded usage, then the user-set setting, then null. */
|
|
121
|
+
export function resolveTokenProfile(
|
|
122
|
+
role: string,
|
|
123
|
+
rows: readonly UsageRow[],
|
|
124
|
+
setting: TokenProfile | null,
|
|
125
|
+
): TokenProfile | null {
|
|
126
|
+
return roleTokenProfile(rows, role) ?? setting;
|
|
127
|
+
}
|