omp-llm-role 1.2.0 → 1.4.0

package/src/usage-ledger.ts

@@ -0,0 +1,127 @@
+/**
+ * The per-role usage ledger — an append-only jsonl file beside the plugin's
+ * state, one row per recorded assistant turn, and the token profile derived
+ * from it.
+ *
+ * - `llm-role-usage.jsonl` — one row per turn (role, model, provider, token
+ *   counts, cost). Bounded to `USAGE_LEDGER_CAP` rows: the normal path is a
+ *   single append; an over-cap file is compacted to its newest rows via a temp
+ *   file + atomic rename, so a crash mid-compaction leaves the original intact.
+ * - Recording is best-effort: a failure never breaks a session.
+ *
+ * The per-task price resolves its token profile from the recorded usage first
+ * (the role's median per-turn counts), then the user-set `tokenProfile` setting.
+ */
+
+import { appendFileSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { quantile } from "./agent-create.ts";
+import { isRecord } from "./guards.ts";
+
+export type UsageRow = {
+  ts: string;
+  role: string;
+  model: string;
+  provider: string;
+  input: number;
+  output: number;
+  cacheRead: number;
+  cacheWrite: number;
+  cost: number;
+};
+
+export type TokenProfile = { inputTokens: number; outputTokens: number };
+
+export const USAGE_LEDGER_FILE = "llm-role-usage.jsonl";
+export const USAGE_LEDGER_CAP = 2000;
+
+const STRING_FIELDS = ["ts", "role", "model", "provider"] as const;
+const NUMBER_FIELDS = ["input", "output", "cacheRead", "cacheWrite", "cost"] as const;
+
+/** A parsed line is a row only when every field is present with the right type;
+ *  anything else is a malformed line the reader skips. */
+function toUsageRow(v: unknown): UsageRow | null {
+  if (!isRecord(v)) return null;
+  for (const k of STRING_FIELDS) if (typeof v[k] !== "string") return null;
+  for (const k of NUMBER_FIELDS) if (typeof v[k] !== "number" || !Number.isFinite(v[k])) return null;
+  return {
+    ts: v.ts as string,
+    role: v.role as string,
+    model: v.model as string,
+    provider: v.provider as string,
+    input: v.input as number,
+    output: v.output as number,
+    cacheRead: v.cacheRead as number,
+    cacheWrite: v.cacheWrite as number,
+    cost: v.cost as number,
+  };
+}
+
+/** Keep the newest `USAGE_LEDGER_CAP` rows: temp file + atomic rename, so a
+ *  crash mid-compaction leaves the original file intact. */
+function compactIfOverCap(dir: string, path: string): void {
+  const lines = readFileSync(path, "utf8")
+    .split("\n")
+    .filter((l) => l.length > 0);
+  if (lines.length <= USAGE_LEDGER_CAP) return;
+  const tmp = join(dir, `${USAGE_LEDGER_FILE}.tmp`);
+  writeFileSync(tmp, `${lines.slice(-USAGE_LEDGER_CAP).join("\n")}\n`);
+  renameSync(tmp, path);
+}
+
+/** Append one row; best-effort — a recording failure never throws. */
+export function appendUsageRow(dir: string, row: UsageRow): void {
+  try {
+    mkdirSync(dir, { recursive: true });
+    const path = join(dir, USAGE_LEDGER_FILE);
+    appendFileSync(path, `${JSON.stringify(row)}\n`);
+    compactIfOverCap(dir, path);
+  } catch {
+    // best-effort: never break a session over a ledger write
+  }
+}
+
+/** Tolerant read: a missing, unreadable, or malformed file degrades to []; an
+ *  unparseable or wrongly-shaped line is skipped. */
+export function readUsageRows(dir: string): UsageRow[] {
+  let text: string;
+  try {
+    text = readFileSync(join(dir, USAGE_LEDGER_FILE), "utf8");
+  } catch {
+    return [];
+  }
+  const rows: UsageRow[] = [];
+  for (const line of text.split("\n")) {
+    if (line.length === 0) continue;
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    const row = toUsageRow(parsed);
+    if (row) rows.push(row);
+  }
+  return rows;
+}
+
+/** The role's median per-turn input/output counts, or null when it has no rows.
+ *  The median is `agent-create.ts`'s `quantile(values, 0.5)` — the repo's one
+ *  quantile convention (linear interpolation), not a second implementation. */
+export function roleTokenProfile(rows: readonly UsageRow[], role: string): TokenProfile | null {
+  const mine = rows.filter((r) => r.role === role);
+  if (mine.length === 0) return null;
+  return {
+    inputTokens: quantile(mine.map((r) => r.input), 0.5),
+    outputTokens: quantile(mine.map((r) => r.output), 0.5),
+  };
+}
+
+/** Precedence: the recorded usage, then the user-set setting, then null. */
+export function resolveTokenProfile(
+  role: string,
+  rows: readonly UsageRow[],
+  setting: TokenProfile | null,
+): TokenProfile | null {
+  return roleTokenProfile(rows, role) ?? setting;
+}

package/web/app.js

@@ -23,6 +23,11 @@ const state = {
   lockPath: "",
   scopes: [],
   activeScope: "",
+  // The ranked pool the server is serving (raw | as-written) and the user-set
+  // token profile the per-task price falls back to (null = unset). Both come
+  // from the bootstrap payload, so a refresh keeps them.
+  poolMode: "raw",
+  tokenProfile: null,
   features: {},
   featureRegistry: [],
   filter: "",
@@ -82,6 +87,26 @@ function fmtCtx(value) {
   return value >= 1000 ? Math.round(value / 1000) + "K" : String(value);
 }
 
+/** Dollars per task, at a precision that keeps the small values readable: a
+ * per-task price spans roughly $0.0001 (a cheap model, a short task) to $1+. */
+function fmtUsd(value) {
+  if (value === null || value === undefined) return "—";
+  const abs = Math.abs(value);
+  if (abs >= 1) return "$" + value.toFixed(2);
+  if (abs >= 0.01) return "$" + value.toFixed(3);
+  if (abs > 0) return "$" + value.toFixed(4);
+  return "$0";
+}
+
+/** Token counts, compact: 20000 → "20K", 1500 → "1.5K", 800 → "800". */
+function fmtTokens(value) {
+  if (value >= 1000) {
+    const k = value / 1000;
+    return (k >= 10 ? Math.round(k) : Math.round(k * 10) / 10) + "K";
+  }
+  return String(value);
+}
+
 function deltaText(delta) {
   if (delta === null || delta === undefined || delta === 0) return "·";
   return delta > 0 ? "+" + delta : "−" + Math.abs(delta);
@@ -121,6 +146,7 @@ const TIPS = {
   hideBlocked: "Drop rows your OpenRouter key cannot run (key = blocked). Applied after the text filter and the top-n slice.",
   lock: "Plugin settings lock file that Export writes to (the active scope's lock file).",
   scope: "Role-config source: the user-level scope, or a project where /project-roles was used. Switching re-reads that scope's roles and universe from disk and points Export at its lock file. A project whose lock file is gone is unavailable.",
+  pool: "Ranked pool. raw = the full cached dataset (account-agnostic). as-written = the same catalog/tier gate and allowed-providers whitelist prune the updater applies before writing, so the leader you see is the model you will get. Best-effort: a failed whitelist fetch degrades to the raw pool. Export is unaffected — it always writes the authored role.",
   refresh: "Refetch llm-stats and OpenRouter data (hits the network).",
   rank: "Rank among this role's eligible models, by value, descending.",
   delta: "Rank change against the role's saved (effective) definition: + means this definition ranks the model higher than the plugin does today.",
@@ -128,6 +154,7 @@ const TIPS = {
   value: "Sort key: value = q − λ·$/M on the thinking-adjusted effective price. λ is the price of one quality point, in $/M.",
   q: "Quality q = Σ (wᵢ/(1−w_price))·tᵢ over the weighted quality metrics. Price is not blended in — it is the penalty axis.",
   price: "Effective price in $/M: OpenRouter standard-route blend (3:1 in:out) scaled by the role's thinking factor when the model supports thinking; unadjusted for bare/off roles.",
+  perTask: "Dollars per task: your own token mix (the role's recorded usage, else the tokenProfile setting) × the role's priced route, thinking-adjusted. Display-only — the ranking still sorts on $/M. — when no token profile is available (no recorded usage and no tokenProfile set).",
   throughput: "Output throughput in tok/s (OpenRouter p50 when matched, else llm-stats).",
   context: "Context window in tokens.",
   explainSub: "value = q − λ·$/M (effective price), where q is the weighted quality sum and λ the price of one quality point.",
@@ -141,6 +168,9 @@ const TIPS = {
   whyNotHigher: "The value gap to the model ranked above, and the per-metric raw target that would close it (inverse of the cardinal transform).",
   dominated: "Models that are both cheaper and at least as good on q.",
   lambda: "λ = price of one quality point, in $/M. Derived as " + LAMBDA_DERIVATION + " unless the role overrides lambda.",
+  leaderFlip: "The price weight w* at which a cheaper challenger overtakes the value leader: λ* = min over cheaper challengers X of (q_L − q_X)/(pEff_L − pEff_X), then w* = 20λ*/(1+20λ*). Absent when no cheaper challenger exists. Exact for the 1/price² blend or a hard provider pin; a first-order estimate under the soft best-route basis (the route choice itself moves with the price weight).",
+  levers: "Two price-lever readouts, computed on the pool you are viewing: the leader-flip threshold (the price weight at which a cheaper challenger overtakes the leader) and the thinking level's effect on the leader. Each is absent when the server sends null — no cheaper challenger, or a bare role.",
+  thinkingLevels: "The leader at each thinking level, with the level's price factor (3ρ+1+T)/(3ρ+1). The factor multiplies the price axis, so it penalizes expensive models more in absolute λ-units — that is why auto favours the cheap model. Absent for a bare role (no thinking level).",
   sum: "Weights must sum to 1.0 (±0.01) or the plugin rejects the role.",
   required: "Eligibility gate, not a weight: a model missing this metric is not ranked at all for the role.",
   focus: "Nine-axis audit of each weighted benchmark/percentile metric, over the same pool the updater ranks on. A below-bar axis is warned; unknown means the metric's scores were not loaded (never a silent ok).",
@@ -155,6 +185,7 @@ const TIPS = {
   focusCrossSource: "Cross-source: the zeroeval per-entry price/context compared against the plugin's own price/context for the models carrying the metric (the median relative divergence, within a 0.5 tolerance). The speed figure is informational only (rps vs tok/s). An annotation, never a gate — it never feeds the 1/price² blend; unknown means no model joined or the payload carried none of the four fields.",
   imageFilter: "Require image input (filters.image) — the gate that shrinks the vision role's eligible set.",
   providerPin: "Hard provider pin: the OpenRouter provider slug this role's requests are routed to (a tiered slug like deepinfra/fp8 is accepted verbatim). Setting it opts the role out of the automatic best-route pin; empty = default OpenRouter routing.",
+  autoPin: "Automatic provider pin (candidate): this model's best route by the role's value-for-money score — the first route the updater's daily run tries. The updater probes that route with a one-token request bound to it (provider: { only: [slug] }) and skips to the next route when OpenRouter does not serve it, dropping the pin entirely (bare selector, OpenRouter's own routing) when none is routable — so the pin that gets written can be a later route, or absent.",
   features: "Opt-in capability presets (issue #40). Each flag applies the plugin's recommended settings for that capability to this role; an explicit knob value always wins. Export writes the flag, not the expanded values, so a future change to the recommendation keeps reaching the role.",
   featureState: "inherit = use the global flag (features.<id> in the lock file); on/off = a per-role override (roles.<role>.features.<id>) that beats the global in either direction.",
   featureKnob: "An individual knob the capability's recommended bundle fills. Leave it at inherit to take the recommendation; set a value to override it (an explicit value always wins over the flag).",
@@ -267,6 +298,20 @@ function scopeSelect() {
   return sel;
 }
 
+/** The pool selector: the raw dataset (account-agnostic) or the as-written pool
+ * (the catalog/tier gate + allowed-providers prune the updater ranks on). The
+ * value is the server's mode, so the header always names the active pool. */
+function poolSelect() {
+  const sel = el("select", { id: "pool", "data-tip": TIPS.pool });
+  sel.append(
+    el("option", { value: "raw", text: "raw pool" }),
+    el("option", { value: "as-written", text: "as-written pool" }),
+  );
+  sel.value = state.poolMode;
+  sel.addEventListener("change", onPoolChange);
+  return sel;
+}
+
 function renderMeta(data) {
   const meta = document.getElementById("meta");
   meta.textContent = "";
@@ -275,6 +320,8 @@ function renderMeta(data) {
     el("span", { class: "sep", text: "·" }),
     scopeSelect(),
     el("span", { class: "sep", text: "·" }),
+    poolSelect(),
+    el("span", { class: "sep", text: "·" }),
     el("span", { "data-tip": TIPS.models, text: data.modelCount + " models" }),
     el("span", { class: "sep", text: "·" }),
     el("span", { "data-tip": TIPS.fetched, text: "fetched " + data.fetchedAt.slice(0, 10) }),
@@ -426,6 +473,7 @@ function renderTable() {
       el("th", { class: "num", "data-tip": TIPS.value, text: "value" }),
       el("th", { class: "num", "data-tip": TIPS.q, text: "q" }),
       el("th", { class: "num", "data-tip": TIPS.price, text: "$/M" }),
+      el("th", { class: "num", "data-tip": TIPS.perTask, text: "$/task" }),
       el("th", { class: "num", "data-tip": TIPS.throughput, text: "tok/s" }),
       el("th", { class: "num", "data-tip": TIPS.context, text: "ctx" }),
     ])),
@@ -450,6 +498,7 @@ function renderTable() {
         el("td", { class: "num", text: fmt(r.value, 3) }),
         el("td", { class: "num", text: fmt(r.q, 3) }),
         el("td", { class: "num", text: fmt(r.priceEff, 2) }),
+        el("td", { class: "num", text: fmtUsd(r.perTask) }),
         el("td", { class: "num", text: r.throughput === null ? "—" : fmt(r.throughput, 0) }),
         el("td", { class: "num", text: r.context === null ? "—" : fmtCtx(r.context) }),
       ]),
@@ -514,7 +563,12 @@ function renderExplain() {
   if (manualPin) {
     panel.append(el("p", { class: "sub pin-line", "data-tip": TIPS.providerPin, text: "pin: " + manualPin + " (manual)" }));
   } else if (ex.autoPin) {
-    panel.append(el("p", { class: "sub pin-line", "data-tip": TIPS.providerPin, text: "auto pin: " + ex.autoPin }));
+    // The candidate route — the first route the updater's value-ordered walk
+    // tries — not a promise of the written pin: the updater probes it and skips
+    // to the next route when OpenRouter does not serve it, or drops the pin
+    // entirely when none is routable (spec #90). Read-only, offline: the
+    // explorer cannot probe, so it names the candidate and says so in the tip.
+    panel.append(el("p", { class: "sub pin-line", "data-tip": TIPS.autoPin, text: "auto pin (candidate): " + ex.autoPin }));
   }
 
   panel.append(el("h3", { "data-tip": TIPS.composition, text: "Value composition" }));
@@ -562,6 +616,23 @@ function renderExplain() {
   const ownNote = ex.ownProvider ? " · best route" : "";
   panel.append(el("p", { class: "cost", "data-tip": TIPS.costPenalty, text: "eff price $" + fmt(ex.cost.priceEff, 2) + "/M" + (ex.cost.priceEff !== ex.cost.billedPrice ? " (billed $" + fmt(ex.cost.billedPrice, 2) + " × " + thinkNote + ")" : " (" + thinkNote + ")") + cacheNote + ownNote + " · penalty = λ·price = " + fmt(ex.cost.penalty, 4) + " · value = q − penalty = " + fmt(ex.cost.value, 4) }));
 
+  // The per-task price breakdown: display-only, so it sits apart from the
+  // ranking's cost line. A muted note when no token profile resolved (never a
+  // silent zero).
+  panel.append(el("h3", { "data-tip": TIPS.perTask, text: "Per-task price" }));
+  if (ex.perTask) {
+    const p = ex.perTask;
+    panel.append(
+      el("p", {
+        class: "cost",
+        "data-tip": TIPS.perTask,
+        text: "per task: " + fmtTokens(p.inputTokens) + " in × $" + fmt(p.inputPrice, 3) + "/M + " + fmtTokens(p.outputTokens) + " out × $" + fmt(p.outputPrice, 3) + "/M × " + p.thinkingFactor.toFixed(3) + " = " + fmtUsd(p.total),
+      }),
+    );
+  } else {
+    panel.append(el("p", { class: "cost muted", "data-tip": TIPS.perTask, text: "per task: — (no token profile — record usage or set tokenProfile)" }));
+  }
+
   panel.append(el("h3", { "data-tip": TIPS.whyNotHigher, text: "Why not higher" }));
   if (ex.gapAbove === null) {
     panel.append(el("p", { class: "muted", text: "Already #1 for this role." }));
@@ -593,6 +664,53 @@ function renderExplain() {
     for (const d of ex.dominators) ul.append(el("li", { text: d.name + " — $" + fmt(d.priceEff, 2) + "/M, q " + fmt(d.q, 3) }));
     panel.append(ul);
   }
+
+  // The two price-lever readouts are role-level (computed on the pool the
+  // ranking above uses), so they close the panel. Each is absent when the
+  // server sends null: no cheaper challenger, or a bare role.
+  const flip = ex.leaderFlip;
+  const levels = ex.thinkingLevels;
+  if (flip || levels) {
+    panel.append(el("h3", { "data-tip": TIPS.levers, text: "Price levers" }));
+    if (flip) {
+      const ratio = flip.wPrice > 0 ? (flip.flipW / flip.wPrice).toFixed(2) + "×" : "—";
+      panel.append(
+        el("p", {
+          class: "cost",
+          "data-tip": TIPS.leaderFlip,
+          text: "leader flip: w* = " + flip.flipW.toFixed(3) + " · now " + flip.wPrice.toFixed(3) + " · distance " + (flip.distance >= 0 ? "+" : "−") + Math.abs(flip.distance).toFixed(3) + " (" + ratio + ")",
+        }),
+      );
+      panel.append(
+        el("p", {
+          class: "cost",
+          text: "challenger: " + flip.challenger.name + " — $" + fmt(flip.challenger.priceEff, 2) + "/M, value " + fmt(flip.challenger.value, 3),
+        }),
+      );
+    }
+    if (levels) {
+      const t = el("table", { class: "comp" });
+      t.append(
+        el("thead", {}, el("tr", {}, [
+          el("th", { text: "level" }),
+          el("th", { class: "num", "data-tip": TIPS.thinking, text: "×price" }),
+          el("th", { "data-tip": TIPS.thinkingLevels, text: "leader" }),
+        ])),
+      );
+      const tb = el("tbody");
+      for (const row of levels) {
+        tb.append(
+          el("tr", { class: row.level === ex.role.thinking ? "active-level" : "" }, [
+            el("td", { text: row.level }),
+            el("td", { class: "num", text: "×" + row.factor.toFixed(3) }),
+            el("td", { text: row.leader ? row.leader.name + " (" + fmt(row.leader.value, 3) + ")" : "—" }),
+          ]),
+        );
+      }
+      t.append(tb);
+      panel.append(t);
+    }
+  }
 }
 
 // ---------------------------------------------------------------------------
@@ -1185,6 +1303,24 @@ async function onScopeChange(event) {
   }
 }
 
+/** Switch the ranked pool (raw ↔ as-written). Unsaved edits are confirmed first
+ * (mirroring the scope switcher); the returned payload rebuilds every panel, so
+ * the header's model count and the table describe the new pool. */
+async function onPoolChange(event) {
+  const mode = event.target.value;
+  if (mode === state.poolMode) return;
+  if (Object.keys(dirtyRoles()).length > 0 && !confirm("Discard unsaved role edits and switch pool?")) {
+    event.target.value = state.poolMode;
+    return;
+  }
+  try {
+    applyBootstrap(await api("/api/pool", { mode }));
+  } catch (err) {
+    alert("Pool switch failed: " + err.message);
+    event.target.value = state.poolMode;
+  }
+}
+
 // ---------------------------------------------------------------------------
 // Boot
 // ---------------------------------------------------------------------------
@@ -1196,6 +1332,8 @@ function applyBootstrap(data) {
   const previousRole = state.role;
   state.scopes = data.scopes || [];
   state.activeScope = data.activeScope;
+  state.poolMode = data.poolMode || "raw";
+  state.tokenProfile = data.tokenProfile ?? null;
   state.universe = data.universe;
   state.effective = {};
   for (const [name, entry] of Object.entries(state.universe)) state.effective[name] = entry.def;

package/web/style.css

@@ -39,6 +39,7 @@ body {
 #meta .sep { color: var(--line); }
 #meta .lock { color: var(--muted); font-family: var(--mono); font-size: 11px; }
 #meta #scope { max-width: 340px; }
+#meta #pool { max-width: 200px; }
 #meta button { margin-left: auto; }
 
 #roles {
@@ -189,6 +190,8 @@ table.comp th { text-align: left; color: var(--muted); font-weight: 500; font-si
 table.comp td.num, table.comp th.num { text-align: right; font-family: var(--mono); font-variant-numeric: tabular-nums; }
 table.comp tr.missing-row td { color: var(--muted); }
 table.comp tr.note-row td { border-bottom: none; font-size: 11px; padding-top: 0; }
+/* The thinking-level readout: the role's current level is the row that matters. */
+table.comp tr.active-level td { color: var(--accent); }
 
 /* Focus-metric audit: one row per weighted benchmark/percentile metric, four
    signal cells. ok = up, below-bar = warn + glyph, unknown = muted (never ok). */