model-compare-opencode-plugin 0.0.0-stage → 0.1.0

package/src/core/pins.ts

@@ -0,0 +1,158 @@
+// src/core/pins.ts
+// Pin/last-known-good state and pick resolution (design §7, §9). Pure
+// decision logic over a validated Catalog — no I/O, no OpenCode imports; the
+// caller supplies runnability, the stored pin, and the clock.
+
+import type { Catalog, Priority, Weights } from "./types"
+import { fnv1aHex } from "./hash"
+import { effectiveWeights, overallScore, rankModels } from "./rank"
+
+// One record per priority and effective-weights hash: both the pin and the
+// last-known-good pick (§7.2).
+export type Pin = { v: 1; id: string; since: number }
+
+export type ResolveInput = {
+  catalog: Catalog | null
+  p: Priority
+  override?: Partial<Weights>
+  // True when OpenCode lists `openrouter/<id>` as an enabled model (§7.1).
+  // Receives bare catalog ids.
+  runnable: (id: string) => boolean
+  // Config value like "openrouter/<id>", or null (§3).
+  fallback: string | null
+  stored: Pin | null
+  now: number
+  pinDays: number
+  minLead: number
+}
+
+export type ResolveResult = {
+  targetId: string | null
+  source: "pick" | "pin" | "fallback" | "none"
+  // The pin state after this resolution: a freshly adopted/kept record, or
+  // null only when nothing was adopted and nothing is stored (§7.8 keeps the
+  // fallback out of the pin; §9 skips, never deletes, an unrunnable record).
+  pin: Pin | null
+  notices: string[]
+}
+
+const PROVIDER_PREFIX = "openrouter/"
+const DAY_MS = 86_400_000
+
+// Config values look like "openrouter/<id>"; catalog ids are bare. Strip
+// exactly one leading prefix — a bare id may itself contain slashes
+// (provider/model), so nothing beyond the prefix is touched.
+function bareModelId(configValue: string): string {
+  return configValue.startsWith(PROVIDER_PREFIX)
+    ? configValue.slice(PROVIDER_PREFIX.length)
+    : configValue
+}
+
+export { fnv1aHex }
+
+export function pinKey(p: Priority, w: Weights): string {
+  return `pin/${p}/${fnv1aHex(JSON.stringify(w))}`
+}
+
+// §9's chain when there is no runnable pick from ranking: configured fallback
+// (checked by bare id) → none. `stored` rides along as the pin state: the
+// fallback itself is never stored as a pin (§7.8).
+function fallbackOrNone(
+  fallback: string | null,
+  runnable: (id: string) => boolean,
+  stored: Pin | null,
+  notices: string[],
+): ResolveResult {
+  if (fallback !== null) {
+    const bare = bareModelId(fallback)
+    if (runnable(bare)) {
+      notices.push(`fallback used: ${bare}`)
+      return { targetId: bare, source: "fallback", pin: stored, notices }
+    }
+  }
+  return { targetId: null, source: "none", pin: stored, notices }
+}
+
+export function resolve(input: ResolveInput): ResolveResult {
+  const { catalog, p, override, runnable, fallback, stored, now, pinDays, minLead } = input
+  const notices: string[] = []
+
+  // Without a fresh catalog (§9): stored pin (if runnable) → fallback (if
+  // runnable) → none. No rank-related notices — there is no ranking here.
+  if (catalog === null) {
+    if (stored && runnable(stored.id)) {
+      return { targetId: stored.id, source: "pin", pin: stored, notices }
+    }
+    if (stored) notices.push(`pin skipped: ${stored.id} is not runnable`)
+    return fallbackOrNone(fallback, runnable, stored, notices)
+  }
+
+  // Fresh catalog (§7.4): rank under the current weights, then walk the order
+  // for the first runnable candidate.
+  const eff = effectiveWeights(catalog, p, override)
+  // "Override ignored" notices: whatever the weight merge chose to ignore.
+  notices.push(...eff.warnings)
+  const ranked = rankModels(catalog, p, override).ids
+  const byId = new Map(catalog.models.map((m) => [m.id, m]))
+  const best = ranked.find((id) => runnable(id)) ?? null
+
+  // §9 when the fresh rules yield no runnable candidate: same chain as the
+  // no-catalog path.
+  if (best === null) {
+    if (stored && runnable(stored.id)) {
+      return { targetId: stored.id, source: "pin", pin: stored, notices }
+    }
+    if (stored) notices.push(`pin skipped: ${stored.id} is not runnable`)
+    return fallbackOrNone(fallback, runnable, stored, notices)
+  }
+
+  // (Re)adopt the best candidate; the target staying the stored pin id (only
+  // possible on the expiry re-adoption below) still counts as "pin".
+  const adopt = (id: string): ResolveResult => ({
+    targetId: id,
+    source: stored !== null && id === stored.id ? "pin" : "pick",
+    pin: { v: 1, id, since: now },
+    notices,
+  })
+
+  if (stored === null) return adopt(best)
+
+  const pinEntry = byId.get(stored.id) ?? null
+
+  // (a) The pinned model is not runnable, or has left the pool — §7.4 says
+  // switch to the best candidate now. (A ranking entry outside the pool could
+  // still name the pin id; that degenerates into a same-id re-adoption.)
+  if (!runnable(stored.id) || pinEntry === null) {
+    if (best !== stored.id) notices.push(`pin switched: ${stored.id} → ${best}`)
+    return adopt(best)
+  }
+
+  // (b) The pin is older than pinDays: re-adopt the best candidate — which
+  // may be the pinned model itself — and reset `since` (§7.4 "which may be
+  // the same model"; §7.6 "since resets every time a pin is adopted or
+  // re-adopted").
+  if (now - stored.since > pinDays * DAY_MS) {
+    if (best !== stored.id) notices.push(`pin switched: ${stored.id} → ${best}`)
+    return adopt(best)
+  }
+
+  // Nothing changed: keep the pin exactly as stored, clock untouched.
+  if (best === stored.id) {
+    return { targetId: stored.id, source: "pin", pin: stored, notices }
+  }
+
+  // (c) Switch only when the challenger leads by ≥ minLead; both scores come
+  // from the current catalog under the current effective weights (§7.4). A
+  // challenger missing from the pool can't be scored, so it can't lead.
+  const bestEntry = byId.get(best) ?? null
+  const lead =
+    bestEntry === null
+      ? Number.NEGATIVE_INFINITY
+      : overallScore(bestEntry, eff.weights) - overallScore(pinEntry, eff.weights)
+  if (lead >= minLead) {
+    notices.push(`pin switched: ${stored.id} → ${best}`)
+    return adopt(best)
+  }
+
+  return { targetId: stored.id, source: "pin", pin: stored, notices }
+}

package/src/core/rank.ts

@@ -0,0 +1,132 @@
+// src/core/rank.ts
+// Weight merging, scoring, and ranking (design §5, §6). Pure functions over a
+// validated Catalog — no I/O, no OpenCode imports.
+
+import type { Catalog, ModelEntry, Priority, Weights } from "./types"
+import { DEFAULT_WEIGHTS } from "./types"
+
+const WEIGHT_KEYS = ["quality", "price", "context", "age"] as const
+
+export type EffectiveWeights = {
+  weights: Weights
+  usedOverride: boolean
+  // Whether at least one override key actually landed in the merged map (an
+  // empty override, or one whose every key the published map's no-data signal
+  // rejected, applies nothing). `applied && !zeroSum` is the honest answer to
+  // "did the override take effect" — the report's "(custom weights)" label
+  // keys off exactly that (src/opencode/commands.ts).
+  applied: boolean
+  zeroSum: boolean
+  warnings: string[]
+}
+
+function sumOf(w: Weights): number {
+  return w.quality + w.price + w.context + w.age
+}
+
+export function effectiveWeights(
+  catalog: Catalog,
+  p: Priority,
+  override?: Partial<Weights>,
+): EffectiveWeights {
+  const warnings: string[] = []
+
+  // `parameters.weights` missing entirely leaves `catalog.weights` empty
+  // (validateCatalog): the pinned-tag defaults apply and overrides replace
+  // keys of those defaults freely. Otherwise the published map for this
+  // priority is the base, and a key it lacks means weight 0 — upstream
+  // removes a key entirely when no candidate has that data (design §5).
+  const hasParametersWeights = Object.keys(catalog.weights).length > 0
+  const published = hasParametersWeights ? (catalog.weights[p] ?? {}) : undefined
+  const base: Weights = published
+    ? {
+        quality: published.quality ?? 0,
+        price: published.price ?? 0,
+        context: published.context ?? 0,
+        age: published.age ?? 0,
+      }
+    : { ...DEFAULT_WEIGHTS[p] }
+
+  if (!override) {
+    return { weights: base, usedOverride: false, applied: false, zeroSum: sumOf(base) === 0, warnings }
+  }
+
+  // Replace the keys the user set. When a published map exists for this
+  // priority, a key it lacks (the no-data signal) is ignored with a
+  // warning; the other keys apply. With no published map at all (no
+  // `parameters.weights`), every override key applies to the DEFAULT base.
+  const merged: Weights = { ...base }
+  let applied = false
+  for (const key of WEIGHT_KEYS) {
+    const value = override[key]
+    if (value === undefined) continue
+    if (published && published[key] === undefined) {
+      warnings.push(
+        `ignoring ${key} weight override: the catalog's ${p} weights have no ${key} key (no data for it)`,
+      )
+      continue
+    }
+    merged[key] = value
+    applied = true
+  }
+
+  const total = sumOf(merged)
+  if (total === 0) {
+    // Runtime zero-sum (design §6): the merged map is a config error the
+    // schema can't catch; use the published (or DEFAULT) weights instead.
+    return { weights: base, usedOverride: true, applied, zeroSum: true, warnings }
+  }
+  const weights = {
+    quality: merged.quality / total,
+    price: merged.price / total,
+    context: merged.context / total,
+    age: merged.age / total,
+  }
+  return { weights, usedOverride: true, applied, zeroSum: false, warnings }
+}
+
+export function overallScore(entry: ModelEntry, w: Weights): number {
+  // Operand order matches upstream's scoring (design §6).
+  return (
+    w.quality * entry.scores.quality +
+    w.price * entry.scores.price +
+    w.context * entry.scores.context +
+    w.age * entry.scores.age
+  )
+}
+
+// Every plugin-side ordering sorts by (-score, -(quality ?? 0),
+// pricing.blended_per_1m ascending, id ascending), mirroring upstream's
+// ranking_key (design §6).
+function byRankingKey(score: (m: ModelEntry) => number): (a: ModelEntry, b: ModelEntry) => number {
+  return (a, b) => {
+    const byScore = score(b) - score(a)
+    if (byScore !== 0) return byScore
+    const byQuality = (b.quality ?? 0) - (a.quality ?? 0)
+    if (byQuality !== 0) return byQuality
+    const byPrice = a.pricing.blended_per_1m - b.pricing.blended_per_1m
+    if (byPrice !== 0) return byPrice
+    return a.id < b.id ? -1 : a.id > b.id ? 1 : 0
+  }
+}
+
+function rankBy(models: ModelEntry[], score: (m: ModelEntry) => number): string[] {
+  return [...models].sort(byRankingKey(score)).map((m) => m.id)
+}
+
+export function rankModels(
+  catalog: Catalog,
+  p: Priority,
+  override?: Partial<Weights>,
+): { ids: string[]; usedFallbackWeights: boolean } {
+  if (!override) {
+    const published = catalog.rankings[p]
+    if (published) return { ids: [...published], usedFallbackWeights: false }
+    // No published ranking: sort by the published overall score (design §6).
+    return { ids: rankBy(catalog.models, (m) => m.scores.overall[p]), usedFallbackWeights: false }
+  }
+  const eff = effectiveWeights(catalog, p, override)
+  // A zero-sum merged map falls back to the published (or DEFAULT) weights,
+  // which `effectiveWeights` returns in `weights` on that path.
+  return { ids: rankBy(catalog.models, (m) => overallScore(m, eff.weights)), usedFallbackWeights: eff.zeroSum }
+}

package/src/core/types.ts

@@ -0,0 +1,50 @@
+// src/core/types.ts
+// Shared vocabulary for the plugin's core (design §4): pure types plus the
+// pinned-tag weight defaults. No OpenCode imports, no I/O.
+
+export type Priority = "balanced" | "price" | "quality"
+
+export type Weights = { quality: number; price: number; context: number; age: number }
+
+export type ModelEntry = {
+  id: string
+  name: string
+  // The raw catalog carries the effective-tier input/output split alongside
+  // the blended value; validation copies the split through only when it is a
+  // finite number (anything else is dropped to undefined), so the split
+  // survives at runtime (optional for older/narrow documents).
+  pricing: {
+    input_per_1m?: number
+    output_per_1m?: number
+    blended_per_1m: number
+  }
+  context: number
+  quality: number | null
+  scores: {
+    quality: number
+    price: number
+    context: number
+    age: number
+    overall: Record<Priority, number>
+  }
+}
+
+export type Catalog = {
+  generatedAt: string
+  weights: Partial<Record<Priority, Partial<Weights>>>
+  rankings: Partial<Record<Priority, string[]>>
+  models: ModelEntry[]
+}
+
+// Upstream.s PRIORITY_WEIGHTS at the pinned tag (v0.3.1; unchanged since v0.3.0). Used when a
+// document carries no `parameters.weights` at all (design §5); individual
+// missing keys fall back to weight 0 instead.
+export const DEFAULT_WEIGHTS: Record<Priority, Weights> = {
+  balanced: { quality: 0.4, price: 0.4, context: 0.1, age: 0.1 },
+  price: { quality: 0.2, price: 0.6, context: 0.1, age: 0.1 },
+  quality: { quality: 0.6, price: 0.2, context: 0.1, age: 0.1 },
+}
+
+// A validated catalog plus when the plugin itself accepted it. The plugin's
+// own clock, not `generated_at`, is the data age (design §5, §9).
+export type Sourced = { doc: Catalog; acquiredAt: number }