model-compare-opencode-plugin 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +674 -0
- package/README.md +186 -2
- package/dist/tui.js +741 -0
- package/package.json +60 -4
- package/src/config.ts +197 -0
- package/src/core/catalog.ts +189 -0
- package/src/core/hash.ts +12 -0
- package/src/core/pins.ts +158 -0
- package/src/core/rank.ts +132 -0
- package/src/core/types.ts +50 -0
- package/src/index.ts +478 -0
- package/src/opencode/aliases.ts +58 -0
- package/src/opencode/commands.ts +251 -0
- package/src/opencode/guard.ts +44 -0
- package/src/opencode/refresh.ts +188 -0
- package/src/opencode/resolution.ts +69 -0
- package/src/opencode/storage.ts +97 -0
- package/src/opencode/usebest.ts +121 -0
- package/src/rpc.ts +78 -0
- package/src/sources/index.ts +115 -0
- package/src/sources/local.ts +175 -0
- package/src/sources/remote.ts +155 -0
- package/src/tui.tsx +772 -0
- package/vendor/UPSTREAM.json +6 -0
- package/vendor/model_compare.py +1950 -0
package/src/core/rank.ts
ADDED
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
// src/core/rank.ts
|
|
2
|
+
// Weight merging, scoring, and ranking (design §5, §6). Pure functions over a
|
|
3
|
+
// validated Catalog — no I/O, no OpenCode imports.
|
|
4
|
+
|
|
5
|
+
import type { Catalog, ModelEntry, Priority, Weights } from "./types"
|
|
6
|
+
import { DEFAULT_WEIGHTS } from "./types"
|
|
7
|
+
|
|
8
|
+
const WEIGHT_KEYS = ["quality", "price", "context", "age"] as const
|
|
9
|
+
|
|
10
|
+
export type EffectiveWeights = {
|
|
11
|
+
weights: Weights
|
|
12
|
+
usedOverride: boolean
|
|
13
|
+
// Whether at least one override key actually landed in the merged map (an
|
|
14
|
+
// empty override, or one whose every key the published map's no-data signal
|
|
15
|
+
// rejected, applies nothing). `applied && !zeroSum` is the honest answer to
|
|
16
|
+
// "did the override take effect" — the report's "(custom weights)" label
|
|
17
|
+
// keys off exactly that (src/opencode/commands.ts).
|
|
18
|
+
applied: boolean
|
|
19
|
+
zeroSum: boolean
|
|
20
|
+
warnings: string[]
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function sumOf(w: Weights): number {
|
|
24
|
+
return w.quality + w.price + w.context + w.age
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function effectiveWeights(
|
|
28
|
+
catalog: Catalog,
|
|
29
|
+
p: Priority,
|
|
30
|
+
override?: Partial<Weights>,
|
|
31
|
+
): EffectiveWeights {
|
|
32
|
+
const warnings: string[] = []
|
|
33
|
+
|
|
34
|
+
// `parameters.weights` missing entirely leaves `catalog.weights` empty
|
|
35
|
+
// (validateCatalog): the pinned-tag defaults apply and overrides replace
|
|
36
|
+
// keys of those defaults freely. Otherwise the published map for this
|
|
37
|
+
// priority is the base, and a key it lacks means weight 0 — upstream
|
|
38
|
+
// removes a key entirely when no candidate has that data (design §5).
|
|
39
|
+
const hasParametersWeights = Object.keys(catalog.weights).length > 0
|
|
40
|
+
const published = hasParametersWeights ? (catalog.weights[p] ?? {}) : undefined
|
|
41
|
+
const base: Weights = published
|
|
42
|
+
? {
|
|
43
|
+
quality: published.quality ?? 0,
|
|
44
|
+
price: published.price ?? 0,
|
|
45
|
+
context: published.context ?? 0,
|
|
46
|
+
age: published.age ?? 0,
|
|
47
|
+
}
|
|
48
|
+
: { ...DEFAULT_WEIGHTS[p] }
|
|
49
|
+
|
|
50
|
+
if (!override) {
|
|
51
|
+
return { weights: base, usedOverride: false, applied: false, zeroSum: sumOf(base) === 0, warnings }
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// Replace the keys the user set. When a published map exists for this
|
|
55
|
+
// priority, a key it lacks (the no-data signal) is ignored with a
|
|
56
|
+
// warning; the other keys apply. With no published map at all (no
|
|
57
|
+
// `parameters.weights`), every override key applies to the DEFAULT base.
|
|
58
|
+
const merged: Weights = { ...base }
|
|
59
|
+
let applied = false
|
|
60
|
+
for (const key of WEIGHT_KEYS) {
|
|
61
|
+
const value = override[key]
|
|
62
|
+
if (value === undefined) continue
|
|
63
|
+
if (published && published[key] === undefined) {
|
|
64
|
+
warnings.push(
|
|
65
|
+
`ignoring ${key} weight override: the catalog's ${p} weights have no ${key} key (no data for it)`,
|
|
66
|
+
)
|
|
67
|
+
continue
|
|
68
|
+
}
|
|
69
|
+
merged[key] = value
|
|
70
|
+
applied = true
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const total = sumOf(merged)
|
|
74
|
+
if (total === 0) {
|
|
75
|
+
// Runtime zero-sum (design §6): the merged map is a config error the
|
|
76
|
+
// schema can't catch; use the published (or DEFAULT) weights instead.
|
|
77
|
+
return { weights: base, usedOverride: true, applied, zeroSum: true, warnings }
|
|
78
|
+
}
|
|
79
|
+
const weights = {
|
|
80
|
+
quality: merged.quality / total,
|
|
81
|
+
price: merged.price / total,
|
|
82
|
+
context: merged.context / total,
|
|
83
|
+
age: merged.age / total,
|
|
84
|
+
}
|
|
85
|
+
return { weights, usedOverride: true, applied, zeroSum: false, warnings }
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
export function overallScore(entry: ModelEntry, w: Weights): number {
|
|
89
|
+
// Operand order matches upstream's scoring (design §6).
|
|
90
|
+
return (
|
|
91
|
+
w.quality * entry.scores.quality +
|
|
92
|
+
w.price * entry.scores.price +
|
|
93
|
+
w.context * entry.scores.context +
|
|
94
|
+
w.age * entry.scores.age
|
|
95
|
+
)
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// Every plugin-side ordering sorts by (-score, -(quality ?? 0),
|
|
99
|
+
// pricing.blended_per_1m ascending, id ascending), mirroring upstream's
|
|
100
|
+
// ranking_key (design §6).
|
|
101
|
+
function byRankingKey(score: (m: ModelEntry) => number): (a: ModelEntry, b: ModelEntry) => number {
|
|
102
|
+
return (a, b) => {
|
|
103
|
+
const byScore = score(b) - score(a)
|
|
104
|
+
if (byScore !== 0) return byScore
|
|
105
|
+
const byQuality = (b.quality ?? 0) - (a.quality ?? 0)
|
|
106
|
+
if (byQuality !== 0) return byQuality
|
|
107
|
+
const byPrice = a.pricing.blended_per_1m - b.pricing.blended_per_1m
|
|
108
|
+
if (byPrice !== 0) return byPrice
|
|
109
|
+
return a.id < b.id ? -1 : a.id > b.id ? 1 : 0
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function rankBy(models: ModelEntry[], score: (m: ModelEntry) => number): string[] {
|
|
114
|
+
return [...models].sort(byRankingKey(score)).map((m) => m.id)
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
export function rankModels(
|
|
118
|
+
catalog: Catalog,
|
|
119
|
+
p: Priority,
|
|
120
|
+
override?: Partial<Weights>,
|
|
121
|
+
): { ids: string[]; usedFallbackWeights: boolean } {
|
|
122
|
+
if (!override) {
|
|
123
|
+
const published = catalog.rankings[p]
|
|
124
|
+
if (published) return { ids: [...published], usedFallbackWeights: false }
|
|
125
|
+
// No published ranking: sort by the published overall score (design §6).
|
|
126
|
+
return { ids: rankBy(catalog.models, (m) => m.scores.overall[p]), usedFallbackWeights: false }
|
|
127
|
+
}
|
|
128
|
+
const eff = effectiveWeights(catalog, p, override)
|
|
129
|
+
// A zero-sum merged map falls back to the published (or DEFAULT) weights,
|
|
130
|
+
// which `effectiveWeights` returns in `weights` on that path.
|
|
131
|
+
return { ids: rankBy(catalog.models, (m) => overallScore(m, eff.weights)), usedFallbackWeights: eff.zeroSum }
|
|
132
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
// src/core/types.ts
|
|
2
|
+
// Shared vocabulary for the plugin's core (design §4): pure types plus the
|
|
3
|
+
// pinned-tag weight defaults. No OpenCode imports, no I/O.
|
|
4
|
+
|
|
5
|
+
export type Priority = "balanced" | "price" | "quality"
|
|
6
|
+
|
|
7
|
+
export type Weights = { quality: number; price: number; context: number; age: number }
|
|
8
|
+
|
|
9
|
+
export type ModelEntry = {
|
|
10
|
+
id: string
|
|
11
|
+
name: string
|
|
12
|
+
// The raw catalog carries the effective-tier input/output split alongside
|
|
13
|
+
// the blended value; validation copies the split through only when it is a
|
|
14
|
+
// finite number (anything else is dropped to undefined), so the split
|
|
15
|
+
// survives at runtime (optional for older/narrow documents).
|
|
16
|
+
pricing: {
|
|
17
|
+
input_per_1m?: number
|
|
18
|
+
output_per_1m?: number
|
|
19
|
+
blended_per_1m: number
|
|
20
|
+
}
|
|
21
|
+
context: number
|
|
22
|
+
quality: number | null
|
|
23
|
+
scores: {
|
|
24
|
+
quality: number
|
|
25
|
+
price: number
|
|
26
|
+
context: number
|
|
27
|
+
age: number
|
|
28
|
+
overall: Record<Priority, number>
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export type Catalog = {
|
|
33
|
+
generatedAt: string
|
|
34
|
+
weights: Partial<Record<Priority, Partial<Weights>>>
|
|
35
|
+
rankings: Partial<Record<Priority, string[]>>
|
|
36
|
+
models: ModelEntry[]
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// Upstream.s PRIORITY_WEIGHTS at the pinned tag (v0.3.1; unchanged since v0.3.0). Used when a
|
|
40
|
+
// document carries no `parameters.weights` at all (design §5); individual
|
|
41
|
+
// missing keys fall back to weight 0 instead.
|
|
42
|
+
export const DEFAULT_WEIGHTS: Record<Priority, Weights> = {
|
|
43
|
+
balanced: { quality: 0.4, price: 0.4, context: 0.1, age: 0.1 },
|
|
44
|
+
price: { quality: 0.2, price: 0.6, context: 0.1, age: 0.1 },
|
|
45
|
+
quality: { quality: 0.6, price: 0.2, context: 0.1, age: 0.1 },
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// A validated catalog plus when the plugin itself accepted it. The plugin's
|
|
49
|
+
// own clock, not `generated_at`, is the data age (design §5, §9).
|
|
50
|
+
export type Sourced = { doc: Catalog; acquiredAt: number }
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,478 @@
|
|
|
1
|
+
// src/index.ts
|
|
2
|
+
// Plugin entrypoint (design §8–§10, §15 item 5): everything the earlier tasks
|
|
3
|
+
// built becomes a working OpenCode v2 plugin. setup parses options (§3:
|
|
4
|
+
// warn-and-default, never throws), seeds state from storage (§8's startup
|
|
5
|
+
// chain: pin → fallback → disabled) without ever blocking on fresh data,
|
|
6
|
+
// registers the alias transform — after seeding, so the registration-time
|
|
7
|
+
// application sees the seeded picks — then the guard hook and the two
|
|
8
|
+
// commands, kicks off the first refresh in the background, schedules the
|
|
9
|
+
// 30-minute refresh tick, and returns a cleanup that clears the timer.
|
|
10
|
+
//
|
|
11
|
+
// `wire` is the testable core — the same setup with injected seams (acquire,
|
|
12
|
+
// timer, clock). The default export hands OpenCode the real context; it is
|
|
13
|
+
// cast once at this boundary because the plugin keeps a structural, mockable
|
|
14
|
+
// view of the context (plain strings instead of OpenCode's branded ids).
|
|
15
|
+
|
|
16
|
+
import { Plugin } from "@opencode/plugin"
|
|
17
|
+
import { parseConfig } from "./config"
|
|
18
|
+
import { pinKey, type Pin } from "./core/pins"
|
|
19
|
+
import { effectiveWeights, rankModels } from "./core/rank"
|
|
20
|
+
import type { Catalog, ModelEntry, Priority, Sourced } from "./core/types"
|
|
21
|
+
import { formatReport, humanizeAge, reportEntries } from "./opencode/commands"
|
|
22
|
+
import { ModelCompare } from "./rpc"
|
|
23
|
+
import { planAlias, type AliasTarget } from "./opencode/aliases"
|
|
24
|
+
import { guardDecision, type ModelRef } from "./opencode/guard"
|
|
25
|
+
import { readNewestCachedDoc, readRecord, type Storage } from "./opencode/storage"
|
|
26
|
+
import { acquire, cacheIdentities } from "./sources"
|
|
27
|
+
import {
|
|
28
|
+
REFRESH_MS,
|
|
29
|
+
adoptPlan,
|
|
30
|
+
createTick,
|
|
31
|
+
runnableSet,
|
|
32
|
+
warnDisabledAliases,
|
|
33
|
+
} from "./opencode/refresh"
|
|
34
|
+
import { PRIORITIES, planPicks } from "./opencode/resolution"
|
|
35
|
+
import {
|
|
36
|
+
ALIAS_PROVIDER,
|
|
37
|
+
aliasId,
|
|
38
|
+
useBestDecision,
|
|
39
|
+
useBestNotice,
|
|
40
|
+
} from "./opencode/usebest"
|
|
41
|
+
|
|
42
|
+
const PROVIDER_PREFIX = "openrouter/"
|
|
43
|
+
|
|
44
|
+
// Mirrors core/pins.ts' bareModelId (kept private there; the config value
|
|
45
|
+
// looks like "openrouter/<id>", catalog ids are bare). Strip exactly one
|
|
46
|
+
// leading prefix — a bare id may itself contain slashes, so nothing beyond
|
|
47
|
+
// the prefix is touched.
|
|
48
|
+
function bareModelId(configValue: string): string {
|
|
49
|
+
return configValue.startsWith(PROVIDER_PREFIX)
|
|
50
|
+
? configValue.slice(PROVIDER_PREFIX.length)
|
|
51
|
+
: configValue
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// ---- the slice of the OpenCode v2 plugin context the plugin touches --------
|
|
55
|
+
|
|
56
|
+
type Registration = { dispose: () => Promise<void> }
|
|
57
|
+
|
|
58
|
+
// The rpc.register return: disposal plus the event emitter the command's
|
|
59
|
+
// execute publishes the report through.
|
|
60
|
+
type RpcRegistration = Registration & {
|
|
61
|
+
events: { emit(name: string, data: unknown): Promise<void> }
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// The editor slice the alias applier touches (§8): `get` returns the concrete
|
|
65
|
+
// model's immutable definition; `update` mutates (or creates) one model.
|
|
66
|
+
export type AliasEditor = {
|
|
67
|
+
get(providerID: string, modelID: string): AliasConcrete | undefined
|
|
68
|
+
update(providerID: string, modelID: string, update: (model: AliasModel) => void): void
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
type AliasConcrete = {
|
|
72
|
+
capabilities: unknown
|
|
73
|
+
limit: unknown
|
|
74
|
+
cost: unknown
|
|
75
|
+
variants: unknown
|
|
76
|
+
time: unknown
|
|
77
|
+
status: unknown
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
type AliasModel = AliasConcrete & {
|
|
81
|
+
modelID?: string
|
|
82
|
+
name: string
|
|
83
|
+
enabled: boolean
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export type CommandDefinition = {
|
|
87
|
+
name: string
|
|
88
|
+
description?: string
|
|
89
|
+
execute: (input: { sessionID: string; prompt: { text: string } }) => Promise<void>
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
// The `http.request` event (§9): the guard keys on the requesting agent, not
|
|
93
|
+
// on event.model, which already shows the effective (fallback) reference.
|
|
94
|
+
export type HttpEvent = {
|
|
95
|
+
sessionID: string
|
|
96
|
+
agent: string
|
|
97
|
+
model: ModelRef
|
|
98
|
+
kind: "primary" | "compaction" | "title" | "generate"
|
|
99
|
+
request: unknown
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export type ModelListEntry = { id: string; providerID: string; enabled: boolean }
|
|
103
|
+
|
|
104
|
+
export type PluginCtx = {
|
|
105
|
+
options: unknown
|
|
106
|
+
app: { version: string }
|
|
107
|
+
storage: Storage
|
|
108
|
+
model: {
|
|
109
|
+
transform(callback: (editor: AliasEditor) => void): Promise<Registration>
|
|
110
|
+
// The real client returns a promise; a plain value is tolerated too.
|
|
111
|
+
list():
|
|
112
|
+
| Promise<{ location: unknown; data: ModelListEntry[] }>
|
|
113
|
+
| { location: unknown; data: ModelListEntry[] }
|
|
114
|
+
reload(): Promise<void>
|
|
115
|
+
}
|
|
116
|
+
provider: { reload(): Promise<void> }
|
|
117
|
+
agent: {
|
|
118
|
+
get(input: { agentID: string }): Promise<{ location: unknown; data: { model?: ModelRef } }>
|
|
119
|
+
reload(): Promise<void>
|
|
120
|
+
}
|
|
121
|
+
session: {
|
|
122
|
+
hook(
|
|
123
|
+
name: "http.request",
|
|
124
|
+
callback: (event: HttpEvent) => Promise<void> | void,
|
|
125
|
+
): Promise<Registration>
|
|
126
|
+
switchModel(input: { sessionID: string; model: ModelRef }): Promise<void>
|
|
127
|
+
// `resume: false` records the notice without scheduling a model turn.
|
|
128
|
+
synthetic(input: { sessionID: string; text: string; resume?: boolean | null }): Promise<unknown>
|
|
129
|
+
}
|
|
130
|
+
command: {
|
|
131
|
+
transform(
|
|
132
|
+
callback: (editor: { add(definition: CommandDefinition): void }) => void,
|
|
133
|
+
): Promise<Registration>
|
|
134
|
+
}
|
|
135
|
+
rpc: {
|
|
136
|
+
register(
|
|
137
|
+
definition: { id: string },
|
|
138
|
+
handlers: Record<string, (input: unknown) => Promise<unknown> | unknown>,
|
|
139
|
+
): Promise<RpcRegistration>
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
export type WireDeps = {
|
|
144
|
+
acquire?: typeof acquire
|
|
145
|
+
setInterval?: (fn: () => void, ms: number) => () => void
|
|
146
|
+
now?: () => number
|
|
147
|
+
// Per-tick hard deadline (F-01); tests shrink it.
|
|
148
|
+
deadlineMs?: number
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
// ---- captured state and small helpers --------------------------------------
|
|
152
|
+
|
|
153
|
+
// The picks and enabled flags a registry state is built from.
|
|
154
|
+
export type AliasPlan = {
|
|
155
|
+
picks: Partial<Record<Priority, AliasTarget>>
|
|
156
|
+
enabled: Partial<Record<Priority, boolean>>
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
// The mutable state the registrations capture (§8). Two views of the picks:
|
|
160
|
+
// - `plan` is the DESIRED registry state: the alias transform reads it at
|
|
161
|
+
// every replay, so a reload always serves the newest plan;
|
|
162
|
+
// - `picks`/`enabled` are the APPLIED view — what the registry is known to
|
|
163
|
+
// serve: the guard and the commands read it. The tick advances it only
|
|
164
|
+
// after the reload chain succeeded, so a failed reload is never reported
|
|
165
|
+
// to users as a change that happened (and is retried by the next tick).
|
|
166
|
+
// `registryDisabled[p]` records that the transform wanted alias p enabled but
|
|
167
|
+
// its concrete model was missing from the editor; the applied view then says
|
|
168
|
+
// disabled, whatever the plan wanted.
|
|
169
|
+
export type WireState = {
|
|
170
|
+
sourced: Sourced | null
|
|
171
|
+
pins: Partial<Record<Priority, Pin>>
|
|
172
|
+
picks: Partial<Record<Priority, AliasTarget>>
|
|
173
|
+
enabled: Partial<Record<Priority, boolean>>
|
|
174
|
+
plan: AliasPlan
|
|
175
|
+
registryDisabled: Partial<Record<Priority, boolean>>
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
// readRecord only guarantees a v=1 object; the pin fields are checked so a
|
|
179
|
+
// garbage record warns and is ignored instead of poisoning resolution (§7.2).
|
|
180
|
+
// The id must be non-empty and `since` a FINITE number — JSON.parse("1e999")
|
|
181
|
+
// and hand-edited records can carry Infinity, which would make every
|
|
182
|
+
// pin-expiry comparison (now - since) NaN.
|
|
183
|
+
function isValidPin(value: Pin): boolean {
|
|
184
|
+
return (
|
|
185
|
+
typeof value.id === "string" &&
|
|
186
|
+
value.id !== "" &&
|
|
187
|
+
typeof value.since === "number" &&
|
|
188
|
+
Number.isFinite(value.since)
|
|
189
|
+
)
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
// ---- the alias transform (design §8) ---------------------------------------
|
|
193
|
+
|
|
194
|
+
// The applier: computes each alias's plan from the captured state and copies
|
|
195
|
+
// the concrete model's identity fields onto the alias. A missing pick or a
|
|
196
|
+
// pick whose model is gone from the editor DISABLES the alias with a warning —
|
|
197
|
+
// never silently skipped (§8). Deep-copies via structuredClone to avoid
|
|
198
|
+
// aliasing the concrete model's objects.
|
|
199
|
+
function applyAliases(state: WireState, editor: AliasEditor): void {
|
|
200
|
+
for (const p of PRIORITIES) {
|
|
201
|
+
const target: AliasTarget = state.plan.picks[p] ?? { id: null, fromFallback: false }
|
|
202
|
+
// The editor's concrete definition doubles as the presence proof for the
|
|
203
|
+
// plan: planAlias only checks for undefined (it never reads model
|
|
204
|
+
// internals — the applier does, below).
|
|
205
|
+
const source = target.id === null ? undefined : editor.get(ALIAS_PROVIDER, target.id)
|
|
206
|
+
const plan = planAlias(p, target, source as unknown as ModelEntry | undefined)
|
|
207
|
+
|
|
208
|
+
if (plan.enabled && plan.modelID !== null && source !== undefined) {
|
|
209
|
+
editor.update(ALIAS_PROVIDER, aliasId(p), (model) => {
|
|
210
|
+
model.modelID = plan.modelID as string
|
|
211
|
+
model.name = plan.name
|
|
212
|
+
model.enabled = true
|
|
213
|
+
model.capabilities = structuredClone(source.capabilities)
|
|
214
|
+
model.limit = structuredClone(source.limit)
|
|
215
|
+
model.cost = structuredClone(source.cost)
|
|
216
|
+
model.variants = structuredClone(source.variants)
|
|
217
|
+
model.time = structuredClone(source.time)
|
|
218
|
+
model.status = source.status
|
|
219
|
+
})
|
|
220
|
+
state.registryDisabled[p] = false
|
|
221
|
+
} else {
|
|
222
|
+
// The plan wanted this alias enabled but the concrete model is gone:
|
|
223
|
+
// the registry now disables it, so the guard and /use-best must agree.
|
|
224
|
+
const wanted = target.id !== null
|
|
225
|
+
state.registryDisabled[p] = wanted
|
|
226
|
+
if (wanted) state.enabled[p] = false
|
|
227
|
+
editor.update(ALIAS_PROVIDER, aliasId(p), (model) => {
|
|
228
|
+
// modelID stays as-is; only the display name and the flag change.
|
|
229
|
+
model.enabled = false
|
|
230
|
+
model.name = plan.name
|
|
231
|
+
})
|
|
232
|
+
for (const warning of plan.warnings) console.warn("[model-compare]", warning)
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
// ---- setup -----------------------------------------------------------------
|
|
238
|
+
|
|
239
|
+
export async function wire(ctx: PluginCtx, deps: WireDeps = {}): Promise<() => void> {
|
|
240
|
+
const doAcquire = deps.acquire ?? acquire
|
|
241
|
+
const now = deps.now ?? Date.now
|
|
242
|
+
|
|
243
|
+
// §3: every option violation warns and defaults; parseConfig never throws.
|
|
244
|
+
const { config, warnings } = parseConfig(ctx.options)
|
|
245
|
+
for (const warning of warnings) console.warn("[model-compare]", warning)
|
|
246
|
+
|
|
247
|
+
const state: WireState = {
|
|
248
|
+
sourced: null,
|
|
249
|
+
pins: {},
|
|
250
|
+
picks: {},
|
|
251
|
+
enabled: {},
|
|
252
|
+
plan: { picks: {}, enabled: {} },
|
|
253
|
+
registryDisabled: {},
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
// Seed state from storage (§8 startup) BEFORE registering the transform:
|
|
257
|
+
// OpenCode applies transforms at registration time (design-spike-verified),
|
|
258
|
+
// so the first application must already see the seeded picks. Registering
|
|
259
|
+
// first would compute every alias from empty state, and a warm start whose
|
|
260
|
+
// first tick resolves the same picks then never reloads — the registry's
|
|
261
|
+
// aliases stay disabled while state.enabled says enabled. This needs only
|
|
262
|
+
// storage + ctx.model.list(); acquire is NOT awaited here.
|
|
263
|
+
//
|
|
264
|
+
// The seed is the §9 chain WITHOUT a fresh catalog — the same resolution
|
|
265
|
+
// the tick falls back to when a refresh fails (planPicks with doc: null):
|
|
266
|
+
// stored pin if runnable → configured fallback if runnable → disabled.
|
|
267
|
+
try {
|
|
268
|
+
const cached = await readNewestCachedDoc(ctx.storage, cacheIdentities(config))
|
|
269
|
+
if (cached !== null) {
|
|
270
|
+
// Warm start: read the pins — keyed by the effective weights of the
|
|
271
|
+
// cached document, since without fresh ranking the weights are
|
|
272
|
+
// unknowable.
|
|
273
|
+
state.sourced = cached
|
|
274
|
+
for (const p of PRIORITIES) {
|
|
275
|
+
const key = pinKey(p, effectiveWeights(cached.doc, p, config.weights[p]).weights)
|
|
276
|
+
const record = await readRecord<Pin>(ctx.storage, key)
|
|
277
|
+
if (record !== null && !isValidPin(record)) {
|
|
278
|
+
console.warn("[model-compare]", `corrupt pin record for ${p} (${key}); ignoring it`)
|
|
279
|
+
continue
|
|
280
|
+
}
|
|
281
|
+
if (record !== null) state.pins[p] = record
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
// The runnable set is only needed when something could use it.
|
|
285
|
+
const needsRunnable = Object.keys(state.pins).length > 0 || config.fallback !== null
|
|
286
|
+
const runnable = needsRunnable ? await runnableSet(ctx) : new Set<string>()
|
|
287
|
+
const plan = planPicks({ config, doc: null, runnable, pins: state.pins, now: now() })
|
|
288
|
+
adoptPlan(state, plan, true)
|
|
289
|
+
for (const notice of plan.notices) console.warn("[model-compare]", notice)
|
|
290
|
+
|
|
291
|
+
if (cached !== null) {
|
|
292
|
+
warnDisabledAliases(state.enabled, "startup")
|
|
293
|
+
} else {
|
|
294
|
+
// Cold start: the pin keys hash the effective weights of a catalog we
|
|
295
|
+
// don't have, so the chain starts at the configured fallback. It is
|
|
296
|
+
// never stored as a pin (§7.8).
|
|
297
|
+
const bare = config.fallback === null ? null : bareModelId(config.fallback)
|
|
298
|
+
if (bare !== null && state.enabled.balanced === true) {
|
|
299
|
+
console.warn(
|
|
300
|
+
"[model-compare]",
|
|
301
|
+
`no cached ranking; aliases start on the configured fallback ${bare}`,
|
|
302
|
+
)
|
|
303
|
+
} else if (bare !== null) {
|
|
304
|
+
console.warn(
|
|
305
|
+
"[model-compare]",
|
|
306
|
+
`no cached ranking; startup fallback ${bare} is not runnable; aliases disabled until first refresh`,
|
|
307
|
+
)
|
|
308
|
+
} else {
|
|
309
|
+
console.warn("[model-compare]", "no cached ranking; aliases disabled until first refresh")
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
} catch (error) {
|
|
313
|
+
// Storage problems degrade to the disabled state, never break setup (§9).
|
|
314
|
+
console.error("[model-compare]", error)
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
// Register the model transform (§8): aliases are computed inside the
|
|
318
|
+
// transform from the captured state, so any replay applies the current pick.
|
|
319
|
+
await ctx.model.transform((editor) => applyAliases(state, editor))
|
|
320
|
+
|
|
321
|
+
// The guard (§9): a primary request whose agent is configured with a
|
|
322
|
+
// DISABLED alias aborts with the exact error; everything else passes. The
|
|
323
|
+
// agent lookup is wrapped — a missing agent must not break the request path.
|
|
324
|
+
await ctx.session.hook("http.request", async (event) => {
|
|
325
|
+
let agentModel: ModelRef | undefined
|
|
326
|
+
try {
|
|
327
|
+
const agent = await ctx.agent.get({ agentID: event.agent })
|
|
328
|
+
agentModel = agent.data?.model
|
|
329
|
+
} catch {
|
|
330
|
+
agentModel = undefined
|
|
331
|
+
}
|
|
332
|
+
const decision = guardDecision(event.kind, agentModel, (p) => state.enabled[p] === true)
|
|
333
|
+
if (decision.error !== undefined) throw new Error(decision.error)
|
|
334
|
+
})
|
|
335
|
+
|
|
336
|
+
// The RPC (V2 plugins RPC): the report method serves the SAME text the
|
|
337
|
+
// /model-compare command synthesizes (one builder — the two can never
|
|
338
|
+
// drift), and the registration carries the report EVENT the command emits
|
|
339
|
+
// so the TUI half can open the report as an instant selectable dialog.
|
|
340
|
+
// Registered BEFORE the commands: the command's execute emits through it.
|
|
341
|
+
const rpcRegistration = await ctx.rpc.register(ModelCompare, {
|
|
342
|
+
report: async () => {
|
|
343
|
+
const ranks = reportRanks()
|
|
344
|
+
if (state.sourced === null || ranks === null) {
|
|
345
|
+
return { text: "model-compare: no ranking data available yet", age: null, entries: [] }
|
|
346
|
+
}
|
|
347
|
+
return {
|
|
348
|
+
text: formatReport(state.sourced, ranks, { picks: state.picks, overrides: config.weights }),
|
|
349
|
+
age: humanizeAge(Date.now() - state.sourced.acquiredAt),
|
|
350
|
+
entries: reportEntries(state.sourced, ranks, { picks: state.picks, overrides: config.weights }),
|
|
351
|
+
}
|
|
352
|
+
},
|
|
353
|
+
})
|
|
354
|
+
|
|
355
|
+
// The report behind /model-compare: the text (transcript + the RPC's
|
|
356
|
+
// report method) and the structured entries (the RPC's report method and
|
|
357
|
+
// event — the TUI renders them as a selectable dialog). One ranking, so
|
|
358
|
+
// the text, the entries and the picks can never disagree. Ranks under the
|
|
359
|
+
// same weights overrides the aliases resolve with, so the report agrees
|
|
360
|
+
// with the picks.
|
|
361
|
+
const reportRanks = () => {
|
|
362
|
+
if (state.sourced === null) return null
|
|
363
|
+
const doc = state.sourced.doc
|
|
364
|
+
return {
|
|
365
|
+
balanced: rankModels(doc, "balanced", config.weights.balanced).ids,
|
|
366
|
+
price: rankModels(doc, "price", config.weights.price).ids,
|
|
367
|
+
quality: rankModels(doc, "quality", config.weights.quality).ids,
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
// The commands (§10).
|
|
372
|
+
await ctx.command.transform((editor) => {
|
|
373
|
+
editor.add({
|
|
374
|
+
name: "use-best",
|
|
375
|
+
description: "Switch this session to a model-compare priority (balanced, price or quality)",
|
|
376
|
+
execute: async ({ sessionID, prompt }) => {
|
|
377
|
+
const decision = useBestDecision({
|
|
378
|
+
argument: prompt.text,
|
|
379
|
+
defaultPriority: config.defaultPriority,
|
|
380
|
+
enabled: (p) => state.enabled[p] === true,
|
|
381
|
+
})
|
|
382
|
+
if (!decision.ok) {
|
|
383
|
+
await ctx.session.synthetic({ sessionID, text: decision.message, resume: false })
|
|
384
|
+
return
|
|
385
|
+
}
|
|
386
|
+
const p = decision.priority
|
|
387
|
+
await ctx.session.switchModel({
|
|
388
|
+
sessionID,
|
|
389
|
+
model: { providerID: ALIAS_PROVIDER, id: aliasId(p) },
|
|
390
|
+
})
|
|
391
|
+
const pickId = state.picks[p]?.id ?? null
|
|
392
|
+
try {
|
|
393
|
+
await ctx.session.synthetic({
|
|
394
|
+
sessionID,
|
|
395
|
+
text: useBestNotice(p, pickId),
|
|
396
|
+
resume: false,
|
|
397
|
+
})
|
|
398
|
+
} catch {
|
|
399
|
+
// The switch already happened; the notice is best-effort.
|
|
400
|
+
}
|
|
401
|
+
},
|
|
402
|
+
})
|
|
403
|
+
editor.add({
|
|
404
|
+
name: "model-compare",
|
|
405
|
+
description: "Show the top 5 models per priority with scores and data age",
|
|
406
|
+
execute: async ({ sessionID }) => {
|
|
407
|
+
// Snapshot the catalog once, before anything awaits: the event
|
|
408
|
+
// emission and the synthetic's admission leave gaps in which a
|
|
409
|
+
// refresh can swap state.sourced, and the event's entries and the
|
|
410
|
+
// transcript text must come from the SAME document.
|
|
411
|
+
const sourced = state.sourced
|
|
412
|
+
const ranks = reportRanks()
|
|
413
|
+
// The EVENT goes first: the TUI's instant dialog must not wait behind
|
|
414
|
+
// the synthetic's admission, which measures ~8 s in this build (the
|
|
415
|
+
// transcript notice is the durable copy, not the fast path).
|
|
416
|
+
if (sourced === null || ranks === null) {
|
|
417
|
+
try {
|
|
418
|
+
await rpcRegistration.events.emit("report", { sessionID, age: null, entries: [] })
|
|
419
|
+
} catch {
|
|
420
|
+
// Emission failures must not fail the command.
|
|
421
|
+
}
|
|
422
|
+
await ctx.session.synthetic({
|
|
423
|
+
sessionID,
|
|
424
|
+
text: "model-compare: no ranking data available yet",
|
|
425
|
+
resume: false,
|
|
426
|
+
})
|
|
427
|
+
return
|
|
428
|
+
}
|
|
429
|
+
const context = { picks: state.picks, overrides: config.weights }
|
|
430
|
+
try {
|
|
431
|
+
await rpcRegistration.events.emit("report", {
|
|
432
|
+
sessionID,
|
|
433
|
+
age: humanizeAge(Date.now() - sourced.acquiredAt),
|
|
434
|
+
entries: reportEntries(sourced, ranks, context),
|
|
435
|
+
})
|
|
436
|
+
} catch {
|
|
437
|
+
// Emission failures must not fail the command.
|
|
438
|
+
}
|
|
439
|
+
await ctx.session.synthetic({
|
|
440
|
+
sessionID,
|
|
441
|
+
text: formatReport(sourced, ranks, context),
|
|
442
|
+
resume: false,
|
|
443
|
+
})
|
|
444
|
+
},
|
|
445
|
+
})
|
|
446
|
+
})
|
|
447
|
+
|
|
448
|
+
// The refresh tick (§8, §9), scheduled every 30 minutes — and, per §8, the
|
|
449
|
+
// FIRST refresh starts right here in the background: a cold start with
|
|
450
|
+
// working network must not wait 30 minutes for its aliases. Not awaited;
|
|
451
|
+
// the tick's non-overlap guard keeps the kick-off and the timer from
|
|
452
|
+
// colliding, and its catch-all keeps failures from escaping setup.
|
|
453
|
+
const tick = createTick(ctx, config, state, {
|
|
454
|
+
acquire: doAcquire,
|
|
455
|
+
now,
|
|
456
|
+
deadlineMs: deps.deadlineMs,
|
|
457
|
+
})
|
|
458
|
+
const cancelTimer =
|
|
459
|
+
deps.setInterval !== undefined
|
|
460
|
+
? deps.setInterval(tick, REFRESH_MS)
|
|
461
|
+
: (() => {
|
|
462
|
+
const handle = setInterval(tick, REFRESH_MS)
|
|
463
|
+
return () => clearInterval(handle)
|
|
464
|
+
})()
|
|
465
|
+
void tick()
|
|
466
|
+
|
|
467
|
+
// Registrations are cleaned up by OpenCode on unload; only the timer is ours.
|
|
468
|
+
return () => {
|
|
469
|
+
cancelTimer()
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
export default Plugin.define({
|
|
474
|
+
id: "model-compare",
|
|
475
|
+
async setup(ctx) {
|
|
476
|
+
return wire(ctx as unknown as PluginCtx, {})
|
|
477
|
+
},
|
|
478
|
+
})
|