model-compare-opencode-plugin 0.5.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -9
- package/dist/tui.js +147 -27
- package/package.json +2 -1
- package/src/core/catalog.ts +98 -16
- package/src/core/reasoning.ts +96 -0
- package/src/core/steering.ts +36 -0
- package/src/core/types.ts +34 -1
- package/src/index.ts +38 -13
- package/src/opencode/aliases.ts +11 -5
- package/src/opencode/commands.ts +41 -3
- package/src/opencode/guard.ts +4 -2
- package/src/opencode/usebest.ts +14 -3
- package/src/rpc.ts +30 -2
- package/src/tui.tsx +135 -23
- package/vendor/UPSTREAM.json +3 -3
- package/vendor/model_compare.py +2140 -430
package/README.md
CHANGED
|
@@ -10,7 +10,9 @@ Each priority is a live alias inside the existing OpenRouter provider — no new
|
|
|
10
10
|
| `openrouter/model-compare/price` | best price pick |
|
|
11
11
|
| `openrouter/model-compare/quality` | best quality pick |
|
|
12
12
|
|
|
13
|
-
The alias re-points at the current pick while OpenCode runs (refresh every 30 minutes, data cache 6 hours). OpenCode's model picker shows it as `Model Compare: <priority> → <pick>`, with `(fallback)`
|
|
13
|
+
The alias re-points at the current pick while OpenCode runs (refresh every 30 minutes, data cache 6 hours). It also pins the catalog's scored reasoning effort on each request, so the model runs at the depth its `quality` score was measured at. OpenCode's model picker shows it as `Model Compare: <priority> → <pick>`, with `(fallback)` while the fallback is serving and `(unavailable)` when the alias is disabled. The alias name deliberately carries no catalog effort or alternative: OpenCode shows the selected variant itself, and a baked-in depth would not follow your `#variant` choice.
|
|
14
|
+
|
|
15
|
+
**Effort vs variant.** The model-compare catalog and the request body call this reasoning level the *effort* (`quality_effort`, pinned as `reasoning.effort`); OpenCode calls the selectable model option a *variant* (`provider/model#variant`). The alias byline shows only the pick; the report dialog's status line uses “variant” and shows the highlighted model's variant and its best higher variant.
|
|
14
16
|
|
|
15
17
|
|
|
16
18
|
## Install
|
|
@@ -69,7 +71,7 @@ opencode run --model "openrouter/model-compare/price" "msg"
|
|
|
69
71
|
Commands:
|
|
70
72
|
|
|
71
73
|
- `/use-best [priority]` — switch the session to `openrouter/model-compare/<p>` (defaults to `defaultPriority`). Refuses a disabled alias.
|
|
72
|
-
- `/model-compare` — top 5 per priority with scores and data age;
|
|
74
|
+
- `/model-compare` — top 5 per priority with scores and data age; each line also shows the reasoning effort behind `quality` (`· high`) and the best selectable measured alternative (`· alt max 56.0`). Priorities whose `weights` override actually takes effect are labelled `(custom weights)` with recomputed scores, and each alias's current pick is marked `← current pick`. The output is a synthetic message (`resume: false`): it costs no model turn and, in a headless session, surfaces in the transcript at the next turn. In the TUI it opens the report dialog instead, with model, quality and price columns and a status line that follows the cursor and shows the highlighted model's variant and its best higher variant.
|
|
73
75
|
|
|
74
76
|
|
|
75
77
|
### Keyboard shortcuts
|
|
@@ -78,14 +80,14 @@ Commands:
|
|
|
78
80
|
|---|---|
|
|
79
81
|
| `alt+m` | picks dialog — all three priorities with their current picks |
|
|
80
82
|
| `alt+e` | toast the last response's served endpoint, alias responses only (`!` prefix flags a downgrade — not in the steering order) |
|
|
81
|
-
| `alt+shift+m` | report dialog — the top-5 table; `return` switches the session to the highlighted model |
|
|
83
|
+
| `alt+shift+m` | report dialog — the top-5 table with a cursor-following variant status line; `return` switches the session to the highlighted model |
|
|
82
84
|
| `up` / `down` | move the dialog cursor |
|
|
83
85
|
| `return` | select the highlighted row (picks dialog: switch to that priority) |
|
|
84
86
|
| `esc` | close the dialog |
|
|
85
87
|
|
|
86
88
|
Both dialogs open anywhere — including the opening screen, where no session exists yet. There, picking a row (a priority in the picks dialog, a model in the report) creates a new session already bound to that model and opens it; merely opening a dialog never creates a session. With a session open, a pick switches that session as before. The TUI has no `/use-best` keybinding: the slash command already covers switching.
|
|
87
89
|
|
|
88
|
-
Status: the footer shows `MC: balanced → <pick>` (balanced only — `disabled`/`unavailable`/`(fallback)` as applicable; all three priorities live in the picks dialog). Toasts announce `/use-best` outcomes and pick changes (never for the first observation after load), the `alt+e` served endpoint, and the cannot-steer notice — see [Steering and served endpoints](#steering-and-served-endpoints). Headless, command feedback is synthetic transcript messages at the next turn; the steering toasts have no headless surface.
|
|
90
|
+
Status: the footer shows `MC: balanced → <pick>` (balanced only — `disabled`/`unavailable`/`(fallback)` as applicable; all three priorities live in the picks dialog). The picker's alias name carries only the pick (plus the fallback marker); the report dialog's status line carries the variant and best higher variant. The footer stays a compact served-model line. Toasts announce `/use-best` outcomes and pick changes (never for the first observation after load), the `alt+e` served endpoint, and the cannot-steer notice — see [Steering and served endpoints](#steering-and-served-endpoints). Headless, command feedback is synthetic transcript messages at the next turn; the steering toasts have no headless surface.
|
|
89
91
|
|
|
90
92
|
|
|
91
93
|
## Configuration
|
|
@@ -155,18 +157,18 @@ The alt+e capture is best-effort by design, and alias-scoped: only a session who
|
|
|
155
157
|
|
|
156
158
|
The last validated catalog is persisted, so ranking and `/model-compare` work offline after a restart. The cache is keyed by source (URL or script path), so changing the source never serves the previous source's data. Script exit code 2 means "no candidates" *or* an argparse usage error — never read it as "no candidates" alone.
|
|
157
159
|
|
|
158
|
-
**Vendored pin:** `vendor/model_compare.py` is byte-identical to upstream `v0.
|
|
160
|
+
**Vendored pin:** `vendor/model_compare.py` is byte-identical to upstream `v0.6.0` (`vendor/UPSTREAM.json` records repo, tag, commit, sha256). The catalog is schema v5 (since upstream 0.6.0): `quality` is the AA intelligence index at the model's effective OpenRouter default reasoning effort, and every entry carries `quality_effort` plus a `reasoning` block (`default_effort`, the expanded `supported_efforts`, the raw flags, and the measured `depths` with AA `$/task`). The plugin accepts schema v5 ONLY — `validateCatalog` rejects anything else — and releases are gated on the live remote serving v5 (the release-gate check pipes `https://model-compare.dev/catalog.json` through `scripts/canary-check.mjs`, which fails closed when the served document is not the expected schema). Each alias request pins exactly the pick's `quality_effort` in its body (`reasoning: { effort }`), so the alias runs at the depth its score represents; an explicit `#variant` selection, or a body that already sets `reasoning`/`reasoning_effort`, wins and is never overwritten. The catalog's `reasoning` sentinel is display-only and never sent to the gateway. Effective-tier pricing semantics still apply: `pricing` scalars hold effective-tier values — the tier in effect at 1M-token prompts — so models with OpenRouter price overrides are ranked at what a long session actually pays.
|
|
159
161
|
|
|
160
|
-
**Vendored 0.
|
|
162
|
+
**Vendored 0.6.0 cold-cache network profile:** since 0.4.0 the vendored script performs per-model endpoint-pricing fetches when building a fresh catalog (76 candidates in the recorded run). Each fetch fails closed per model — after the script's internal retries, a failed endpoint fetch drops that model and the catalog shrinks silently (429-class responses raise without retry) — so a degraded network costs pool coverage, not pricing correctness. Unkeyed runs hydrate quality from the published catalog (one fetch of `https://model-compare.dev/catalog.json`, cached under `published-catalog`; keyed runs consult the AA API — the primary quality source — and never fetch the published catalog); every quality hydration fails closed to "no quality data" per model, and a missing record at the selected effort shows `-` rather than borrowing another effort's score. A cold cache under a slow network can exceed the plugin's 60-second local budget, in which case the plugin warns and falls back (`auto` → remote, `local` → its cached document).
|
|
161
163
|
|
|
162
164
|
**Upgrading the pin** — reviewed PRs only; nothing merges automatically:
|
|
163
165
|
|
|
164
166
|
1. `node scripts/sync-vendor.mjs <tag>` — fetches the script at that commit, reports the sha256 change, re-pins `UPSTREAM.json`, prints the matching CHANGELOG section (`--check` verifies offline).
|
|
165
|
-
2. Conformance tests check `--version`, schema
|
|
167
|
+
2. Conformance tests check `--version`, schema v5, required fields, recomputed `overall` scores, and the standard-library-only import allowlist.
|
|
166
168
|
3. The upgrade PR body lists script lines touching imports, URLs, environment variables, subprocess calls and file writes, for reviewer attention.
|
|
167
169
|
4. Release with a changelog line naming the upstream tag; users run `opencode plugin update` (OpenCode checks but doesn't apply).
|
|
168
170
|
|
|
169
|
-
A scheduled workflow opens a draft upgrade PR when upstream publishes a newer release (never a downgrade); a nightly canary exercises the vendored script against live upstream data and asserts the generated catalog is schema
|
|
171
|
+
A scheduled workflow opens a draft upgrade PR when upstream publishes a newer release (never a downgrade); a nightly canary exercises the vendored script against live upstream data and asserts the generated catalog is schema v5 (`scripts/canary-check.mjs` pins `EXPECTED_SCHEMA_VERSION = 5` and exits nonzero on any other version); releases additionally gate on the live remote itself (see the publishing gates below); a weekly drift check compares the vendored sha256 against upstream's pinned commit and the published rankings against the published scores.
|
|
170
172
|
|
|
171
173
|
|
|
172
174
|
## Troubleshooting
|
|
@@ -201,16 +203,18 @@ Run a checkout instead of the npm package:
|
|
|
201
203
|
OpenCode's local loader resolves a directory target strictly as `<dir>/index.*` — `exports` are ignored for local paths, and file targets are rejected. `src/` has the physical entry files (`src/index.ts`, `src/tui.tsx`); the plugin's imports resolve from the repo's `node_modules` (`npm ci` once at the repo root). The repo ships this setup as [`.test/`](.test/README.md). Two caveats: the watcher only watches package-style entries, so source edits need an OpenCode restart; and a bare package name is always npm-installed by OpenCode — it is not read from `node_modules` — so local testing must use the directory form.
|
|
202
204
|
|
|
203
205
|
```sh
|
|
206
|
+
npm run dev # launch the TUI against this checkout
|
|
204
207
|
npm test # unit + vendored-conformance suites (hermetic)
|
|
205
208
|
npm run build # regenerate dist/tui.js from src/tui.tsx
|
|
206
209
|
MC_INTEGRATION=1 npx vitest run tests/integration/ # live-server suite (opt-in)
|
|
207
210
|
```
|
|
208
211
|
|
|
212
|
+
- `npm run dev` writes a scratch project under `/tmp/opencode/model-compare-dev`, points its plugin entry at this checkout's `src/`, hides the global OpenCode config for the run (so an installed `model-compare-opencode-plugin` cannot shadow the local copy), and runs `opencode --standalone` there. The isolation is config-only: the scratch project shares your OpenCode data directory (auth, plugin state, sessions), so alias switches made in the dev session rewrite your live aliases. Pass extra flags after `--` (for example `npm run dev -- --auto`), or use `--print-config` to inspect the generated project config. `MC_DEV_DIR` overrides the scratch location. Source edits need an OpenCode restart.
|
|
209
213
|
- Local development loads `src/` directly and needs no build; the *published* package does — OpenCode's Solid JSX transform skips `node_modules` paths, so the npm-installed raw TSX cannot load (see [Install](#install) and the troubleshooting entry). `npm run build` precompiles `src/tui.tsx` through the same Babel pipeline as the host's loader (taken from the pinned `@opentui/solid`) into `dist/tui.js`, rebasing relative imports from `src/`-relative to `dist/`-relative; `prepublishOnly` runs it, and CI builds before the packaging check. `dist/` is regenerated, never committed.
|
|
210
214
|
- The integration files skip unless `MC_INTEGRATION=1`, so plain `npm test` stays hermetic. The suite drives a real `opencode serve` on 127.0.0.1:14721 and needs the `opencode` CLI and `python3` (skips cleanly when either is missing); one case needs network access and the `sqlite3` CLI.
|
|
211
215
|
- `node scripts/pack-smoke.mjs` (also in CI, after `node scripts/build-tui.mjs`) packs the plugin and checks the tarball against the packaging list (files, `exports`, relative imports, and that the TUI artifact is precompiled — no react import). It does **not** load the package in OpenCode — install the tarball into a scratch project and confirm OpenCode loads it before publishing.
|
|
212
216
|
- Publishing: cut a GitHub release tagged `vX.Y.Z` and `.github/workflows/release.yaml` stages the package on npm through trusted publishing (tokenless OIDC — no token secret; the npmjs.com trusted-publisher entry must name `release.yaml`, the `canonical/model-compare-opencode-plugin` repository and the `release` environment, with "Allow npm publish" and "Allow npm dist-tag" unchecked — stage-only, npm's recommended posture). Nothing goes live until a maintainer approves it with 2FA on npmjs.com (package page → Staged Packages) or via `npm stage approve`. The workflow fails closed when the tag does not match `package.json`'s version, and the gates (build, typecheck, tests, pack-smoke, live-remote canary) run before staging.
|
|
213
|
-
- `fixtures/catalog-v0.
|
|
217
|
+
- `fixtures/catalog-v0.6.0.json` is the sole contract fixture — a recorded schema-v5 catalog; `fixtures/catalog-no-quality.json` is derived from it by `scripts/make-no-quality-fixture.py` (the vendored script's own quality-blind weights, so its scores are internally consistent). No pre-v5 fixture exists; the v5-only gate rejects one. `node scripts/sync-vendor.mjs --check` verifies the pin's sha256 offline.
|
|
214
218
|
- Repo layout: `src/core/` (pure ranking/pinning), `src/sources/` (local + remote acquisition), `src/opencode/` (aliases, guard, commands, storage, refresh), `dist/` (generated `tui.js` artifact, gitignored), `vendor/` (pinned upstream script), `tests/{unit,conformance,integration}`.
|
|
215
219
|
|
|
216
220
|
|
package/dist/tui.js
CHANGED
|
@@ -139,6 +139,68 @@ export function cappedRowsHeight(contentLines, terminalHeight) {
|
|
|
139
139
|
return Math.min(contentLines, Math.max(1, Math.floor(terminalHeight / 2) - 6));
|
|
140
140
|
}
|
|
141
141
|
|
|
142
|
+
// The terminal display width of one code point, in columns: 2 for the ranges
|
|
143
|
+
// terminals render double-width — East Asian Wide/Fullwidth (CJK, Hangul) and
|
|
144
|
+
// the emoji/symbol planes — and 1 for everything else. Deliberately
|
|
145
|
+
// CONSERVATIVE: it never undercounts the wide content the catalog can carry.
|
|
146
|
+
// Blocks that are only partially wide are covered at BLOCK width — the
|
|
147
|
+
// narrow tails inside the SMP symbol planes (alchemical symbols, chess)
|
|
148
|
+
// count 2 anyway — and variation selector 16 (emoji presentation) counts 1
|
|
149
|
+
// itself, so a base+VS16 sequence such as ☺️ always totals ≥ 2 even when the
|
|
150
|
+
// base alone would not. Overcounting only makes a clamp cut
|
|
151
|
+
// earlier; it can never let a line render wider than its budget. (Genuinely
|
|
152
|
+
// East-Asian-Ambiguous characters OUTSIDE these ranges — ±, an en dash,
|
|
153
|
+
// Greek — count 1; a terminal font that renders those wide is beyond the
|
|
154
|
+
// clamp's scope. Catalog text is sanitized to a single line by
|
|
155
|
+
// sanitizeCatalogText, so width is the only wrap risk left.)
|
|
156
|
+
function charWidth(codePoint) {
|
|
157
|
+
// Variation selectors (VS1–VS16) are zero-width in reality. VS16 forces
|
|
158
|
+
// emoji presentation on the preceding base, so the pair renders 2 columns:
|
|
159
|
+
// counting the selector itself as 1 keeps any base+VS16 sequence at ≥ 2
|
|
160
|
+
// even when the base is NOT in a wide block (⌨️ = U+2328+VS16), while a
|
|
161
|
+
// wide base + VS16 overcounts by one — the safe direction.
|
|
162
|
+
if (codePoint >= 0xfe00 && codePoint <= 0xfe0f) return 1;
|
|
163
|
+
if (codePoint >= 0x1100 && codePoint <= 0x115f ||
|
|
164
|
+
// Hangul Jamo
|
|
165
|
+
codePoint >= 0x2e80 && codePoint <= 0xa4cf ||
|
|
166
|
+
// CJK radicals … Yi
|
|
167
|
+
codePoint >= 0xa960 && codePoint <= 0xa97f ||
|
|
168
|
+
// Hangul Jamo Ext. A
|
|
169
|
+
codePoint >= 0xac00 && codePoint <= 0xd7a3 ||
|
|
170
|
+
// Hangul syllables
|
|
171
|
+
codePoint >= 0xf900 && codePoint <= 0xfaff ||
|
|
172
|
+
// CJK compat ideographs
|
|
173
|
+
codePoint >= 0xfe10 && codePoint <= 0xfe19 ||
|
|
174
|
+
// vertical forms
|
|
175
|
+
codePoint >= 0xfe30 && codePoint <= 0xfe6f ||
|
|
176
|
+
// CJK compat forms
|
|
177
|
+
codePoint >= 0xff00 && codePoint <= 0xff60 ||
|
|
178
|
+
// fullwidth forms
|
|
179
|
+
codePoint >= 0xffe0 && codePoint <= 0xffe6 ||
|
|
180
|
+
// fullwidth signs
|
|
181
|
+
codePoint >= 0x2600 && codePoint <= 0x27bf ||
|
|
182
|
+
// misc symbols + dingbats (☺ ✨ ⚠ — ambiguous, commonly 2 under emoji presentation)
|
|
183
|
+
codePoint >= 0x2b00 && codePoint <= 0x2bff ||
|
|
184
|
+
// misc symbols and arrows (⭐)
|
|
185
|
+
codePoint >= 0x1f000 && codePoint <= 0x1faff ||
|
|
186
|
+
// the SMP symbol planes: mahjong/cards, enclosed ideographs, pictographs, transport (🚀), supplemental, symbols ext-A
|
|
187
|
+
codePoint >= 0x20000 && codePoint <= 0x2fffd ||
|
|
188
|
+
// CJK ext B+
|
|
189
|
+
codePoint >= 0x30000 && codePoint <= 0x3fffd // CJK ext G+
|
|
190
|
+
) {
|
|
191
|
+
return 2;
|
|
192
|
+
}
|
|
193
|
+
return 1;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
// A string's terminal display width (the sum of its code points' widths).
|
|
197
|
+
// Exported for the unit rig, which asserts rendered lines against it.
|
|
198
|
+
export function strWidth(value) {
|
|
199
|
+
let width = 0;
|
|
200
|
+
for (const char of value) width += charWidth(char.codePointAt(0));
|
|
201
|
+
return width;
|
|
202
|
+
}
|
|
203
|
+
|
|
142
204
|
// One report entry's defensive normalization, shared by the two paths that
|
|
143
205
|
// carry report data across a trust boundary (the /model-compare command's
|
|
144
206
|
// event and the TUI's direct RPC call): an entry missing any identity field
|
|
@@ -157,6 +219,8 @@ function toReportEntries(raw) {
|
|
|
157
219
|
value: entry.value,
|
|
158
220
|
score: typeof entry.score === "string" ? entry.score : "-",
|
|
159
221
|
quality: typeof entry.quality === "string" ? entry.quality : "-",
|
|
222
|
+
effort: typeof entry.effort === "string" ? entry.effort : "",
|
|
223
|
+
alternative: typeof entry.alternative === "string" ? entry.alternative : "",
|
|
160
224
|
price: typeof entry.price === "string" ? entry.price : "-",
|
|
161
225
|
current: entry.current === true,
|
|
162
226
|
fromFallback: entry.fromFallback === true
|
|
@@ -627,14 +691,26 @@ export default Plugin.define({
|
|
|
627
691
|
})();
|
|
628
692
|
});
|
|
629
693
|
});
|
|
694
|
+
_$insert(_el$2, () => {
|
|
695
|
+
const row = selectable[cursor()];
|
|
696
|
+
return row.status === undefined ? null : (() => {
|
|
697
|
+
var _el$18 = _$createElement("box"),
|
|
698
|
+
_el$19 = _$createElement("text");
|
|
699
|
+
_$insertNode(_el$18, _el$19);
|
|
700
|
+
_$setProp(_el$18, "paddingLeft", 4);
|
|
701
|
+
_$setProp(_el$19, "fg", muted);
|
|
702
|
+
_$insert(_el$19, () => row.status);
|
|
703
|
+
return _el$18;
|
|
704
|
+
})();
|
|
705
|
+
}, _el$8);
|
|
630
706
|
_$insert(_el$2, () => footerLines.map(line => (() => {
|
|
631
|
-
var _el$
|
|
632
|
-
_el$
|
|
633
|
-
_$insertNode(_el$
|
|
634
|
-
_$setProp(_el$
|
|
635
|
-
_$setProp(_el$
|
|
636
|
-
_$insert(_el$
|
|
637
|
-
return _el$
|
|
707
|
+
var _el$20 = _$createElement("box"),
|
|
708
|
+
_el$21 = _$createElement("text");
|
|
709
|
+
_$insertNode(_el$20, _el$21);
|
|
710
|
+
_$setProp(_el$20, "paddingLeft", 4);
|
|
711
|
+
_$setProp(_el$21, "fg", muted);
|
|
712
|
+
_$insert(_el$21, line);
|
|
713
|
+
return _el$20;
|
|
638
714
|
})()), _el$8);
|
|
639
715
|
_$insertNode(_el$8, _el$9);
|
|
640
716
|
_$setProp(_el$8, "paddingLeft", 4);
|
|
@@ -694,33 +770,76 @@ export default Plugin.define({
|
|
|
694
770
|
notify("no ranking data available yet", "error");
|
|
695
771
|
return;
|
|
696
772
|
}
|
|
697
|
-
// The approved table
|
|
698
|
-
//
|
|
699
|
-
//
|
|
700
|
-
//
|
|
701
|
-
//
|
|
702
|
-
//
|
|
703
|
-
//
|
|
704
|
-
//
|
|
705
|
-
//
|
|
706
|
-
//
|
|
707
|
-
//
|
|
773
|
+
// The approved table now carries three columns: model name, quality
|
|
774
|
+
// and price. The scored variant (the catalog's effort) and the best
|
|
775
|
+
// higher variant moved into a cursor-following status line below the
|
|
776
|
+
// list (see ListRow.status), so the table stays readable. No score
|
|
777
|
+
// column — the transcript's text report keeps it. Column widths are clamped
|
|
778
|
+
// (qual ≤ 5, $ in/out ≤ 13) and oversized cells are truncated to their
|
|
779
|
+
// clamped width, so with the name clamp below the row budget holds
|
|
780
|
+
// even for hostile/absurd column strings: what the clamp guarantees is
|
|
781
|
+
// that a row and the header stay within the dialog's fixed 52-char
|
|
782
|
+
// content width (hostile values get truncated rather than pushed into
|
|
783
|
+
// wrapping — the 40-char rankless fallback label did exactly that
|
|
784
|
+
// before the name clamp).
|
|
708
785
|
const labels = data.entries.map(entry => `${entry.rank === null ? "·" : entry.rank}. ${entry.id}${entry.fromFallback ? " (fallback)" : ""}`);
|
|
709
786
|
const labelOf = new Map(data.entries.map((entry, index) => [entry, labels[index]]));
|
|
710
|
-
const colW = (values, floor, cap) => Math.min(cap, Math.max(floor, ...values.map(
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
//
|
|
714
|
-
const
|
|
787
|
+
const colW = (values, floor, cap) => Math.min(cap, Math.max(floor, ...values.map(strWidth)));
|
|
788
|
+
// Quality is a one-decimal AA index (max "100.0"); the caps leave the
|
|
789
|
+
// 52-char row budget intact (see `fixed` below) while fitting real
|
|
790
|
+
// values: "10.00/100.00" (price).
|
|
791
|
+
const qualW = colW(data.entries.map(entry => entry.quality), "qual".length, 5);
|
|
792
|
+
// The status line's variant and best-higher-variant cells keep their
|
|
793
|
+
// old column caps: the status box is a direct child of the 60-wide
|
|
794
|
+
// dialog with only its own 4-char left inset, so one text line has 56
|
|
795
|
+
// columns — the worst case, 9 ("variant: ") + 7 + 24 (" · best higher
|
|
796
|
+
// variant: ") + 13 = 53, leaves 3 columns of slack. Clamping each cell
|
|
797
|
+
// (in display columns, see cell) keeps hostile catalog strings from
|
|
798
|
+
// wrapping.
|
|
799
|
+
const effortW = 7;
|
|
800
|
+
const altW = 13;
|
|
801
|
+
const priceW = colW(data.entries.map(entry => entry.price), "$ in/out".length, 13);
|
|
802
|
+
// A cell wider than its clamped column is truncated to fit — measured
|
|
803
|
+
// in terminal COLUMNS (strWidth), not string length: a 7-code-point
|
|
804
|
+
// CJK effort label renders 14 columns wide and would push the row
|
|
805
|
+
// past its budget if length were the measure.
|
|
806
|
+
const cell = (value, width) => {
|
|
807
|
+
if (strWidth(value) <= width) return value;
|
|
808
|
+
// Reserve one column for the ellipsis; walk whole code points so a
|
|
809
|
+
// wide char is never cut in half.
|
|
810
|
+
let used = 0;
|
|
811
|
+
let out = "";
|
|
812
|
+
for (const char of value) {
|
|
813
|
+
const w = charWidth(char.codePointAt(0));
|
|
814
|
+
if (used + w > width - 1) break;
|
|
815
|
+
out += char;
|
|
816
|
+
used += w;
|
|
817
|
+
}
|
|
818
|
+
return `${out}…`;
|
|
819
|
+
};
|
|
820
|
+
// The pads match cell's measure, so a truncated wide cell and its
|
|
821
|
+
// padding land on the same column arithmetic (padEnd keeps the
|
|
822
|
+
// prices' flush right edges — see below).
|
|
823
|
+
const padEndW = (value, width) => value + " ".repeat(Math.max(0, width - strWidth(value)));
|
|
824
|
+
const padStartW = (value, width) => " ".repeat(Math.max(0, width - strWidth(value))) + value;
|
|
715
825
|
// Prices use padEnd (not padStart): padded to a common width the
|
|
716
826
|
// right edges stay flush while the fixed 2-space gap between columns
|
|
717
827
|
// stays exactly 2 (padStart pushed each value right by its slack).
|
|
718
828
|
// The default dialog is 60 wide: 2 list insets + 6 row insets leave 52
|
|
719
829
|
// for row content; the header shares that budget (plus its 3-indent).
|
|
830
|
+
// Column caps keep the budget even for hostile strings: 5+13 plus the
|
|
831
|
+
// two 2-char paddings and the 2-char row indent = 24, leaving plenty
|
|
832
|
+
// for the name (the name column shrinks first for real content).
|
|
720
833
|
const fixed = 2 + qualW + 2 + priceW;
|
|
721
|
-
const nameW = Math.max(
|
|
722
|
-
const trim = label => label
|
|
834
|
+
const nameW = Math.max(6, Math.min(52 - fixed, Math.max(...labels.map(strWidth))));
|
|
835
|
+
const trim = label => cell(label, nameW);
|
|
723
836
|
const headerLine = " " + " ".repeat(nameW) + " " + "qual".padStart(qualW) + " " + "$ in/out".padEnd(priceW);
|
|
837
|
+
// Catalog v5 calls this the effort the quality was measured at, and the
|
|
838
|
+
// best selectable measured alternative ("max 56.0"). OpenCode calls the
|
|
839
|
+
// selectable option a VARIANT, so the TUI uses "variant" throughout to
|
|
840
|
+
// match what the user sees in OpenCode's model picker. A missing value
|
|
841
|
+
// is "-", and "best higher variant" spells out what "alt" meant.
|
|
842
|
+
const statusOf = entry => `variant: ${cell(entry.effort || "-", effortW)} · best higher variant: ${cell(entry.alternative || "-", altW)}`;
|
|
724
843
|
const rows = [{
|
|
725
844
|
label: "",
|
|
726
845
|
headerLine
|
|
@@ -734,8 +853,9 @@ export default Plugin.define({
|
|
|
734
853
|
});
|
|
735
854
|
for (const entry of section) {
|
|
736
855
|
rows.push({
|
|
737
|
-
label: trim(labelOf.get(entry) ?? "")
|
|
738
|
-
columns: " " + cell(entry.quality, qualW)
|
|
856
|
+
label: padEndW(trim(labelOf.get(entry) ?? ""), nameW),
|
|
857
|
+
columns: " " + padStartW(cell(entry.quality, qualW), qualW) + " " + padEndW(cell(entry.price, priceW), priceW),
|
|
858
|
+
status: statusOf(entry),
|
|
739
859
|
// F-20 id split: the row DISPLAYS the sanitized id (in `label`)
|
|
740
860
|
// but picks `value`, the raw catalog id switchToModel needs.
|
|
741
861
|
value: entry.value,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "model-compare-opencode-plugin",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.1",
|
|
4
4
|
"description": "OpenCode plugin that switches OpenRouter model aliases to the best value-for-money pick from canonical/model-compare's ranking (balanced, price, quality).",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"opencode",
|
|
@@ -39,6 +39,7 @@
|
|
|
39
39
|
},
|
|
40
40
|
"scripts": {
|
|
41
41
|
"build": "node scripts/build-tui.mjs",
|
|
42
|
+
"dev": "node scripts/dev-local.mjs",
|
|
42
43
|
"test": "vitest run",
|
|
43
44
|
"typecheck": "tsc --noEmit",
|
|
44
45
|
"prepublishOnly": "npm run build && npm run typecheck && npm test && node scripts/pack-smoke.mjs"
|
package/src/core/catalog.ts
CHANGED
|
@@ -1,9 +1,16 @@
|
|
|
1
1
|
// src/core/catalog.ts
|
|
2
|
-
// Catalog document validation (design §5). Accepts schema
|
|
2
|
+
// Catalog document validation (design §5). Accepts schema v5 documents (only) from
|
|
3
3
|
// either source (vendored script or remote catalog) and projects them onto
|
|
4
4
|
// the plugin's Catalog type. Validation only — ranking lives elsewhere.
|
|
5
5
|
|
|
6
|
-
import type {
|
|
6
|
+
import type {
|
|
7
|
+
Catalog,
|
|
8
|
+
ModelEntry,
|
|
9
|
+
Priority,
|
|
10
|
+
Reasoning,
|
|
11
|
+
ReasoningDepth,
|
|
12
|
+
Weights,
|
|
13
|
+
} from "./types"
|
|
7
14
|
|
|
8
15
|
const PRIORITIES: readonly Priority[] = ["balanced", "price", "quality"]
|
|
9
16
|
const SCORE_KEYS = ["quality", "price", "context", "age"] as const
|
|
@@ -44,18 +51,67 @@ function steer(raw: unknown): ModelEntry["steering"] {
|
|
|
44
51
|
return steering
|
|
45
52
|
}
|
|
46
53
|
|
|
54
|
+
function finiteOrNull(value: unknown): number | null {
|
|
55
|
+
return typeof value === "number" && Number.isFinite(value) ? value : null
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function stringOrNull(value: unknown): string | null {
|
|
59
|
+
return typeof value === "string" ? value : null
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function boolOrNull(value: unknown): boolean | null {
|
|
63
|
+
return typeof value === "boolean" ? value : null
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// The v5 reasoning block (display + request injection). Field-by-field
|
|
67
|
+
// projection, never a spread of the raw object: a hostile catalogURL serves
|
|
68
|
+
// arbitrary JSON here, and only the fields the plugin reads survive. A
|
|
69
|
+
// non-object block reads as absent; a malformed field reads as its neutral
|
|
70
|
+
// value (empty list / null) rather than rejecting the whole catalog — the
|
|
71
|
+
// block is advisory metadata, not ranking input. Depth records without a
|
|
72
|
+
// non-empty string effort are dropped; every other depth field is nullable.
|
|
73
|
+
function projectReasoning(raw: unknown): Reasoning | undefined {
|
|
74
|
+
if (!isObject(raw)) return undefined
|
|
75
|
+
const supported =
|
|
76
|
+
Array.isArray(raw.supported_efforts) && raw.supported_efforts.every((effort) => typeof effort === "string")
|
|
77
|
+
? [...(raw.supported_efforts as string[])]
|
|
78
|
+
: []
|
|
79
|
+
const depths: ReasoningDepth[] = []
|
|
80
|
+
if (Array.isArray(raw.depths)) {
|
|
81
|
+
for (const record of raw.depths) {
|
|
82
|
+
if (!isObject(record)) continue
|
|
83
|
+
if (typeof record.effort !== "string" || record.effort === "") continue
|
|
84
|
+
depths.push({
|
|
85
|
+
effort: record.effort,
|
|
86
|
+
index: finiteOrNull(record.index),
|
|
87
|
+
cost_per_task_usd: finiteOrNull(record.cost_per_task_usd),
|
|
88
|
+
source: stringOrNull(record.source),
|
|
89
|
+
as_of: stringOrNull(record.as_of),
|
|
90
|
+
})
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
return {
|
|
94
|
+
default_effort: stringOrNull(raw.default_effort),
|
|
95
|
+
supported_efforts: supported,
|
|
96
|
+
default_enabled: boolOrNull(raw.default_enabled),
|
|
97
|
+
mandatory: boolOrNull(raw.mandatory),
|
|
98
|
+
supports_max_tokens: boolOrNull(raw.supports_max_tokens),
|
|
99
|
+
depths,
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
47
103
|
export function validateCatalog(
|
|
48
104
|
raw: unknown,
|
|
49
105
|
): { ok: true; catalog: Catalog } | { ok: false; reason: string } {
|
|
50
106
|
if (!isObject(raw)) return { ok: false, reason: "catalog must be a JSON object" }
|
|
51
107
|
|
|
52
|
-
// Supported schema:
|
|
53
|
-
// schema_version
|
|
54
|
-
//
|
|
55
|
-
//
|
|
56
|
-
// reason string the canary and the stored-document path rely on.
|
|
57
|
-
if (raw.schema_version !==
|
|
58
|
-
return { ok: false, reason: `schema_version must be
|
|
108
|
+
// Supported schema: v5 only — upstream's current contract (0.6.0's
|
|
109
|
+
// schema_version 5). v5 changes the meaning of `quality` (the AA score at
|
|
110
|
+
// the selected/default reasoning effort) and adds per-model
|
|
111
|
+
// `quality_effort` and `reasoning`; every other version is rejected with
|
|
112
|
+
// the reason string the canary and the stored-document path rely on.
|
|
113
|
+
if (raw.schema_version !== 5) {
|
|
114
|
+
return { ok: false, reason: `schema_version must be 5, got ${JSON.stringify(raw.schema_version)}` }
|
|
59
115
|
}
|
|
60
116
|
if (typeof raw.generated_at !== "string") {
|
|
61
117
|
return { ok: false, reason: "generated_at must be a string" }
|
|
@@ -160,12 +216,17 @@ export function validateCatalog(
|
|
|
160
216
|
const input = pricing.input_per_1m
|
|
161
217
|
const output = pricing.output_per_1m
|
|
162
218
|
// The steering block projects (or reads as absent) per the steer() rules.
|
|
163
|
-
// The
|
|
164
|
-
//
|
|
165
|
-
//
|
|
219
|
+
// The v5 quality_effort/reasoning fields project the same way: strip the
|
|
220
|
+
// raw keys before the spread so the camelCase projection is the only
|
|
221
|
+
// representation on the entry (a hostile block never survives verbatim).
|
|
222
|
+
const { steering: _rawSteering, quality_effort: _rawQualityEffort, reasoning: _rawReasoning, ...passthrough } =
|
|
223
|
+
entry
|
|
166
224
|
const steering = steer(entry.steering)
|
|
225
|
+
const qualityEffort =
|
|
226
|
+
typeof entry.quality_effort === "string" && entry.quality_effort !== "" ? entry.quality_effort : null
|
|
227
|
+
const reasoning = projectReasoning(entry.reasoning)
|
|
167
228
|
models.push({
|
|
168
|
-
...
|
|
229
|
+
...passthrough,
|
|
169
230
|
id: entry.id,
|
|
170
231
|
name: entry.name,
|
|
171
232
|
pricing: {
|
|
@@ -177,6 +238,8 @@ export function validateCatalog(
|
|
|
177
238
|
context: entry.context,
|
|
178
239
|
scores: { ...components, overall },
|
|
179
240
|
quality,
|
|
241
|
+
qualityEffort,
|
|
242
|
+
reasoning,
|
|
180
243
|
steering,
|
|
181
244
|
})
|
|
182
245
|
}
|
|
@@ -220,14 +283,33 @@ export function validateStoredCatalog(
|
|
|
220
283
|
stored: unknown,
|
|
221
284
|
): { ok: true; catalog: Catalog } | { ok: false; reason: string } {
|
|
222
285
|
if (!isObject(stored)) return { ok: false, reason: "stored catalog must be an object" }
|
|
286
|
+
// v5 projections always carry qualityEffort (null or a string). A record
|
|
287
|
+
// written by a pre-v5 plugin lacks it: accepting it would mix v4's max-row
|
|
288
|
+
// quality with v5's default-effort semantics, so it is a cache miss.
|
|
289
|
+
if (Array.isArray(stored.models)) {
|
|
290
|
+
for (const [i, entry] of stored.models.entries()) {
|
|
291
|
+
if (isObject(entry) && !Object.hasOwn(entry, "qualityEffort")) {
|
|
292
|
+
return { ok: false, reason: `models[${i}].qualityEffort is missing (not a v5 catalog)` }
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
}
|
|
223
296
|
const models = Array.isArray(stored.models)
|
|
224
297
|
? stored.models.map((entry) => {
|
|
225
|
-
if (!isObject(entry)
|
|
226
|
-
|
|
298
|
+
if (!isObject(entry)) return entry
|
|
299
|
+
// The projection renamed quality_effort to qualityEffort and unwrapped
|
|
300
|
+
// steering; rebuild the raw v5 shape. A hand-edited record re-projects
|
|
301
|
+
// (or reads as absent) exactly like a raw one would.
|
|
302
|
+
const raw: Record<string, unknown> = {
|
|
303
|
+
...entry,
|
|
304
|
+
quality_effort: entry.qualityEffort ?? null,
|
|
305
|
+
}
|
|
306
|
+
if (entry.reasoning !== undefined) raw.reasoning = entry.reasoning
|
|
307
|
+
if (entry.steering !== undefined) raw.steering = { model: entry.id, provider: entry.steering }
|
|
308
|
+
return raw
|
|
227
309
|
})
|
|
228
310
|
: stored.models
|
|
229
311
|
return validateCatalog({
|
|
230
|
-
schema_version:
|
|
312
|
+
schema_version: 5,
|
|
231
313
|
generated_at: stored.generatedAt,
|
|
232
314
|
parameters: { weights: stored.weights },
|
|
233
315
|
models,
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
// src/core/reasoning.ts
|
|
2
|
+
// Reasoning-depth helpers (catalog v5): gateway-effort validation, the
|
|
3
|
+
// selectable-depth rule, and the best measured alternative to the effort a
|
|
4
|
+
// model was scored at. Pure functions over a projected ModelEntry — no I/O,
|
|
5
|
+
// no OpenCode imports.
|
|
6
|
+
//
|
|
7
|
+
// The selectable rule is upstream's site rule (web/site/app.js depthRows):
|
|
8
|
+
// OpenRouter can select an effort only when the model's expanded
|
|
9
|
+
// `supported_efforts` lists it (or the model is a single-state toggle that
|
|
10
|
+
// exposes `#none`). An AA row for an unlisted effort is display-only and is
|
|
11
|
+
// never advertised as an alternative. The catalog's `reasoning` sentinel is a
|
|
12
|
+
// producer label for a single-state reasoning model, not a gateway value.
|
|
13
|
+
|
|
14
|
+
import type { ModelEntry, Reasoning } from "./types"
|
|
15
|
+
|
|
16
|
+
// What OpenRouter accepts as a discrete effort. Mirrors upstream's
|
|
17
|
+
// REASONING_GATEWAY_EFFORTS. Deliberately excludes the catalog sentinels
|
|
18
|
+
// "reasoning" and "default".
|
|
19
|
+
export const GATEWAY_EFFORTS = ["max", "xhigh", "high", "medium", "low", "minimal", "none"] as const
|
|
20
|
+
|
|
21
|
+
// Deepest-first display order, mirroring upstream's AA_EFFORT_ORDER.
|
|
22
|
+
const EFFORT_ORDER: readonly string[] = [...GATEWAY_EFFORTS, "reasoning"]
|
|
23
|
+
|
|
24
|
+
export function gatewayEffort(value: string | null | undefined): string | null {
|
|
25
|
+
return value !== null && value !== undefined && (GATEWAY_EFFORTS as readonly string[]).includes(value)
|
|
26
|
+
? value
|
|
27
|
+
: null
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function effortOrder(effort: string): number {
|
|
31
|
+
const index = EFFORT_ORDER.indexOf(effort)
|
|
32
|
+
return index === -1 ? EFFORT_ORDER.length : index
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
// Upstream's v5 non-reasoning rule (the producer's is_reasoning flag is not
|
|
36
|
+
// published): no reasoning metadata at all, so the bare id is the only form.
|
|
37
|
+
function isNonReasoningBlock(reasoning: Reasoning): boolean {
|
|
38
|
+
return (
|
|
39
|
+
reasoning.default_effort === "none" &&
|
|
40
|
+
reasoning.default_enabled !== false &&
|
|
41
|
+
reasoning.mandatory !== true &&
|
|
42
|
+
reasoning.supports_max_tokens !== true
|
|
43
|
+
)
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// A single-state reasoning model with reasoning off by default exposes only
|
|
47
|
+
// the "#none" toggle.
|
|
48
|
+
function isSingleStateToggle(reasoning: Reasoning): boolean {
|
|
49
|
+
return reasoning.mandatory !== true && reasoning.default_enabled === false
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export function isSelectableDepth(model: ModelEntry, effort: string): boolean {
|
|
53
|
+
if (effort === "reasoning") return false
|
|
54
|
+
const reasoning = model.reasoning
|
|
55
|
+
if (reasoning === undefined) return false
|
|
56
|
+
if (reasoning.supported_efforts.includes(effort)) return true
|
|
57
|
+
if (effort !== "none" || reasoning.supported_efforts.length > 0) return false
|
|
58
|
+
if (isNonReasoningBlock(reasoning)) return false
|
|
59
|
+
return isSingleStateToggle(reasoning)
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export type DepthAlternative = {
|
|
63
|
+
effort: string
|
|
64
|
+
index: number
|
|
65
|
+
cost: number | null
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// The best measured, selectable effort that scores strictly higher than the
|
|
69
|
+
// effort `quality` already represents. Unselectable rows never qualify; ties
|
|
70
|
+
// resolve to the deeper effort (the display order).
|
|
71
|
+
export function bestDepthAlternative(model: ModelEntry): DepthAlternative | null {
|
|
72
|
+
const base = model.quality
|
|
73
|
+
const selected = model.qualityEffort
|
|
74
|
+
const reasoning = model.reasoning
|
|
75
|
+
if (base == null || selected == null || reasoning === undefined) return null
|
|
76
|
+
|
|
77
|
+
const depths = [...reasoning.depths].sort((a, b) => effortOrder(a.effort) - effortOrder(b.effort))
|
|
78
|
+
let best: DepthAlternative | null = null
|
|
79
|
+
for (const depth of depths) {
|
|
80
|
+
if (depth.index === null || depth.effort === selected) continue
|
|
81
|
+
if (!isSelectableDepth(model, depth.effort)) continue
|
|
82
|
+
if (depth.index <= base) continue
|
|
83
|
+
if (best === null || depth.index > best.index) {
|
|
84
|
+
best = { effort: depth.effort, index: depth.index, cost: depth.cost_per_task_usd }
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
return best
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// The report's compact alternative label: "<effort> <index>" ("max 56.0"),
|
|
91
|
+
// the same shape upstream's site shows under QUAL. Empty when there is none.
|
|
92
|
+
export function formatDepthAlternative(model: ModelEntry | undefined): string {
|
|
93
|
+
if (model === undefined) return ""
|
|
94
|
+
const best = bestDepthAlternative(model)
|
|
95
|
+
return best === null ? "" : `${best.effort} ${best.index.toFixed(1)}`
|
|
96
|
+
}
|