@jeffreyjyz/mpc 0.1.4 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +95 -93
- package/package.json +13 -5
- package/src/cli/engine/cost.ts +1 -4
- package/src/cli/engine/index.ts +3 -12
- package/src/cli/engine/project.ts +1 -4
- package/src/cli/engine/rows.ts +2 -12
- package/src/cli/engine/score.ts +6 -18
- package/src/cli/flow/check.ts +6 -17
- package/src/cli/flow/collect.ts +1 -3
- package/src/cli/flow/columns.ts +1 -3
- package/src/cli/flow/sort.ts +7 -24
- package/src/cli/options.ts +1 -3
- package/src/cli/parse/cac.ts +13 -52
- package/src/cli/parse/fields.ts +2 -6
- package/src/cli/parse/map.ts +7 -24
- package/src/cli/parse/plugins.ts +3 -11
- package/src/cli/parse/validate.ts +4 -12
- package/src/cli/run.ts +6 -27
- package/src/constants/cli.ts +1 -5
- package/src/constants/view.ts +3 -12
- package/src/data/bench/cc.ts +3 -12
- package/src/data/bench/index.ts +6 -22
- package/src/data/bench/resolve.ts +5 -18
- package/src/data/scrape/catalog/catalog.ts +6 -25
- package/src/data/scrape/catalog/numeric.ts +2 -10
- package/src/data/scrape/index.ts +2 -9
- package/src/data/scrape/tabs.ts +2 -5
- package/src/data/shape.ts +4 -18
- package/src/data/sources/aa/api.ts +2 -6
- package/src/data/sources/aa/parse.ts +3 -12
- package/src/data/sources/aa/web.ts +1 -3
- package/src/data/sources/cc/catalog.ts +4 -16
- package/src/data/sources/cc/plans.ts +1 -3
- package/src/data/sources/opencode.ts +2 -9
- package/src/data/usage/index.ts +5 -19
- package/src/data/usage/logs.ts +2 -6
- package/src/data/usage/opencodeDb.ts +4 -11
- package/src/data/usage/opencodeV2.ts +1 -4
- package/src/index.ts +1 -3
- package/src/view/footer.ts +8 -31
- package/src/view/layout/fit.ts +4 -16
- package/src/view/layout/segments.ts +5 -22
- package/src/view/layout/table.ts +4 -14
- package/src/view/layout/usage.ts +4 -12
- package/src/view/render.ts +1 -6
- package/src/view/text/export.ts +2 -7
- package/src/view/text/format.ts +2 -8
- package/src/view/text/styles.ts +3 -10
package/README.md
CHANGED
|
@@ -60,17 +60,17 @@ mpc --usage --json # full projection
|
|
|
60
60
|
|
|
61
61
|
Sources, merged when more than one is present:
|
|
62
62
|
|
|
63
|
-
| source
|
|
64
|
-
|
|
|
65
|
-
| **opencode's message store** (`~/.local/share/opencode/opencode.db`, `--usage-db`, `OPENCODE_DB`)
|
|
63
|
+
| source | covers |
|
|
64
|
+
| ---------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------- |
|
|
65
|
+
| **opencode's message store** (`~/.local/share/opencode/opencode.db`, `--usage-db`, `OPENCODE_DB`) | every request opencode ran, for every provider — complete and backfilled |
|
|
66
66
|
| external per-request log (`--usage-log`, `MPC_USAGE_LOG`, default `$XDG_CACHE_HOME/mpc/usage.jsonl`) | any harness that writes one JSON line per request; consulted only when the DB is absent |
|
|
67
|
-
| `cmduse model --json --since <ISO>` (windowed model reports)
|
|
68
|
-
| session-log scan (`~/.commandcode/projects`)
|
|
69
|
-
| `--usage-file`
|
|
67
|
+
| `cmduse model --json --since <ISO>` (windowed model reports) | CommandCode CLI sessions on this machine |
|
|
68
|
+
| session-log scan (`~/.commandcode/projects`) | fallback when cmduse lacks the window |
|
|
69
|
+
| `--usage-file` | anything else you have |
|
|
70
70
|
|
|
71
71
|
The DB is read-only via `bun:sqlite`; assistant messages carry `cost`, `tokens` and `modelID`/`providerID`, so no plugin is required for the opencode side. Set `CMDUSE_BIN` to test against a dev cmduse (`cmdusedev`).
|
|
72
72
|
|
|
73
|
-
**Scope caveat.** Only
|
|
73
|
+
**Scope caveat.** Only _local_ sources exist — the account API exposes totals, not per-model usage, and Studio's API surface is the same endpoint. The report prints a coverage line (`local usage N of M account requests (x%)`) and warns below 90%, so a partial mix is visible rather than silently wrong.
|
|
74
74
|
|
|
75
75
|
```text
|
|
76
76
|
MODEL your req your $ CC $/req CC $/mo OC $/req OC $/mo cheaper
|
|
@@ -80,7 +80,7 @@ totals your mix · CC $2.4761/mo · OpenCode $2.7686/mo
|
|
|
80
80
|
head-to-head 2 of 2 models · CC $2.4761/mo · OpenCode $2.7686/mo · cheaper CommandCode by $0.2925 (12%)
|
|
81
81
|
```
|
|
82
82
|
|
|
83
|
-
Usage is per-model **totals**, so `your $` is the list value of the tokens, `CC $/mo` is what that subscription would cost you, and models that exceed a plan's allowance are flagged `over cap`. The `head-to-head` line answers "which plan is cheaper
|
|
83
|
+
Usage is per-model **totals**, so `your $` is the list value of the tokens, `CC $/mo` is what that subscription would cost you, and models that exceed a plan's allowance are flagged `over cap`. The `head-to-head` line answers "which plan is cheaper _for my mix_", so it is restricted to the models **both** plans price — a model only one provider sells would otherwise pad that side's total and "prove" the other cheaper on traffic it cannot serve. One-sided rows are excluded and counted. Unknown models are listed, never dropped. Accepts the cmduse shape, `{"entries": [...]}`, or a bare array of `{ model, requests, tokensIn, cacheRead, tokensOut }`.
|
|
84
84
|
|
|
85
85
|
## Config file
|
|
86
86
|
|
|
@@ -88,15 +88,15 @@ Every flag persists. `mpc` reads `~/.config/mpc/config.json` (or `$XDG_CONFIG_HO
|
|
|
88
88
|
|
|
89
89
|
```json
|
|
90
90
|
{
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
91
|
+
"ccPlan": "goat",
|
|
92
|
+
"ocPlan": "go",
|
|
93
|
+
"columns": ["model", "oc-per1k", "cc-per1k", "val"],
|
|
94
|
+
"presets": { "cheap": ["model", "oc-per1k", "cc-per1k", "cost"] },
|
|
95
|
+
"metric": "val",
|
|
96
|
+
"benchWeight": 0.4,
|
|
97
|
+
"valWeights": [0.4, 0.1, 0.25, 0.15, 0.1],
|
|
98
|
+
"color": false,
|
|
99
|
+
"plugins": ["./work.json", "mpc-preset-openai"]
|
|
100
100
|
}
|
|
101
101
|
```
|
|
102
102
|
|
|
@@ -114,78 +114,78 @@ Relative paths resolve against the config file's directory. A plugin is a `.json
|
|
|
114
114
|
|
|
115
115
|
## Options
|
|
116
116
|
|
|
117
|
-
| Flag
|
|
118
|
-
|
|
|
119
|
-
| `--cc-plan <id>`
|
|
120
|
-
| `--oc-plan <id>`
|
|
121
|
-
| `--in <n>`
|
|
122
|
-
| `--cache <n>`
|
|
123
|
-
| `--out <n>`
|
|
124
|
-
| `--reasoning <n>`
|
|
125
|
-
| `--cache-write <n>`
|
|
126
|
-
| `--shape <spec>`
|
|
127
|
-
| `--since <date>`
|
|
128
|
-
| `--metric <name>`
|
|
129
|
-
| `--model <re>`
|
|
130
|
-
| `--only <scope>`
|
|
131
|
-
| `--width <n>`
|
|
132
|
-
| `--minimal`
|
|
133
|
-
| `--medium`
|
|
134
|
-
| `--no-fit`
|
|
135
|
-
| `--columns <ids>`
|
|
136
|
-
| `--bench <src>`
|
|
137
|
-
| `--bench-weight <n>`
|
|
138
|
-
| `--tps-weight <n>`
|
|
139
|
-
| `--bench-name <label>`
|
|
140
|
-
| `--aa-key <key>`
|
|
141
|
-
| `--no-fallback`
|
|
142
|
-
| `--no-ability`
|
|
143
|
-
| `--preset <name>`
|
|
144
|
-
| `--val-weights <a,b,c,d,e>` | see above
|
|
145
|
-
| `--scale <mode>`
|
|
146
|
-
| `--inherit-suffixes <s>`
|
|
147
|
-
| `--window <five,week>`
|
|
148
|
-
| `--cost-thresholds <g,y>`
|
|
149
|
-
| `--val-thresholds <y,g>`
|
|
150
|
-
| `--format <name>`
|
|
151
|
-
| `--color <mode>`
|
|
152
|
-
| `--config <path>`
|
|
153
|
-
| `--no-config`
|
|
154
|
-
| `--plugin <paths>`
|
|
155
|
-
| `--print-config`
|
|
156
|
-
| `-h, --help`
|
|
157
|
-
| `-v, --version`
|
|
158
|
-
| `--peak`
|
|
159
|
-
| `--asc`
|
|
160
|
-
| `--detail`
|
|
161
|
-
| `--json`
|
|
162
|
-
| `--no-color`
|
|
163
|
-
| `--check`
|
|
117
|
+
| Flag | Default | Meaning |
|
|
118
|
+
| --------------------------- | -------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
119
|
+
| `--cc-plan <id>` | `goat` | CommandCode plan: `go`, `goat`, `pro`, `max10`, `max20` |
|
|
120
|
+
| `--oc-plan <id>` | `go` | OpenCode Go plan: `go`, `go-plus` (same token rates, higher per-model limits) |
|
|
121
|
+
| `--in <n>` | `800` | fixed input tokens per request |
|
|
122
|
+
| `--cache <n>` | `50000` | fixed cache-read tokens per request |
|
|
123
|
+
| `--out <n>` | `200` | fixed output tokens per request |
|
|
124
|
+
| `--reasoning <n>` | `0` | reasoning tokens per request, billed at the output rate on top of output |
|
|
125
|
+
| `--cache-write <n>` | `0` | cache-write tokens per request; a model that publishes no cache-write rate is priced at its input rate |
|
|
126
|
+
| `--shape <spec>` | `auto` | `auto` = reqshape's measured profile when it has ≥ `SHAPE_MIN_REQS` (500) reqs, else the fixed workload; `measured` forces reqshape; `off` keeps the fixed workload; a path reads a saved `reqshape --format json` payload |
|
|
127
|
+
| `--since <date>` | — | measured shape: only requests on or after this date |
|
|
128
|
+
| `--metric <name>` | `val` | sort by `val`, `cost`, `perreq`, `req` or `name`; `cost` and `perreq` ascend (lower better), `val`/`req` descend, `--asc` flips; rows with no `VAL` always sort last |
|
|
129
|
+
| `--model <re>` | — | filter rows by name (regex, substring fallback) |
|
|
130
|
+
| `--only <scope>` | `all` | `all` = union of both catalogs, `both` = only shared models |
|
|
131
|
+
| `--width <n>` | terminal | force table width; by default the table trims to the terminal and drops optional columns (`rates`, `req/$`, `5h`/`wk`) until it fits |
|
|
132
|
+
| `--minimal` | off | fewest columns: model + req/mo both sides + win + val |
|
|
133
|
+
| `--medium` | off | preset: allowance and the rate views per side + win + cost + val |
|
|
134
|
+
| `--no-fit` | off | keep the preset's full width instead of trimming to the terminal |
|
|
135
|
+
| `--columns <ids>` | preset | comma-separated columns to show, in order (overrides presets and trimming); `--columns help` lists ids |
|
|
136
|
+
| `--bench <src>` | `aa` with a key, else `cc` | ability scores: `cc`, `aa`, `aa-web`, `file:<path>`, `url:<url>` |
|
|
137
|
+
| `--bench-weight <n>` | `0.35` | ability share of `VAL`, 0-1 |
|
|
138
|
+
| `--tps-weight <n>` | `0.10` | output-speed share of `VAL`, 0-1 |
|
|
139
|
+
| `--bench-name <label>` | source | footer label for the ability source |
|
|
140
|
+
| `--aa-key <key>` | `AA_API_KEY` | Artificial Analysis API key |
|
|
141
|
+
| `--no-fallback` | off | skip filling ability misses from the other sources |
|
|
142
|
+
| `--no-ability` | off | hide `ability` and `VAL` |
|
|
143
|
+
| `--preset <name>` | — | named column set from `presets` in config |
|
|
144
|
+
| `--val-weights <a,b,c,d,e>` | see above | VAL weights (ability, tps, volume, cache, output) |
|
|
145
|
+
| `--scale <mode>` | `log` | `log` or `linear` normalisation for skewed terms |
|
|
146
|
+
| `--inherit-suffixes <s>` | built-in | speed-variant suffixes that inherit ability |
|
|
147
|
+
| `--window <five,week>` | derived | override rolling-window ratios |
|
|
148
|
+
| `--cost-thresholds <g,y>` | `30,60` | COST colour cut-offs |
|
|
149
|
+
| `--val-thresholds <y,g>` | `40,70` | VAL colour cut-offs |
|
|
150
|
+
| `--format <name>` | `table` | `table`, `json`, `csv`, `md` |
|
|
151
|
+
| `--color <mode>` | `auto` | `auto`, `always`, `never` |
|
|
152
|
+
| `--config <path>` | XDG | config file |
|
|
153
|
+
| `--no-config` | off | ignore config and plugins |
|
|
154
|
+
| `--plugin <paths>` | — | extra config plugins, comma-separated |
|
|
155
|
+
| `--print-config` | — | print effective settings and exit |
|
|
156
|
+
| `-h, --help` | — | generated help (cac) |
|
|
157
|
+
| `-v, --version` | — | print version |
|
|
158
|
+
| `--peak` | off | use peak-rate rows instead of off-peak (DeepSeek) |
|
|
159
|
+
| `--asc` | off | sort ascending |
|
|
160
|
+
| `--detail` | off | every column, untrimmed; the default is every column trimmed to the terminal width |
|
|
161
|
+
| `--json` | off | emit JSON instead of a table |
|
|
162
|
+
| `--no-color` | off | disable ANSI colour |
|
|
163
|
+
| `--check` | off | validate sources, list unmatched models, exit |
|
|
164
164
|
|
|
165
165
|
## Data sources (all fetched live)
|
|
166
166
|
|
|
167
|
-
| Source
|
|
168
|
-
|
|
|
169
|
-
| `cmduse plans --json`
|
|
170
|
-
| `commandcode.ai/docs/plans/{goat,pro,max}` | per-model credits + token rates
|
|
171
|
-
| `opencode.ai/v2/docs/console/go/`
|
|
172
|
-
| `opencode.ai/zen/go/v1/models`
|
|
167
|
+
| Source | Provides |
|
|
168
|
+
| --------------------------------------------- | ---------------------------------------------------------------- |
|
|
169
|
+
| `cmduse plans --json` | CommandCode plan price, credits, 5h/weekly windows |
|
|
170
|
+
| `commandcode.ai/docs/plans/{goat,pro,max,go}` | per-model credits + token rates |
|
|
171
|
+
| `opencode.ai/v2/docs/console/go/` | oc-go per-model monthly limit (per Go plan) + shared token rates |
|
|
172
|
+
| `opencode.ai/zen/go/v1/models` | oc-go live model list (`--check` drift) |
|
|
173
173
|
|
|
174
174
|
## Reading the table
|
|
175
175
|
|
|
176
|
-
| Column
|
|
177
|
-
|
|
|
178
|
-
| `in/out/cache` (`--detail`)
|
|
179
|
-
| `allow`
|
|
180
|
-
| `req/5h`, `req/wk` (`--detail`) | requests the plan's rolling 5-hour / weekly window allows
|
|
181
|
-
| `req/mo`
|
|
182
|
-
| `$/1K`
|
|
183
|
-
| `req/$`
|
|
184
|
-
| `ability`
|
|
185
|
-
| `tps`
|
|
186
|
-
| `WIN`
|
|
187
|
-
| `COST`
|
|
188
|
-
| `VAL`
|
|
176
|
+
| Column | Meaning |
|
|
177
|
+
| ------------------------------- | ------------------------------------------------------------------- |
|
|
178
|
+
| `in/out/cache` (`--detail`) | model token rates, USD per 1M tokens |
|
|
179
|
+
| `allow` | monthly credits this plan devotes to that model |
|
|
180
|
+
| `req/5h`, `req/wk` (`--detail`) | requests the plan's rolling 5-hour / weekly window allows |
|
|
181
|
+
| `req/mo` | requests the allowance buys (`allowance / costPerRequest`) |
|
|
182
|
+
| `$/1K` | what 1,000 requests cost you on the plan |
|
|
183
|
+
| `req/$` | requests one dollar of subscription buys |
|
|
184
|
+
| `ability` | benchmark score for the model |
|
|
185
|
+
| `tps` | output tokens per second |
|
|
186
|
+
| `WIN` | side with the lower per-request cost |
|
|
187
|
+
| `COST` | 0-100 volume index: `requests/mo`, **lower is better** (no ability) |
|
|
188
|
+
| `VAL` | 0-100 ability-aware value score |
|
|
189
189
|
|
|
190
190
|
`--columns a,b,c` picks and orders columns; ids are listed under `--columns help`, and `cc-*` mirrors the `oc-*` set. Without it, plain `mpc` shows every column that fits the terminal, `--minimal` and `--medium` narrow the set, and `--detail` prints all of them untrimmed.
|
|
191
191
|
|
|
@@ -208,13 +208,13 @@ payPerRequest = planPrice * costPerRequest / allowance # what you
|
|
|
208
208
|
multiplier = allowance / planPrice # $usage per $paid
|
|
209
209
|
```
|
|
210
210
|
|
|
211
|
-
Reasoning bills at the output rate
|
|
211
|
+
Reasoning bills at the output rate _on top of_ output — opencode's own provider-priced rows reproduce exactly that way (a GLM-5.3 turn of 8,689 input / 14 output / 38 reasoning / 128 cache-read is priced at `0.01242668`, which only matches when reasoning joins the output term). A cache-write rate the model does not publish falls back to its input rate rather than to free.
|
|
212
212
|
|
|
213
213
|
### Measuring instead of assuming
|
|
214
214
|
|
|
215
215
|
By default mpc asks **reqshape** for the shape of your real traffic, read from opencode's own store, and prices **both** plans on that single per-req profile, so the comparison isolates price and allowance from traffic and the footer prints one workload line. reqshape only leads once it has `SHAPE_MIN_REQS` (500) measured requests — below that the fixed 800/50K/200 workload is steadier. The footer's `shape` line says plainly which was used, the sample size behind the choice, and the alternative flag.
|
|
216
216
|
|
|
217
|
-
`req/mo` then answers "how many of
|
|
217
|
+
`req/mo` then answers "how many of _my_ requests fit this allowance" rather than "how many of a hypothetical 800/50K/200 ones do". Force the fixed workload with `--shape off`, force reqshape regardless of sample with `--shape measured`, or save a payload once with `reqshape --format json > shape.json` and reuse it with `--shape shape.json`; `--since <date>` narrows the window. A missing `reqshape` binary (`REQSHAPE_BIN` overrides it) is a warning, not a failure: mpc keeps the fixed workload and carries on.
|
|
218
218
|
|
|
219
219
|
The **index** behind `COST` is a 0-100 volume score across every model-provider entry:
|
|
220
220
|
|
|
@@ -235,12 +235,12 @@ VAL = 100 * (0.35*ability + 0.10*tps + 0.25*volume + 0.15*cache + 0.15*output)
|
|
|
235
235
|
|
|
236
236
|
`--bench-weight` and `--tps-weight` set the ability and speed shares; the remaining weight splits volume/cache/output 50/25/25. Ability and `tps` come from the same source, so `--bench cc` reads both CommandCode's `Intelligence` and `Tok/s` columns. Speed variants (`…Fast`, `…HighSpeed`, `…UltraSpeed`, `…FlashX`) inherit their base model's ability — same weights — and, when the benchmark publishes no throughput for the variant, the base's throughput ×`SPEED_TPS_FACTOR` (a speed tier is multiples of its base). Without that a Fast model was scored at the neutral rate, ranking it below its slower base. Unscored models show `ability —` and `VAL —` and are excluded from the ability normalisation range.
|
|
237
237
|
|
|
238
|
-
| `--bench`
|
|
239
|
-
|
|
|
240
|
-
| `aa`
|
|
241
|
-
| `aa-web`
|
|
242
|
-
| `cc`
|
|
243
|
-
| `file:<path>` / `url:<url>` | your JSON, `{ "model": score }` or `[{ model, score }]` | whatever you supply
|
|
238
|
+
| `--bench` | source | coverage |
|
|
239
|
+
| --------------------------- | ------------------------------------------------------- | ----------------------------------- |
|
|
240
|
+
| `aa` | Artificial Analysis API (paginated) | full; needs `AA_API_KEY` |
|
|
241
|
+
| `aa-web` | Artificial Analysis models page scrape | partial (only the models AA embeds) |
|
|
242
|
+
| `cc` | CommandCode's `Intelligence` column | every matched model |
|
|
243
|
+
| `file:<path>` / `url:<url>` | your JSON, `{ "model": score }` or `[{ model, score }]` | whatever you supply |
|
|
244
244
|
|
|
245
245
|
The default is `aa` when an AA key is available (`AA_API_KEY` or `--aa-key`), otherwise `cc`; an explicit `--bench` always wins. After the lead source, misses are filled from the same benchmark's keyless page scrape (`aa-web`) and then from CommandCode, the last resort — `--no-fallback` disables the fills. Because CC's `Intelligence` column and AA publish the same index, a `cc`-led run asks AA only when it still lacks throughput (CC's plan pages no longer publish `Tok/s`). Everything is fetched live, every run — **`mpc` writes no cache**: `aa-web` reads the index and speed straight out of the models page's embedded React Server Components payload, so a source that is down costs only the columns it alone supplies, never a stale value.
|
|
246
246
|
|
|
@@ -262,8 +262,10 @@ Names from the two catalogs are collapsed onto one canonical key (lowercase, ven
|
|
|
262
262
|
```sh
|
|
263
263
|
bun install
|
|
264
264
|
bun test # unit tests (parsers, normalisation, metrics)
|
|
265
|
-
bun
|
|
266
|
-
bun
|
|
265
|
+
bun check # types (Bun's own check)
|
|
266
|
+
bun fmt # Bun's formatter
|
|
267
|
+
bun lint # Biome lint
|
|
268
|
+
bun fmt --check # verify formatting
|
|
267
269
|
```
|
|
268
270
|
|
|
269
271
|
## Links
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@jeffreyjyz/mpc",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.6",
|
|
4
4
|
"description": "Compare model pricing across opencode Go and Command Code plans",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -21,9 +21,8 @@
|
|
|
21
21
|
"scripts": {
|
|
22
22
|
"start": "bun run src/index.ts",
|
|
23
23
|
"test": "bun test",
|
|
24
|
-
"
|
|
25
|
-
"
|
|
26
|
-
"typecheck": "tsc --noEmit"
|
|
24
|
+
"lint": "biome lint --write .",
|
|
25
|
+
"typecheck": "bun check"
|
|
27
26
|
},
|
|
28
27
|
"devDependencies": {
|
|
29
28
|
"@biomejs/biome": "^2.5.13",
|
|
@@ -32,5 +31,14 @@
|
|
|
32
31
|
},
|
|
33
32
|
"dependencies": {
|
|
34
33
|
"cac": "^7.0.0"
|
|
35
|
-
}
|
|
34
|
+
},
|
|
35
|
+
"repository": {
|
|
36
|
+
"type": "git",
|
|
37
|
+
"url": "git+https://github.com/JeffreyJYZ/cmdcode-tools.git",
|
|
38
|
+
"directory": "oc-cmd-compare"
|
|
39
|
+
},
|
|
40
|
+
"bugs": {
|
|
41
|
+
"url": "https://github.com/JeffreyJYZ/cmdcode-tools/issues"
|
|
42
|
+
},
|
|
43
|
+
"homepage": "https://github.com/JeffreyJYZ/cmdcode-tools/tree/main/oc-cmd-compare#readme"
|
|
36
44
|
}
|
package/src/cli/engine/cost.ts
CHANGED
|
@@ -10,10 +10,7 @@ import type { ModelPricing, PlanInfo, Workload } from "#~/types.ts";
|
|
|
10
10
|
* output ones at the 4.4 $/M output rate). A model that publishes no
|
|
11
11
|
* cache-write rate is priced at its input rate rather than assumed free.
|
|
12
12
|
*/
|
|
13
|
-
export function costPerRequest(
|
|
14
|
-
pricing: ModelPricing,
|
|
15
|
-
workload: Workload,
|
|
16
|
-
): number {
|
|
13
|
+
export function costPerRequest(pricing: ModelPricing, workload: Workload): number {
|
|
17
14
|
const output = workload.output + workload.reasoning;
|
|
18
15
|
return (
|
|
19
16
|
(workload.input * pricing.input +
|
package/src/cli/engine/index.ts
CHANGED
|
@@ -1,11 +1,5 @@
|
|
|
1
1
|
import { DEFAULT_SCORE } from "#~/constants/scoring.ts";
|
|
2
|
-
import type {
|
|
3
|
-
CatalogEntry,
|
|
4
|
-
EntryMetrics,
|
|
5
|
-
PlanInfo,
|
|
6
|
-
ProviderId,
|
|
7
|
-
Workload,
|
|
8
|
-
} from "#~/types.ts";
|
|
2
|
+
import type { CatalogEntry, EntryMetrics, PlanInfo, ProviderId, Workload } from "#~/types.ts";
|
|
9
3
|
import { lookupAbility, lookupTps } from "./ability.ts";
|
|
10
4
|
import { costPerRequest, windowRatios } from "./cost.ts";
|
|
11
5
|
import { assignIndex, assignValueIndex, type ScoreConfig } from "./score.ts";
|
|
@@ -20,14 +14,11 @@ export function buildMetrics(
|
|
|
20
14
|
): EntryMetrics[] {
|
|
21
15
|
const metrics: EntryMetrics[] = entries.map((entry) => {
|
|
22
16
|
const plan = plans.get(entry.provider);
|
|
23
|
-
if (!plan)
|
|
24
|
-
throw new Error(`no plan loaded for provider ${entry.provider}`);
|
|
17
|
+
if (!plan) throw new Error(`no plan loaded for provider ${entry.provider}`);
|
|
25
18
|
const workload = workloads[entry.provider];
|
|
26
19
|
const cost = costPerRequest(entry.pricing, workload);
|
|
27
20
|
const free = cost === 0;
|
|
28
|
-
const requestsPerMonth = free
|
|
29
|
-
? Number.POSITIVE_INFINITY
|
|
30
|
-
: entry.allowance / cost;
|
|
21
|
+
const requestsPerMonth = free ? Number.POSITIVE_INFINITY : entry.allowance / cost;
|
|
31
22
|
const ratios = windowRatios(plan, config.window);
|
|
32
23
|
return {
|
|
33
24
|
provider: entry.provider,
|
|
@@ -72,10 +72,7 @@ function side(
|
|
|
72
72
|
): SideProjection {
|
|
73
73
|
const requests = Math.max(usage.requests, 1e-9);
|
|
74
74
|
const costPerRequest = periodCost / requests;
|
|
75
|
-
const payPerRequest =
|
|
76
|
-
entry.allowance > 0
|
|
77
|
-
? (plan.price * costPerRequest) / entry.allowance
|
|
78
|
-
: 0;
|
|
75
|
+
const payPerRequest = entry.allowance > 0 ? (plan.price * costPerRequest) / entry.allowance : 0;
|
|
79
76
|
const creditsDrawn = periodCost / months;
|
|
80
77
|
return {
|
|
81
78
|
costPerRequest,
|
package/src/cli/engine/rows.ts
CHANGED
|
@@ -10,10 +10,7 @@ import type {
|
|
|
10
10
|
import { buildMetrics } from "./index.ts";
|
|
11
11
|
import type { ScoreConfig } from "./score.ts";
|
|
12
12
|
|
|
13
|
-
function indexByKey(
|
|
14
|
-
entries: CatalogEntry[],
|
|
15
|
-
metrics: EntryMetrics[],
|
|
16
|
-
): Map<string, EntryMetrics> {
|
|
13
|
+
function indexByKey(entries: CatalogEntry[], metrics: EntryMetrics[]): Map<string, EntryMetrics> {
|
|
17
14
|
const byKey = new Map<string, EntryMetrics>();
|
|
18
15
|
entries.forEach((entry, i) => {
|
|
19
16
|
const metric = metrics[i];
|
|
@@ -37,14 +34,7 @@ export function buildRows(
|
|
|
37
34
|
["cc", ccPlan],
|
|
38
35
|
]);
|
|
39
36
|
// Score every entry from both providers against one shared scale.
|
|
40
|
-
const all = buildMetrics(
|
|
41
|
-
[...ocEntries, ...ccEntries],
|
|
42
|
-
plans,
|
|
43
|
-
workloads,
|
|
44
|
-
ability,
|
|
45
|
-
tps,
|
|
46
|
-
config,
|
|
47
|
-
);
|
|
37
|
+
const all = buildMetrics([...ocEntries, ...ccEntries], plans, workloads, ability, tps, config);
|
|
48
38
|
const ocByKey = indexByKey(ocEntries, all.slice(0, ocEntries.length));
|
|
49
39
|
const ccByKey = indexByKey(ccEntries, all.slice(ocEntries.length));
|
|
50
40
|
|
package/src/cli/engine/score.ts
CHANGED
|
@@ -13,12 +13,8 @@ export interface ScoreConfig {
|
|
|
13
13
|
window?: [number, number];
|
|
14
14
|
}
|
|
15
15
|
|
|
16
|
-
function normalise(
|
|
17
|
-
weights
|
|
18
|
-
fallback: number[],
|
|
19
|
-
): number[] {
|
|
20
|
-
const values =
|
|
21
|
-
weights && weights.length === fallback.length ? weights : fallback;
|
|
16
|
+
function normalise(weights: number[] | undefined, fallback: number[]): number[] {
|
|
17
|
+
const values = weights && weights.length === fallback.length ? weights : fallback;
|
|
22
18
|
const total = values.reduce((a, b) => a + b, 0);
|
|
23
19
|
return total > 0 ? values.map((v) => v / total) : fallback;
|
|
24
20
|
}
|
|
@@ -32,9 +28,7 @@ const scale = (values: number[], mode: ScaleMode): number[] =>
|
|
|
32
28
|
* "cost" here means "how many requests the plan buys". */
|
|
33
29
|
export function assignIndex(metrics: EntryMetrics[]): void {
|
|
34
30
|
const priced = metrics.filter((m) => Number.isFinite(m.requestsPerMonth));
|
|
35
|
-
const volume = minmax(
|
|
36
|
-
priced.map((m) => Math.log10(Math.max(m.requestsPerMonth, 1))),
|
|
37
|
-
);
|
|
31
|
+
const volume = minmax(priced.map((m) => Math.log10(Math.max(m.requestsPerMonth, 1))));
|
|
38
32
|
|
|
39
33
|
priced.forEach((m, i) => {
|
|
40
34
|
m.index = Math.round(100 * (volume[i] ?? 0.5));
|
|
@@ -46,10 +40,7 @@ export function assignIndex(metrics: EntryMetrics[]): void {
|
|
|
46
40
|
}
|
|
47
41
|
|
|
48
42
|
/** VAL: ability / speed / volume / cache / output. Unscored models stay null. */
|
|
49
|
-
export function assignValueIndex(
|
|
50
|
-
metrics: EntryMetrics[],
|
|
51
|
-
config: ScoreConfig,
|
|
52
|
-
): void {
|
|
43
|
+
export function assignValueIndex(metrics: EntryMetrics[], config: ScoreConfig): void {
|
|
53
44
|
const weights = normalise(config.valWeights, [
|
|
54
45
|
config.abilityWeight,
|
|
55
46
|
config.tpsWeight,
|
|
@@ -67,9 +58,7 @@ export function assignValueIndex(
|
|
|
67
58
|
speeded.map((m) => m.tps ?? 1),
|
|
68
59
|
config.scale,
|
|
69
60
|
);
|
|
70
|
-
const volume = minmax(
|
|
71
|
-
priced.map((m) => Math.log10(Math.max(m.requestsPerMonth, 1))),
|
|
72
|
-
);
|
|
61
|
+
const volume = minmax(priced.map((m) => Math.log10(Math.max(m.requestsPerMonth, 1))));
|
|
73
62
|
const cache = scale(
|
|
74
63
|
priced.map((m) => m.pricing.cacheRead),
|
|
75
64
|
config.scale,
|
|
@@ -112,7 +101,6 @@ export function assignValueIndex(
|
|
|
112
101
|
const low = values.length > 0 ? Math.min(...values) : 0;
|
|
113
102
|
const high = values.length > 0 ? Math.max(...values) : 0;
|
|
114
103
|
for (const [m, value] of composites) {
|
|
115
|
-
m.valueIndex =
|
|
116
|
-
high > low ? Math.round((100 * (value - low)) / (high - low)) : 100;
|
|
104
|
+
m.valueIndex = high > low ? Math.round((100 * (value - low)) / (high - low)) : 100;
|
|
117
105
|
}
|
|
118
106
|
}
|
package/src/cli/flow/check.ts
CHANGED
|
@@ -24,16 +24,10 @@ export async function runCheck(
|
|
|
24
24
|
|
|
25
25
|
console.log(`OpenCode entries parsed: ${ocEntries.length}`);
|
|
26
26
|
console.log(`CommandCode entries: ${ccEntries.length}`);
|
|
27
|
-
console.log(
|
|
28
|
-
|
|
29
|
-
);
|
|
30
|
-
console.log(
|
|
31
|
-
`free OpenCode models: ${ocEntries.filter((e) => e.allowance === 0).length}`,
|
|
32
|
-
);
|
|
27
|
+
console.log(`matched models: ${rows.filter((r) => r.oc && r.cc).length}`);
|
|
28
|
+
console.log(`free OpenCode models: ${ocEntries.filter((e) => e.allowance === 0).length}`);
|
|
33
29
|
|
|
34
|
-
const scored = rows.filter(
|
|
35
|
-
(r) => r.oc?.ability != null || r.cc?.ability != null,
|
|
36
|
-
).length;
|
|
30
|
+
const scored = rows.filter((r) => r.oc?.ability != null || r.cc?.ability != null).length;
|
|
37
31
|
console.log(
|
|
38
32
|
`ability source: ${ability.label || "disabled"} (${ability.intelligence.size} scores, ${ability.tps.size} speed, ${scored}/${rows.length} rows scored)`,
|
|
39
33
|
);
|
|
@@ -44,8 +38,7 @@ export async function runCheck(
|
|
|
44
38
|
.map((r) => r.key);
|
|
45
39
|
if (unscored.length > 0) {
|
|
46
40
|
const head = unscored.slice(0, 20).join(", ");
|
|
47
|
-
const more =
|
|
48
|
-
unscored.length > 20 ? ` +${unscored.length - 20} more` : "";
|
|
41
|
+
const more = unscored.length > 20 ? ` +${unscored.length - 20} more` : "";
|
|
49
42
|
console.log(`ability missing (${unscored.length}): ${head}${more}`);
|
|
50
43
|
}
|
|
51
44
|
|
|
@@ -56,11 +49,7 @@ export async function runCheck(
|
|
|
56
49
|
`OpenCode ids missing from docs table (${unpriced.length}): ${unpriced.join(", ") || "—"}`,
|
|
57
50
|
);
|
|
58
51
|
|
|
59
|
-
console.log(
|
|
60
|
-
|
|
61
|
-
);
|
|
62
|
-
console.log(
|
|
63
|
-
`only in CommandCode (${onlyCc.length}): ${onlyCc.join(", ") || "—"}`,
|
|
64
|
-
);
|
|
52
|
+
console.log(`\nonly in OpenCode (${onlyOc.length}): ${onlyOc.join(", ") || "—"}`);
|
|
53
|
+
console.log(`only in CommandCode (${onlyCc.length}): ${onlyCc.join(", ") || "—"}`);
|
|
65
54
|
return 0;
|
|
66
55
|
}
|
package/src/cli/flow/collect.ts
CHANGED
|
@@ -35,9 +35,7 @@ export async function collect(options: Options) {
|
|
|
35
35
|
cc: workload,
|
|
36
36
|
};
|
|
37
37
|
// Ability is loaded after the catalogs so fills can be limited to our rows.
|
|
38
|
-
const keys = new Set(
|
|
39
|
-
[...ocEntries, ...ccEntries].map((entry) => entry.key),
|
|
40
|
-
);
|
|
38
|
+
const keys = new Set([...ocEntries, ...ccEntries].map((entry) => entry.key));
|
|
41
39
|
const ability = options.noAbility
|
|
42
40
|
? {
|
|
43
41
|
intelligence: new Map<string, number>(),
|
package/src/cli/flow/columns.ts
CHANGED
|
@@ -8,9 +8,7 @@ import { SETS, TIERS, type Tier } from "#~/constants/cli.ts";
|
|
|
8
8
|
function tierOf(options: Options): Tier {
|
|
9
9
|
const picked = TIERS.filter((tier) => options[tier]);
|
|
10
10
|
if (picked.length > 1) {
|
|
11
|
-
throw new Error(
|
|
12
|
-
`--${picked.join(" and --")} conflict — pick one column preset`,
|
|
13
|
-
);
|
|
11
|
+
throw new Error(`--${picked.join(" and --")} conflict — pick one column preset`);
|
|
14
12
|
}
|
|
15
13
|
return picked[0] ?? "detail";
|
|
16
14
|
}
|
package/src/cli/flow/sort.ts
CHANGED
|
@@ -17,8 +17,7 @@ const maxValue = (r: CompareRow): number | null => {
|
|
|
17
17
|
);
|
|
18
18
|
return values.length === 0 ? null : Math.max(...values);
|
|
19
19
|
};
|
|
20
|
-
const maxIndex = (r: CompareRow): number =>
|
|
21
|
-
Math.max(r.oc?.index ?? -1, r.cc?.index ?? -1);
|
|
20
|
+
const maxIndex = (r: CompareRow): number => Math.max(r.oc?.index ?? -1, r.cc?.index ?? -1);
|
|
22
21
|
const maxRequests = (r: CompareRow): number =>
|
|
23
22
|
Math.max(r.oc?.requestsPerMonth ?? -1, r.cc?.requestsPerMonth ?? -1);
|
|
24
23
|
const minPay = (r: CompareRow): number => {
|
|
@@ -29,40 +28,24 @@ const minPay = (r: CompareRow): number => {
|
|
|
29
28
|
};
|
|
30
29
|
|
|
31
30
|
/** `asc` means "lower is better"; it flips each metric's default direction. */
|
|
32
|
-
export function sortRows(
|
|
33
|
-
rows: CompareRow[],
|
|
34
|
-
metric: Metric,
|
|
35
|
-
asc: boolean,
|
|
36
|
-
): CompareRow[] {
|
|
31
|
+
export function sortRows(rows: CompareRow[], metric: Metric, asc: boolean): CompareRow[] {
|
|
37
32
|
switch (metric) {
|
|
38
33
|
case "name": {
|
|
39
|
-
const sorted = [...rows].sort((a, b) =>
|
|
40
|
-
a.name.localeCompare(b.name),
|
|
41
|
-
);
|
|
34
|
+
const sorted = [...rows].sort((a, b) => a.name.localeCompare(b.name));
|
|
42
35
|
return asc ? sorted : sorted.reverse();
|
|
43
36
|
}
|
|
44
37
|
case "val": {
|
|
45
38
|
const scored = rows
|
|
46
39
|
.filter((r) => maxValue(r) !== null)
|
|
47
|
-
.sort(
|
|
48
|
-
(a, b) =>
|
|
49
|
-
((maxValue(a) ?? 0) - (maxValue(b) ?? 0)) *
|
|
50
|
-
(asc ? 1 : -1),
|
|
51
|
-
);
|
|
40
|
+
.sort((a, b) => ((maxValue(a) ?? 0) - (maxValue(b) ?? 0)) * (asc ? 1 : -1));
|
|
52
41
|
const unscored = rows.filter((r) => maxValue(r) === null);
|
|
53
42
|
return [...scored, ...unscored];
|
|
54
43
|
}
|
|
55
44
|
case "cost":
|
|
56
|
-
return [...rows].sort(
|
|
57
|
-
(a, b) => (maxIndex(a) - maxIndex(b)) * (asc ? -1 : 1),
|
|
58
|
-
);
|
|
45
|
+
return [...rows].sort((a, b) => (maxIndex(a) - maxIndex(b)) * (asc ? -1 : 1));
|
|
59
46
|
case "perreq":
|
|
60
|
-
return [...rows].sort(
|
|
61
|
-
(a, b) => (minPay(a) - minPay(b)) * (asc ? -1 : 1),
|
|
62
|
-
);
|
|
47
|
+
return [...rows].sort((a, b) => (minPay(a) - minPay(b)) * (asc ? -1 : 1));
|
|
63
48
|
case "req":
|
|
64
|
-
return [...rows].sort(
|
|
65
|
-
(a, b) => (maxRequests(a) - maxRequests(b)) * (asc ? 1 : -1),
|
|
66
|
-
);
|
|
49
|
+
return [...rows].sort((a, b) => (maxRequests(a) - maxRequests(b)) * (asc ? 1 : -1));
|
|
67
50
|
}
|
|
68
51
|
}
|
package/src/cli/options.ts
CHANGED
|
@@ -63,9 +63,7 @@ export function parseMetric(value: string | undefined): Metric {
|
|
|
63
63
|
|
|
64
64
|
export function parseFormat(value: string | undefined): Format {
|
|
65
65
|
if (!FORMATS.includes(value as Format)) {
|
|
66
|
-
throw new Error(
|
|
67
|
-
`unknown --format "${value}" (table | json | csv | md)`,
|
|
68
|
-
);
|
|
66
|
+
throw new Error(`unknown --format "${value}" (table | json | csv | md)`);
|
|
69
67
|
}
|
|
70
68
|
return value as Format;
|
|
71
69
|
}
|