@jeffreyjyz/mpc 0.1.4 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +95 -93
- package/package.json +13 -5
- package/src/cli/engine/cost.ts +1 -4
- package/src/cli/engine/index.ts +3 -12
- package/src/cli/engine/project.ts +1 -4
- package/src/cli/engine/rows.ts +2 -12
- package/src/cli/engine/score.ts +6 -18
- package/src/cli/flow/check.ts +6 -17
- package/src/cli/flow/collect.ts +1 -3
- package/src/cli/flow/columns.ts +1 -3
- package/src/cli/flow/sort.ts +7 -24
- package/src/cli/options.ts +1 -3
- package/src/cli/parse/cac.ts +13 -52
- package/src/cli/parse/fields.ts +2 -6
- package/src/cli/parse/map.ts +7 -24
- package/src/cli/parse/plugins.ts +3 -11
- package/src/cli/parse/validate.ts +4 -12
- package/src/cli/run.ts +6 -27
- package/src/constants/cli.ts +1 -5
- package/src/constants/view.ts +3 -12
- package/src/data/bench/cc.ts +3 -12
- package/src/data/bench/index.ts +6 -22
- package/src/data/bench/resolve.ts +5 -18
- package/src/data/scrape/catalog/catalog.ts +6 -25
- package/src/data/scrape/catalog/numeric.ts +2 -10
- package/src/data/scrape/index.ts +2 -9
- package/src/data/scrape/tabs.ts +2 -5
- package/src/data/shape.ts +4 -18
- package/src/data/sources/aa/api.ts +2 -6
- package/src/data/sources/aa/parse.ts +3 -12
- package/src/data/sources/aa/web.ts +1 -3
- package/src/data/sources/cc/catalog.ts +4 -16
- package/src/data/sources/cc/plans.ts +1 -3
- package/src/data/sources/opencode.ts +2 -9
- package/src/data/usage/index.ts +5 -19
- package/src/data/usage/logs.ts +2 -6
- package/src/data/usage/opencodeDb.ts +4 -11
- package/src/data/usage/opencodeV2.ts +1 -4
- package/src/index.ts +1 -3
- package/src/view/footer.ts +8 -31
- package/src/view/layout/fit.ts +4 -16
- package/src/view/layout/segments.ts +5 -22
- package/src/view/layout/table.ts +4 -14
- package/src/view/layout/usage.ts +4 -12
- package/src/view/render.ts +1 -6
- package/src/view/text/export.ts +2 -7
- package/src/view/text/format.ts +2 -8
- package/src/view/text/styles.ts +3 -10
package/src/cli/parse/cac.ts
CHANGED
|
@@ -6,48 +6,27 @@ import { type Bag, toOptions } from "./map.ts";
|
|
|
6
6
|
/** Raw CLI bag: exactly what the user typed, no config, no defaults. */
|
|
7
7
|
export function parseFlags(argv: string[]): Bag {
|
|
8
8
|
const cli = cac("mpc");
|
|
9
|
-
cli.option(
|
|
10
|
-
"--cc-plan <id>",
|
|
11
|
-
"CommandCode plan: go, goat, pro, max10, max20",
|
|
12
|
-
)
|
|
9
|
+
cli.option("--cc-plan <id>", "CommandCode plan: go, goat, pro, max10, max20")
|
|
13
10
|
.option("--oc-plan <id>", "OpenCode Go plan: go, go-plus")
|
|
14
11
|
.option("--in <n>", "input tokens per request")
|
|
15
12
|
.option("--cache <n>", "cache-read tokens per request")
|
|
16
13
|
.option("--out <n>", "output tokens per request")
|
|
17
|
-
.option(
|
|
18
|
-
"--reasoning <n>",
|
|
19
|
-
"reasoning tokens, billed at the output rate",
|
|
20
|
-
)
|
|
14
|
+
.option("--reasoning <n>", "reasoning tokens, billed at the output rate")
|
|
21
15
|
.option("--cache-write <n>", "cache-write tokens per request")
|
|
22
16
|
.option(
|
|
23
17
|
"--shape <spec>",
|
|
24
18
|
"auto | measured | <file> | off: reqshape shape (auto = measured with enough data)",
|
|
25
19
|
)
|
|
26
|
-
.option(
|
|
27
|
-
"--since <date>",
|
|
28
|
-
"measured shape: only requests on or after this date",
|
|
29
|
-
)
|
|
20
|
+
.option("--since <date>", "measured shape: only requests on or after this date")
|
|
30
21
|
.option("--metric <name>", "val | cost | perreq | req | name")
|
|
31
22
|
.option("--model <re>", "only rows whose name matches")
|
|
32
23
|
.option("--only <scope>", "both | all")
|
|
33
|
-
.option(
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
)
|
|
37
|
-
.option(
|
|
38
|
-
"--medium",
|
|
39
|
-
"allowance and the rate views per side + win + cost + val",
|
|
40
|
-
)
|
|
41
|
-
.option(
|
|
42
|
-
"--no-fit",
|
|
43
|
-
"keep the preset's full width, do not trim to the terminal",
|
|
44
|
-
)
|
|
24
|
+
.option("--minimal", "fewest columns: model + req/mo each side + win + val")
|
|
25
|
+
.option("--medium", "allowance and the rate views per side + win + cost + val")
|
|
26
|
+
.option("--no-fit", "keep the preset's full width, do not trim to the terminal")
|
|
45
27
|
.option("--detail", "every column, untrimmed")
|
|
46
28
|
.option("--width <n>", "force table width")
|
|
47
|
-
.option(
|
|
48
|
-
"--columns <ids>",
|
|
49
|
-
"comma-separated columns (--columns help to list)",
|
|
50
|
-
)
|
|
29
|
+
.option("--columns <ids>", "comma-separated columns (--columns help to list)")
|
|
51
30
|
.option("--preset <name>", "named column preset from config")
|
|
52
31
|
.option(
|
|
53
32
|
"--bench <src>",
|
|
@@ -57,10 +36,7 @@ export function parseFlags(argv: string[]): Bag {
|
|
|
57
36
|
.option("--tps-weight <n>", "speed share of VAL")
|
|
58
37
|
.option("--val-weights <w>", "ability,tps,volume,cache,output shares")
|
|
59
38
|
.option("--scale <mode>", "log | linear for skewed terms")
|
|
60
|
-
.option(
|
|
61
|
-
"--inherit-suffixes <s>",
|
|
62
|
-
"comma-separated speed-variant suffixes",
|
|
63
|
-
)
|
|
39
|
+
.option("--inherit-suffixes <s>", "comma-separated speed-variant suffixes")
|
|
64
40
|
.option("--window <pair>", "override five-hour,weekly window ratios")
|
|
65
41
|
.option("--cost-thresholds <p>", "green,yellow COST cut-offs")
|
|
66
42
|
.option("--val-thresholds <p>", "yellow,green VAL cut-offs")
|
|
@@ -74,34 +50,19 @@ export function parseFlags(argv: string[]): Bag {
|
|
|
74
50
|
.option("--format <name>", "table | json | csv | md")
|
|
75
51
|
.option("--color <mode>", "auto | always | never")
|
|
76
52
|
.option("--check", "validate live sources and exit")
|
|
77
|
-
.option(
|
|
78
|
-
"--config <path>",
|
|
79
|
-
"config file (default: ~/.config/mpc/config.json)",
|
|
80
|
-
)
|
|
53
|
+
.option("--config <path>", "config file (default: ~/.config/mpc/config.json)")
|
|
81
54
|
.option("--no-config", "ignore the config file")
|
|
82
55
|
.option("--plugin <paths>", "extra config plugins, comma-separated")
|
|
83
56
|
.option("--print-config", "print the effective settings and exit")
|
|
84
|
-
.option(
|
|
85
|
-
"--usage",
|
|
86
|
-
"project your real usage from cmduse onto both plans",
|
|
87
|
-
)
|
|
57
|
+
.option("--usage", "project your real usage from cmduse onto both plans")
|
|
88
58
|
.option("--usage-file <path>", "read usage from a JSON file instead")
|
|
89
|
-
.option(
|
|
90
|
-
"--usage-window <w>",
|
|
91
|
-
"usage range: period | all | <n>d (default period)",
|
|
92
|
-
)
|
|
59
|
+
.option("--usage-window <w>", "usage range: period | all | <n>d (default period)")
|
|
93
60
|
.option(
|
|
94
61
|
"--usage-db <path>",
|
|
95
62
|
"opencode message store (default ~/.local/share/opencode/opencode.db)",
|
|
96
63
|
)
|
|
97
|
-
.option(
|
|
98
|
-
|
|
99
|
-
"provider usage log (default ~/.cache/mpc/usage.jsonl)",
|
|
100
|
-
)
|
|
101
|
-
.option(
|
|
102
|
-
"--usage-months <n>",
|
|
103
|
-
"treat the logged usage as covering N months",
|
|
104
|
-
);
|
|
64
|
+
.option("--usage-log <path>", "provider usage log (default ~/.cache/mpc/usage.jsonl)")
|
|
65
|
+
.option("--usage-months <n>", "treat the logged usage as covering N months");
|
|
105
66
|
cli.help();
|
|
106
67
|
cli.version(pkg.version);
|
|
107
68
|
const parsed = cli.parse(["node", "mpc", ...argv], { run: false });
|
package/src/cli/parse/fields.ts
CHANGED
|
@@ -8,9 +8,7 @@ export function numbers(bag: Bag, key: string): number[] | undefined {
|
|
|
8
8
|
const parts = Array.isArray(value) ? value : String(value).split(",");
|
|
9
9
|
const nums = parts.map((part) => Number(part));
|
|
10
10
|
if (nums.some((n) => !Number.isFinite(n))) {
|
|
11
|
-
throw new Error(
|
|
12
|
-
`--${key} expects comma-separated numbers, got "${value}"`,
|
|
13
|
-
);
|
|
11
|
+
throw new Error(`--${key} expects comma-separated numbers, got "${value}"`);
|
|
14
12
|
}
|
|
15
13
|
return nums;
|
|
16
14
|
}
|
|
@@ -28,9 +26,7 @@ export function presets(bag: Bag): Record<string, string[]> {
|
|
|
28
26
|
const value = bag.presets;
|
|
29
27
|
if (!value || typeof value !== "object" || Array.isArray(value)) return {};
|
|
30
28
|
const out: Record<string, string[]> = {};
|
|
31
|
-
for (const [name, ids] of Object.entries(
|
|
32
|
-
value as Record<string, unknown>,
|
|
33
|
-
)) {
|
|
29
|
+
for (const [name, ids] of Object.entries(value as Record<string, unknown>)) {
|
|
34
30
|
out[name] = Array.isArray(ids)
|
|
35
31
|
? ids.map((id) => String(id).trim())
|
|
36
32
|
: String(ids)
|
package/src/cli/parse/map.ts
CHANGED
|
@@ -9,22 +9,13 @@ import {
|
|
|
9
9
|
} from "#~/cli/options.ts";
|
|
10
10
|
import { ARG_DEFAULTS, BENCH_DEFAULT } from "#~/constants/cli.ts";
|
|
11
11
|
import { SHAPE_MEASURED } from "#~/constants/shape.ts";
|
|
12
|
-
import {
|
|
13
|
-
colorMode,
|
|
14
|
-
numbers,
|
|
15
|
-
pair,
|
|
16
|
-
presets,
|
|
17
|
-
usageWindow,
|
|
18
|
-
words,
|
|
19
|
-
} from "./fields.ts";
|
|
12
|
+
import { colorMode, numbers, pair, presets, usageWindow, words } from "./fields.ts";
|
|
20
13
|
import { assertKnown, assertValues, type Bag, int, share } from "./validate.ts";
|
|
21
14
|
|
|
22
15
|
export type { Bag } from "./validate.ts";
|
|
23
16
|
|
|
24
17
|
const pick = (bag: Bag, key: string): unknown =>
|
|
25
|
-
bag[key] === undefined
|
|
26
|
-
? ARG_DEFAULTS[key as keyof typeof ARG_DEFAULTS]
|
|
27
|
-
: bag[key];
|
|
18
|
+
bag[key] === undefined ? ARG_DEFAULTS[key as keyof typeof ARG_DEFAULTS] : bag[key];
|
|
28
19
|
|
|
29
20
|
/** `--bench` default is contextual: the AA API leads only when a key exists. */
|
|
30
21
|
function benchDefault(bag: Bag): string {
|
|
@@ -83,33 +74,25 @@ export function toOptions(bag: Bag): Options {
|
|
|
83
74
|
const suffixes = words(bag, "inheritSuffixes");
|
|
84
75
|
return suffixes.length > 0 ? suffixes : undefined;
|
|
85
76
|
})(),
|
|
86
|
-
window:
|
|
87
|
-
window && window.length === 2
|
|
88
|
-
? [window[0] ?? 0, window[1] ?? 0]
|
|
89
|
-
: undefined,
|
|
77
|
+
window: window && window.length === 2 ? [window[0] ?? 0, window[1] ?? 0] : undefined,
|
|
90
78
|
costThresholds: pair(pick(bag, "costThresholds"), "cost-thresholds"),
|
|
91
79
|
valThresholds: pair(pick(bag, "valThresholds"), "val-thresholds"),
|
|
92
|
-
benchName:
|
|
93
|
-
bag.benchName === undefined ? undefined : String(bag.benchName),
|
|
80
|
+
benchName: bag.benchName === undefined ? undefined : String(bag.benchName),
|
|
94
81
|
aaKey: bag.aaKey === undefined ? undefined : String(bag.aaKey),
|
|
95
82
|
noFallback: bag.fallback === false,
|
|
96
83
|
noAbility: bag.ability === false,
|
|
97
84
|
peak: bag.peak === true,
|
|
98
85
|
asc: bag.asc === true,
|
|
99
86
|
json: bag.json === true,
|
|
100
|
-
format: parseFormat(
|
|
101
|
-
bag.format === undefined ? "table" : String(bag.format),
|
|
102
|
-
) as Format,
|
|
87
|
+
format: parseFormat(bag.format === undefined ? "table" : String(bag.format)) as Format,
|
|
103
88
|
colorMode: colorMode(bag.color),
|
|
104
89
|
check: bag.check === true,
|
|
105
90
|
printConfig: bag.printConfig === true,
|
|
106
91
|
usage: bag.usage === true,
|
|
107
|
-
usageFile:
|
|
108
|
-
bag.usageFile === undefined ? undefined : String(bag.usageFile),
|
|
92
|
+
usageFile: bag.usageFile === undefined ? undefined : String(bag.usageFile),
|
|
109
93
|
usageLog: bag.usageLog === undefined ? undefined : String(bag.usageLog),
|
|
110
94
|
usageDb: bag.usageDb === undefined ? undefined : String(bag.usageDb),
|
|
111
|
-
usageMonths:
|
|
112
|
-
bag.usageMonths === undefined ? 1 : Number(bag.usageMonths),
|
|
95
|
+
usageMonths: bag.usageMonths === undefined ? 1 : Number(bag.usageMonths),
|
|
113
96
|
usageWindow: usageWindow(bag.usageWindow),
|
|
114
97
|
plugins: words(bag, "plugins"),
|
|
115
98
|
};
|
package/src/cli/parse/plugins.ts
CHANGED
|
@@ -33,19 +33,14 @@ async function loadOne(spec: string, relativeTo: string): Promise<Bag> {
|
|
|
33
33
|
configDir: relativeTo,
|
|
34
34
|
};
|
|
35
35
|
|
|
36
|
-
if (
|
|
37
|
-
target.endsWith(".json") ||
|
|
38
|
-
(existsSync(target) && target.endsWith(".json"))
|
|
39
|
-
) {
|
|
36
|
+
if (target.endsWith(".json") || (existsSync(target) && target.endsWith(".json"))) {
|
|
40
37
|
return asBag(JSON.parse(readFileSync(target, "utf8")), spec);
|
|
41
38
|
}
|
|
42
39
|
try {
|
|
43
40
|
const mod = (await import(target)) as { default?: unknown };
|
|
44
41
|
const exported = mod.default ?? mod;
|
|
45
42
|
const value =
|
|
46
|
-
typeof exported === "function"
|
|
47
|
-
? await (exported as PluginFactory)(ctx)
|
|
48
|
-
: exported;
|
|
43
|
+
typeof exported === "function" ? await (exported as PluginFactory)(ctx) : exported;
|
|
49
44
|
return asBag(value, spec);
|
|
50
45
|
} catch (error) {
|
|
51
46
|
throw new Error(
|
|
@@ -55,10 +50,7 @@ async function loadOne(spec: string, relativeTo: string): Promise<Bag> {
|
|
|
55
50
|
}
|
|
56
51
|
|
|
57
52
|
/** Load plugins in order and layer them: later plugins win over earlier ones. */
|
|
58
|
-
export async function loadPlugins(
|
|
59
|
-
specs: string[],
|
|
60
|
-
relativeTo: string,
|
|
61
|
-
): Promise<Bag> {
|
|
53
|
+
export async function loadPlugins(specs: string[], relativeTo: string): Promise<Bag> {
|
|
62
54
|
const merged: Bag = {};
|
|
63
55
|
for (const spec of specs) {
|
|
64
56
|
Object.assign(merged, await loadOne(spec, relativeTo));
|
|
@@ -32,13 +32,9 @@ export function share(name: string, value: unknown): number {
|
|
|
32
32
|
|
|
33
33
|
/** Reject undeclared flags before mapping. */
|
|
34
34
|
export function assertKnown(bag: Bag): void {
|
|
35
|
-
const extra = Object.keys(bag).filter(
|
|
36
|
-
(key) => !KNOWN.has(key) && !INTERNAL.has(key),
|
|
37
|
-
);
|
|
35
|
+
const extra = Object.keys(bag).filter((key) => !KNOWN.has(key) && !INTERNAL.has(key));
|
|
38
36
|
if (extra.length > 0) {
|
|
39
|
-
throw new Error(
|
|
40
|
-
`unknown flag "--${extra[0]}" — see --help for the full list`,
|
|
41
|
-
);
|
|
37
|
+
throw new Error(`unknown flag "--${extra[0]}" — see --help for the full list`);
|
|
42
38
|
}
|
|
43
39
|
}
|
|
44
40
|
|
|
@@ -49,12 +45,8 @@ function flag(key: string): string {
|
|
|
49
45
|
|
|
50
46
|
/** Reject a value-taking flag used bare (`--shape` with no argument). */
|
|
51
47
|
export function assertValues(bag: Bag): void {
|
|
52
|
-
const bare = Object.keys(bag).filter(
|
|
53
|
-
(key) => VALUED.has(key) && bag[key] === true,
|
|
54
|
-
);
|
|
48
|
+
const bare = Object.keys(bag).filter((key) => VALUED.has(key) && bag[key] === true);
|
|
55
49
|
if (bare.length > 0) {
|
|
56
|
-
throw new Error(
|
|
57
|
-
`${flag(String(bare[0]))} expects a value — see --help`,
|
|
58
|
-
);
|
|
50
|
+
throw new Error(`${flag(String(bare[0]))} expects a value — see --help`);
|
|
59
51
|
}
|
|
60
52
|
}
|
package/src/cli/run.ts
CHANGED
|
@@ -30,12 +30,7 @@ export async function run(argv: string[]): Promise<number> {
|
|
|
30
30
|
// cac prints the help text / version line itself and returns the parsed bag
|
|
31
31
|
// (with the `h`/`v` aliases) instead of exiting, so stop here — otherwise the
|
|
32
32
|
// full table would print after the help, and the aliases would trip assertKnown.
|
|
33
|
-
if (
|
|
34
|
-
bag.help === true ||
|
|
35
|
-
bag.h === true ||
|
|
36
|
-
bag.version === true ||
|
|
37
|
-
bag.v === true
|
|
38
|
-
) {
|
|
33
|
+
if (bag.help === true || bag.h === true || bag.version === true || bag.v === true) {
|
|
39
34
|
return 0;
|
|
40
35
|
}
|
|
41
36
|
if (bag.printConfig === true) {
|
|
@@ -63,16 +58,8 @@ export async function run(argv: string[]): Promise<number> {
|
|
|
63
58
|
);
|
|
64
59
|
}
|
|
65
60
|
|
|
66
|
-
const {
|
|
67
|
-
|
|
68
|
-
ccEntries,
|
|
69
|
-
ocPlanInfo,
|
|
70
|
-
ccPlanInfo,
|
|
71
|
-
rows,
|
|
72
|
-
ability,
|
|
73
|
-
workloads,
|
|
74
|
-
shapeNote,
|
|
75
|
-
} = await collect(options);
|
|
61
|
+
const { ocEntries, ccEntries, ocPlanInfo, ccPlanInfo, rows, ability, workloads, shapeNote } =
|
|
62
|
+
await collect(options);
|
|
76
63
|
if (options.check) {
|
|
77
64
|
return runCheck(options, ocEntries, ccEntries, rows, ability);
|
|
78
65
|
}
|
|
@@ -91,9 +78,7 @@ export async function run(argv: string[]): Promise<number> {
|
|
|
91
78
|
{ months: options.usageMonths },
|
|
92
79
|
);
|
|
93
80
|
if (options.format === "json" || options.json) {
|
|
94
|
-
console.log(
|
|
95
|
-
JSON.stringify({ source: usage.label, ...projection }, null, 2),
|
|
96
|
-
);
|
|
81
|
+
console.log(JSON.stringify({ source: usage.label, ...projection }, null, 2));
|
|
97
82
|
return 0;
|
|
98
83
|
}
|
|
99
84
|
renderUsage(projection, {
|
|
@@ -107,11 +92,7 @@ export async function run(argv: string[]): Promise<number> {
|
|
|
107
92
|
}
|
|
108
93
|
|
|
109
94
|
const result = sortRows(
|
|
110
|
-
rows.filter(
|
|
111
|
-
(r) =>
|
|
112
|
-
matches(r, options.model) &&
|
|
113
|
-
(options.only === "all" || (r.oc && r.cc)),
|
|
114
|
-
),
|
|
95
|
+
rows.filter((r) => matches(r, options.model) && (options.only === "all" || (r.oc && r.cc))),
|
|
115
96
|
options.metric,
|
|
116
97
|
options.asc,
|
|
117
98
|
);
|
|
@@ -119,9 +100,7 @@ export async function run(argv: string[]): Promise<number> {
|
|
|
119
100
|
ocPlan: ocPlanInfo,
|
|
120
101
|
ccPlan: ccPlanInfo,
|
|
121
102
|
workloads,
|
|
122
|
-
abilityLabel: options.noAbility
|
|
123
|
-
? undefined
|
|
124
|
-
: (options.benchName ?? ability.label),
|
|
103
|
+
abilityLabel: options.noAbility ? undefined : (options.benchName ?? ability.label),
|
|
125
104
|
abilityNote: options.noAbility ? undefined : ability.note,
|
|
126
105
|
shapeNote,
|
|
127
106
|
window: options.window,
|
package/src/constants/cli.ts
CHANGED
|
@@ -1,11 +1,7 @@
|
|
|
1
1
|
import type { Format, Metric } from "#~/cli/options.ts";
|
|
2
2
|
import { SHAPE_AUTO } from "#~/constants/shape.ts";
|
|
3
3
|
import { DEFAULT_OC_PLAN } from "#~/constants/sources.ts";
|
|
4
|
-
import {
|
|
5
|
-
DETAIL_COLUMNS,
|
|
6
|
-
MEDIUM_COLUMNS,
|
|
7
|
-
MINIMAL_COLUMNS,
|
|
8
|
-
} from "#~/constants/view.ts";
|
|
4
|
+
import { DETAIL_COLUMNS, MEDIUM_COLUMNS, MINIMAL_COLUMNS } from "#~/constants/view.ts";
|
|
9
5
|
|
|
10
6
|
/** Workload defaults asserted by `--in` / `--cache` / `--out` when unset. */
|
|
11
7
|
export const OPTION_DEFAULTS = {
|
package/src/constants/view.ts
CHANGED
|
@@ -95,21 +95,12 @@ export const LEGEND: [string, string][] = [
|
|
|
95
95
|
["5h wk mo", "requests the rolling 5-hour / weekly / monthly window buys"],
|
|
96
96
|
["$/1K", "your cost per 1,000 requests, at the plan's price"],
|
|
97
97
|
["req/$", "requests one dollar of subscription buys"],
|
|
98
|
-
[
|
|
99
|
-
|
|
100
|
-
"active CommandCode promotion: -98%, Free, 2x usage (expiry line is in --json)",
|
|
101
|
-
],
|
|
102
|
-
[
|
|
103
|
-
"WIN",
|
|
104
|
-
"cheaper side: OC / CC / tie · 'x only' = only that provider has it",
|
|
105
|
-
],
|
|
98
|
+
["DEAL", "active CommandCode promotion: -98%, Free, 2x usage (expiry line is in --json)"],
|
|
99
|
+
["WIN", "cheaper side: OC / CC / tie · 'x only' = only that provider has it"],
|
|
106
100
|
["ability", "benchmark score for the model (source above)"],
|
|
107
101
|
["tps", "output tokens per second (source above)"],
|
|
108
102
|
["COST", "0-100 volume index: requests/mo, lower is better (no ability)"],
|
|
109
|
-
[
|
|
110
|
-
"VAL",
|
|
111
|
-
"0-100 ability-aware value: ability + speed + volume + cache + output",
|
|
112
|
-
],
|
|
103
|
+
["VAL", "0-100 ability-aware value: ability + speed + volume + cache + output"],
|
|
113
104
|
];
|
|
114
105
|
|
|
115
106
|
export const CSV_HEADER = [
|
package/src/data/bench/cc.ts
CHANGED
|
@@ -1,21 +1,12 @@
|
|
|
1
1
|
import { PAGES, TPS_HEADER } from "#~/constants/sources.ts";
|
|
2
|
-
import {
|
|
3
|
-
extractNumericColumn,
|
|
4
|
-
fetchText,
|
|
5
|
-
parseTables,
|
|
6
|
-
} from "#~/data/scrape/index.ts";
|
|
2
|
+
import { extractNumericColumn, fetchText, parseTables } from "#~/data/scrape/index.ts";
|
|
7
3
|
import { type BenchData, emptyData } from "./types.ts";
|
|
8
4
|
|
|
9
5
|
export async function loadCc(): Promise<BenchData> {
|
|
10
|
-
const pages = await Promise.all(
|
|
11
|
-
PAGES.map(async (url) => parseTables(await fetchText(url))),
|
|
12
|
-
);
|
|
6
|
+
const pages = await Promise.all(PAGES.map(async (url) => parseTables(await fetchText(url))));
|
|
13
7
|
const data = emptyData();
|
|
14
8
|
for (const tables of pages) {
|
|
15
|
-
for (const [key, score] of extractNumericColumn(
|
|
16
|
-
tables,
|
|
17
|
-
/intelligence/i,
|
|
18
|
-
)) {
|
|
9
|
+
for (const [key, score] of extractNumericColumn(tables, /intelligence/i)) {
|
|
19
10
|
if (!data.intelligence.has(key)) data.intelligence.set(key, score);
|
|
20
11
|
}
|
|
21
12
|
for (const [key, tps] of extractNumericColumn(tables, TPS_HEADER)) {
|
package/src/data/bench/index.ts
CHANGED
|
@@ -3,18 +3,9 @@ import { loadCc } from "./cc.ts";
|
|
|
3
3
|
import { loadAaFallback, resolvePrimary } from "./resolve.ts";
|
|
4
4
|
import type { AbilityOptions, AbilityResult } from "./types.ts";
|
|
5
5
|
|
|
6
|
-
export type {
|
|
7
|
-
AbilityOptions,
|
|
8
|
-
AbilityResult,
|
|
9
|
-
AbilitySource,
|
|
10
|
-
BenchData,
|
|
11
|
-
} from "./types.ts";
|
|
6
|
+
export type { AbilityOptions, AbilityResult, AbilitySource, BenchData } from "./types.ts";
|
|
12
7
|
|
|
13
|
-
function merge(
|
|
14
|
-
into: Map<string, number>,
|
|
15
|
-
from: Map<string, number>,
|
|
16
|
-
keys?: Set<string>,
|
|
17
|
-
): number {
|
|
8
|
+
function merge(into: Map<string, number>, from: Map<string, number>, keys?: Set<string>): number {
|
|
18
9
|
let added = 0;
|
|
19
10
|
for (const [key, score] of from) {
|
|
20
11
|
if (keys && !keys.has(key)) continue;
|
|
@@ -37,9 +28,7 @@ function prune(map: Map<string, number>, keys?: Set<string>): void {
|
|
|
37
28
|
* model scored anywhere shows a value. Nothing is cached: a source that is down
|
|
38
29
|
* costs the columns it alone supplies, never a stale value.
|
|
39
30
|
*/
|
|
40
|
-
export async function loadAbility(
|
|
41
|
-
opts: AbilityOptions,
|
|
42
|
-
): Promise<AbilityResult> {
|
|
31
|
+
export async function loadAbility(opts: AbilityOptions): Promise<AbilityResult> {
|
|
43
32
|
const primary = await resolvePrimary(opts);
|
|
44
33
|
// CommandCode and Artificial Analysis publish the same intelligence index, so
|
|
45
34
|
// a CC primary normally needs no second round-trip. That premise only covered
|
|
@@ -69,15 +58,10 @@ export async function loadAbility(
|
|
|
69
58
|
if (primary.scheme !== "aa-web" && primary.scheme !== "aa") {
|
|
70
59
|
const fallback = await loadAaFallback(opts);
|
|
71
60
|
const added =
|
|
72
|
-
merge(
|
|
73
|
-
|
|
74
|
-
fallback.data.intelligence,
|
|
75
|
-
opts.keys,
|
|
76
|
-
) + merge(primary.data.tps, fallback.data.tps, opts.keys);
|
|
61
|
+
merge(primary.data.intelligence, fallback.data.intelligence, opts.keys) +
|
|
62
|
+
merge(primary.data.tps, fallback.data.tps, opts.keys);
|
|
77
63
|
if (added > 0) {
|
|
78
|
-
fills.push(
|
|
79
|
-
`${added} values from Artificial Analysis (${fallback.how})`,
|
|
80
|
-
);
|
|
64
|
+
fills.push(`${added} values from Artificial Analysis (${fallback.how})`);
|
|
81
65
|
}
|
|
82
66
|
}
|
|
83
67
|
|
|
@@ -20,9 +20,7 @@ function parseJsonScores(text: string): Map<string, number> {
|
|
|
20
20
|
add(record.model ?? record.name, record.score ?? record.value);
|
|
21
21
|
}
|
|
22
22
|
} else if (body && typeof body === "object") {
|
|
23
|
-
for (const [name, score] of Object.entries(
|
|
24
|
-
body as Record<string, unknown>,
|
|
25
|
-
)) {
|
|
23
|
+
for (const [name, score] of Object.entries(body as Record<string, unknown>)) {
|
|
26
24
|
add(name, score);
|
|
27
25
|
}
|
|
28
26
|
}
|
|
@@ -31,8 +29,7 @@ function parseJsonScores(text: string): Map<string, number> {
|
|
|
31
29
|
|
|
32
30
|
async function loadFile(path: string): Promise<Map<string, number>> {
|
|
33
31
|
const file = Bun.file(path);
|
|
34
|
-
if (!(await file.exists()))
|
|
35
|
-
throw new Error(`bench file not found: ${path}`);
|
|
32
|
+
if (!(await file.exists())) throw new Error(`bench file not found: ${path}`);
|
|
36
33
|
return parseJsonScores(await file.text());
|
|
37
34
|
}
|
|
38
35
|
|
|
@@ -42,10 +39,7 @@ async function loadUrl(url: string): Promise<Map<string, number>> {
|
|
|
42
39
|
|
|
43
40
|
export async function resolvePrimary(opts: AbilityOptions): Promise<Resolved> {
|
|
44
41
|
const [scheme, arg] = opts.source.includes(":")
|
|
45
|
-
? [
|
|
46
|
-
opts.source.split(":")[0] ?? "",
|
|
47
|
-
opts.source.slice(opts.source.indexOf(":") + 1),
|
|
48
|
-
]
|
|
42
|
+
? [opts.source.split(":")[0] ?? "", opts.source.slice(opts.source.indexOf(":") + 1)]
|
|
49
43
|
: [opts.source, undefined];
|
|
50
44
|
|
|
51
45
|
switch (scheme) {
|
|
@@ -64,9 +58,7 @@ export async function resolvePrimary(opts: AbilityOptions): Promise<Resolved> {
|
|
|
64
58
|
case "aa": {
|
|
65
59
|
const key = opts.key ?? process.env.AA_API_KEY;
|
|
66
60
|
if (!key) {
|
|
67
|
-
throw new Error(
|
|
68
|
-
"--bench aa needs a key: set AA_API_KEY or pass --aa-key",
|
|
69
|
-
);
|
|
61
|
+
throw new Error("--bench aa needs a key: set AA_API_KEY or pass --aa-key");
|
|
70
62
|
}
|
|
71
63
|
const data = await loadAaApi(key);
|
|
72
64
|
if (data.intelligence.size === 0) {
|
|
@@ -119,9 +111,4 @@ export async function loadAaFallback(
|
|
|
119
111
|
return { data: await loadAaWeb(), how: "AA web scrape" };
|
|
120
112
|
}
|
|
121
113
|
|
|
122
|
-
export type {
|
|
123
|
-
AbilityOptions,
|
|
124
|
-
AbilityResult,
|
|
125
|
-
AbilitySource,
|
|
126
|
-
BenchData,
|
|
127
|
-
} from "./types.ts";
|
|
114
|
+
export type { AbilityOptions, AbilityResult, AbilitySource, BenchData } from "./types.ts";
|
|
@@ -1,11 +1,5 @@
|
|
|
1
1
|
import { UNLIMITED } from "#~/constants/data.ts";
|
|
2
|
-
import {
|
|
3
|
-
cellText,
|
|
4
|
-
headerIndex,
|
|
5
|
-
nameCell,
|
|
6
|
-
parseMoney,
|
|
7
|
-
type Table,
|
|
8
|
-
} from "#~/data/scrape/tables.ts";
|
|
2
|
+
import { cellText, headerIndex, nameCell, parseMoney, type Table } from "#~/data/scrape/tables.ts";
|
|
9
3
|
import { displayName, normalizeKey } from "#~/keys.ts";
|
|
10
4
|
import type { CatalogEntry, ModelPricing, ProviderId } from "#~/types.ts";
|
|
11
5
|
import { dealIn } from "./deal.ts";
|
|
@@ -29,10 +23,8 @@ function pricingOf(
|
|
|
29
23
|
return {
|
|
30
24
|
input: parseMoney(cells[cols.in] ?? "") ?? 0,
|
|
31
25
|
output: parseMoney(cells[cols.out] ?? "") ?? 0,
|
|
32
|
-
cacheRead:
|
|
33
|
-
|
|
34
|
-
cacheWrite:
|
|
35
|
-
cols.write >= 0 ? parseMoney(cells[cols.write] ?? "") : null,
|
|
26
|
+
cacheRead: cols.cache >= 0 ? (parseMoney(cells[cols.cache] ?? "") ?? 0) : 0,
|
|
27
|
+
cacheWrite: cols.write >= 0 ? parseMoney(cells[cols.write] ?? "") : null,
|
|
36
28
|
};
|
|
37
29
|
}
|
|
38
30
|
|
|
@@ -41,17 +33,8 @@ function pricingOf(
|
|
|
41
33
|
* header carries Input + Output + an allowance column qualify, so the wide
|
|
42
34
|
* "models included" tables are ignored.
|
|
43
35
|
*/
|
|
44
|
-
export function extractCatalog(
|
|
45
|
-
|
|
46
|
-
options: CatalogOptions,
|
|
47
|
-
): CatalogEntry[] {
|
|
48
|
-
const {
|
|
49
|
-
provider,
|
|
50
|
-
plan,
|
|
51
|
-
creditHeader,
|
|
52
|
-
defaultAllowance,
|
|
53
|
-
peak = false,
|
|
54
|
-
} = options;
|
|
36
|
+
export function extractCatalog(tables: Table[], options: CatalogOptions): CatalogEntry[] {
|
|
37
|
+
const { provider, plan, creditHeader, defaultAllowance, peak = false } = options;
|
|
55
38
|
const best = new Map<string, { entry: CatalogEntry; score: number }>();
|
|
56
39
|
|
|
57
40
|
for (const table of tables) {
|
|
@@ -77,9 +60,7 @@ export function extractCatalog(
|
|
|
77
60
|
// Its allowance is meaningless (cost is 0), so record 0 — that keeps
|
|
78
61
|
// it out of the plan's summed credits and marks it free for --check.
|
|
79
62
|
const creditCell = cells[creditCol] ?? "";
|
|
80
|
-
const unlimited = creditHeader
|
|
81
|
-
? UNLIMITED.test(cellText(creditCell))
|
|
82
|
-
: false;
|
|
63
|
+
const unlimited = creditHeader ? UNLIMITED.test(cellText(creditCell)) : false;
|
|
83
64
|
const allowance = creditHeader
|
|
84
65
|
? (parseMoney(creditCell) ?? (unlimited ? 0 : null))
|
|
85
66
|
: (defaultAllowance ?? 0);
|
|
@@ -1,19 +1,11 @@
|
|
|
1
|
-
import {
|
|
2
|
-
cellText,
|
|
3
|
-
headerIndex,
|
|
4
|
-
nameCell,
|
|
5
|
-
type Table,
|
|
6
|
-
} from "#~/data/scrape/tables.ts";
|
|
1
|
+
import { cellText, headerIndex, nameCell, type Table } from "#~/data/scrape/tables.ts";
|
|
7
2
|
import { normalizeKey } from "#~/keys.ts";
|
|
8
3
|
|
|
9
4
|
/**
|
|
10
5
|
* Pull one numeric column out of any tables that carry it, keyed by model.
|
|
11
6
|
* Used for benchmark columns like CommandCode's "Intelligence".
|
|
12
7
|
*/
|
|
13
|
-
export function extractNumericColumn(
|
|
14
|
-
tables: Table[],
|
|
15
|
-
header: RegExp,
|
|
16
|
-
): Map<string, number> {
|
|
8
|
+
export function extractNumericColumn(tables: Table[], header: RegExp): Map<string, number> {
|
|
17
9
|
const scores = new Map<string, number>();
|
|
18
10
|
for (const table of tables) {
|
|
19
11
|
const headerCells = (table[0] ?? []).map(cellText);
|
package/src/data/scrape/index.ts
CHANGED
|
@@ -3,20 +3,13 @@ export { dealIn } from "./catalog/deal.ts";
|
|
|
3
3
|
export { extractNumericColumn } from "./catalog/numeric.ts";
|
|
4
4
|
export { parseRoleRows } from "./roleRows.ts";
|
|
5
5
|
export type { Table } from "./tables.ts";
|
|
6
|
-
export {
|
|
7
|
-
cellText,
|
|
8
|
-
headerIndex,
|
|
9
|
-
nameCell,
|
|
10
|
-
parseMoney,
|
|
11
|
-
parseTables,
|
|
12
|
-
} from "./tables.ts";
|
|
6
|
+
export { cellText, headerIndex, nameCell, parseMoney, parseTables } from "./tables.ts";
|
|
13
7
|
export { parseTabTables, type TabTable } from "./tabs.ts";
|
|
14
8
|
|
|
15
9
|
export async function fetchText(url: string): Promise<string> {
|
|
16
10
|
const res = await fetch(url, {
|
|
17
11
|
headers: { "user-agent": "mpc/0.1 (+model price compare)" },
|
|
18
12
|
});
|
|
19
|
-
if (!res.ok)
|
|
20
|
-
throw new Error(`GET ${url} -> ${res.status} ${res.statusText}`);
|
|
13
|
+
if (!res.ok) throw new Error(`GET ${url} -> ${res.status} ${res.statusText}`);
|
|
21
14
|
return res.text();
|
|
22
15
|
}
|
package/src/data/scrape/tabs.ts
CHANGED
|
@@ -34,9 +34,7 @@ export async function parseTabTables(html: string): Promise<TabTable[]> {
|
|
|
34
34
|
.on('[role="tab"]', {
|
|
35
35
|
element(el) {
|
|
36
36
|
const href = el.getAttribute("href");
|
|
37
|
-
panelId = href
|
|
38
|
-
? href.replace(/^#/, "")
|
|
39
|
-
: el.getAttribute("aria-controls");
|
|
37
|
+
panelId = href ? href.replace(/^#/, "") : el.getAttribute("aria-controls");
|
|
40
38
|
anchorText = "";
|
|
41
39
|
el.onEndTag(() => {
|
|
42
40
|
if (panelId) labels.set(panelId, anchorText.trim());
|
|
@@ -60,8 +58,7 @@ export async function parseTabTables(html: string): Promise<TabTable[]> {
|
|
|
60
58
|
table = [];
|
|
61
59
|
const start = table;
|
|
62
60
|
el.onEndTag(() => {
|
|
63
|
-
if (start)
|
|
64
|
-
tables.push({ label: label ?? "", table: start });
|
|
61
|
+
if (start) tables.push({ label: label ?? "", table: start });
|
|
65
62
|
table = null;
|
|
66
63
|
});
|
|
67
64
|
},
|