@jeffreyjyz/mpc 0.1.3 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -3
- package/package.json +1 -1
- package/src/cli/flow/collect.ts +0 -1
- package/src/cli/options.ts +0 -1
- package/src/cli/parse/cac.ts +0 -1
- package/src/cli/parse/map.ts +0 -1
- package/src/constants/cli.ts +0 -1
- package/src/constants/sources.ts +1 -4
- package/src/data/bench/index.ts +12 -46
- package/src/data/bench/resolve.ts +42 -22
- package/src/data/bench/types.ts +0 -1
- package/src/data/scrape/tabs.ts +13 -9
- package/src/data/sources/aa/web.ts +96 -39
- package/src/data/bench/store.ts +0 -169
package/README.md
CHANGED
|
@@ -139,7 +139,6 @@ Relative paths resolve against the config file's directory. A plugin is a `.json
|
|
|
139
139
|
| `--bench-name <label>` | source | footer label for the ability source |
|
|
140
140
|
| `--aa-key <key>` | `AA_API_KEY` | Artificial Analysis API key |
|
|
141
141
|
| `--no-fallback` | off | skip filling ability misses from the other sources |
|
|
142
|
-
| `--refresh` | off | ignore the `aa-web` cache |
|
|
143
142
|
| `--no-ability` | off | hide `ability` and `VAL` |
|
|
144
143
|
| `--preset <name>` | — | named column set from `presets` in config |
|
|
145
144
|
| `--val-weights <a,b,c,d,e>` | see above | VAL weights (ability, tps, volume, cache, output) |
|
|
@@ -169,7 +168,7 @@ Relative paths resolve against the config file's directory. A plugin is a `.json
|
|
|
169
168
|
| --- | --- |
|
|
170
169
|
| `cmduse plans --json` | CommandCode plan price, credits, 5h/weekly windows |
|
|
171
170
|
| `commandcode.ai/docs/plans/{goat,pro,max}` | per-model credits + token rates |
|
|
172
|
-
| `opencode.ai/docs/go/` | oc-go per-model monthly limit (per Go plan) + shared token rates |
|
|
171
|
+
| `opencode.ai/v2/docs/console/go/` | oc-go per-model monthly limit (per Go plan) + shared token rates |
|
|
173
172
|
| `opencode.ai/zen/go/v1/models` | oc-go live model list (`--check` drift) |
|
|
174
173
|
|
|
175
174
|
## Reading the table
|
|
@@ -243,7 +242,7 @@ VAL = 100 * (0.35*ability + 0.10*tps + 0.25*volume + 0.15*cache + 0.15*output)
|
|
|
243
242
|
| `cc` | CommandCode's `Intelligence` column | every matched model |
|
|
244
243
|
| `file:<path>` / `url:<url>` | your JSON, `{ "model": score }` or `[{ model, score }]` | whatever you supply |
|
|
245
244
|
|
|
246
|
-
The default is `aa` when an AA key is available (`AA_API_KEY` or `--aa-key`), otherwise `cc`; an explicit `--bench` always wins. After the lead source, misses are filled from the same benchmark's keyless page scrape (`aa-web`) and then from CommandCode, the last resort — `--no-fallback` disables the fills. Because CC's `Intelligence` column and AA publish the same index, a `cc`-led run asks AA only when it still lacks throughput (CC's plan pages no longer publish `Tok/s`).
|
|
245
|
+
The default is `aa` when an AA key is available (`AA_API_KEY` or `--aa-key`), otherwise `cc`; an explicit `--bench` always wins. After the lead source, misses are filled from the same benchmark's keyless page scrape (`aa-web`) and then from CommandCode, the last resort — `--no-fallback` disables the fills. Because CC's `Intelligence` column and AA publish the same index, a `cc`-led run asks AA only when it still lacks throughput (CC's plan pages no longer publish `Tok/s`). Everything is fetched live, every run — **`mpc` writes no cache**: `aa-web` reads the index and speed straight out of the models page's embedded React Server Components payload, so a source that is down costs only the columns it alone supplies, never a stale value.
|
|
247
246
|
|
|
248
247
|
## Model matching
|
|
249
248
|
|
package/package.json
CHANGED
package/src/cli/flow/collect.ts
CHANGED
package/src/cli/options.ts
CHANGED
package/src/cli/parse/cac.ts
CHANGED
|
@@ -67,7 +67,6 @@ export function parseFlags(argv: string[]): Bag {
|
|
|
67
67
|
.option("--bench-name <label>", "footer label for the source")
|
|
68
68
|
.option("--aa-key <key>", "Artificial Analysis key (else AA_API_KEY)")
|
|
69
69
|
.option("--no-fallback", "do not fill misses from other sources")
|
|
70
|
-
.option("--refresh", "ignore the aa-web cache")
|
|
71
70
|
.option("--no-ability", "hide ability and VAL")
|
|
72
71
|
.option("--peak", "peak-rate rows (OpenCode DeepSeek)")
|
|
73
72
|
.option("--asc", "flip the default sort direction")
|
package/src/cli/parse/map.ts
CHANGED
|
@@ -93,7 +93,6 @@ export function toOptions(bag: Bag): Options {
|
|
|
93
93
|
bag.benchName === undefined ? undefined : String(bag.benchName),
|
|
94
94
|
aaKey: bag.aaKey === undefined ? undefined : String(bag.aaKey),
|
|
95
95
|
noFallback: bag.fallback === false,
|
|
96
|
-
refresh: bag.refresh === true,
|
|
97
96
|
noAbility: bag.ability === false,
|
|
98
97
|
peak: bag.peak === true,
|
|
99
98
|
asc: bag.asc === true,
|
package/src/constants/cli.ts
CHANGED
package/src/constants/sources.ts
CHANGED
|
@@ -76,7 +76,7 @@ export const OC_PLANS: Record<string, OcPlanDef> = {
|
|
|
76
76
|
/** The Go plan used when `--oc-plan` is not given. */
|
|
77
77
|
export const DEFAULT_OC_PLAN = "go";
|
|
78
78
|
|
|
79
|
-
export const DOC_URL = "https://opencode.ai/docs/go/";
|
|
79
|
+
export const DOC_URL = "https://opencode.ai/v2/docs/console/go/";
|
|
80
80
|
export const OC_MODELS_URL = "https://opencode.ai/zen/go/v1/models";
|
|
81
81
|
|
|
82
82
|
export const API_URL = "https://artificialanalysis.ai/api/v2/data/llms/models";
|
|
@@ -88,6 +88,3 @@ export const PAGES = [
|
|
|
88
88
|
"https://commandcode.ai/docs/plans/pro",
|
|
89
89
|
];
|
|
90
90
|
export const TPS_HEADER = /tok\s*\/?\s*s|tokens?\s*per\s*sec/i;
|
|
91
|
-
|
|
92
|
-
/** How long a cached benchmark result is considered fresh, in milliseconds. */
|
|
93
|
-
export const CACHE_TTL_MS = 7 * 24 * 60 * 60 * 1000;
|
package/src/data/bench/index.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
+
import { loadAaWeb } from "#~/data/sources/aa/web.ts";
|
|
1
2
|
import { loadCc } from "./cc.ts";
|
|
2
|
-
import { loadAaFallback,
|
|
3
|
-
import { fillFromCache, readAbilityCache, writeAbilityCache } from "./store.ts";
|
|
3
|
+
import { loadAaFallback, resolvePrimary } from "./resolve.ts";
|
|
4
4
|
import type { AbilityOptions, AbilityResult } from "./types.ts";
|
|
5
5
|
|
|
6
6
|
export type {
|
|
@@ -32,65 +32,31 @@ function prune(map: Map<string, number>, keys?: Set<string>): void {
|
|
|
32
32
|
}
|
|
33
33
|
|
|
34
34
|
/**
|
|
35
|
-
* Load benchmark scores. The chosen source leads; unless
|
|
36
|
-
* model it misses is filled from the other sources so a
|
|
37
|
-
* shows a value.
|
|
38
|
-
*
|
|
39
|
-
* blanking a column.
|
|
35
|
+
* Load benchmark scores, live, every run. The chosen source leads; unless
|
|
36
|
+
* --no-fallback, any model it misses is filled from the other sources so a
|
|
37
|
+
* model scored anywhere shows a value. Nothing is cached: a source that is down
|
|
38
|
+
* costs the columns it alone supplies, never a stale value.
|
|
40
39
|
*/
|
|
41
40
|
export async function loadAbility(
|
|
42
41
|
opts: AbilityOptions,
|
|
43
42
|
): Promise<AbilityResult> {
|
|
44
|
-
let live: AbilityResult;
|
|
45
|
-
try {
|
|
46
|
-
live = await loadLiveAbility(opts);
|
|
47
|
-
} catch (error) {
|
|
48
|
-
// Refresh must not strand the caller: an outage still has the cache.
|
|
49
|
-
const cached = await readAbilityCache(false);
|
|
50
|
-
if (!cached) throw error;
|
|
51
|
-
return {
|
|
52
|
-
...cached,
|
|
53
|
-
label: "cached (benchmark sources unavailable)",
|
|
54
|
-
note:
|
|
55
|
-
error instanceof Error
|
|
56
|
-
? error.message.slice(0, 120)
|
|
57
|
-
: undefined,
|
|
58
|
-
};
|
|
59
|
-
}
|
|
60
|
-
const cached = await readAbilityCache(Boolean(opts.refresh));
|
|
61
|
-
const filled = cached ? fillFromCache(live, cached) : 0;
|
|
62
|
-
await writeAbilityCache(live);
|
|
63
|
-
if (filled === 0) return live;
|
|
64
|
-
return {
|
|
65
|
-
...live,
|
|
66
|
-
note: [live.note, `${filled} values from the last cached run`]
|
|
67
|
-
.filter(Boolean)
|
|
68
|
-
.join(", "),
|
|
69
|
-
};
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
async function loadLiveAbility(opts: AbilityOptions): Promise<AbilityResult> {
|
|
73
43
|
const primary = await resolvePrimary(opts);
|
|
74
44
|
// CommandCode and Artificial Analysis publish the same intelligence index, so
|
|
75
45
|
// a CC primary normally needs no second round-trip. That premise only covered
|
|
76
46
|
// the index though: CC's plan pages dropped their `Tok/s` column (Sep 2026),
|
|
77
|
-
// so a CC primary with no speed values has to ask AA after all
|
|
78
|
-
// every model reports speed null and the consumers drop the row.
|
|
47
|
+
// so a CC primary with no speed values has to ask AA after all.
|
|
79
48
|
const needsSpeed = primary.data.tps.size === 0;
|
|
80
49
|
if (!opts.fallback || (primary.scheme === "cc" && !needsSpeed)) {
|
|
81
50
|
return { ...primary.data, label: primary.label };
|
|
82
51
|
}
|
|
83
52
|
|
|
84
53
|
const fills: string[] = [];
|
|
85
|
-
// The API leads, so give the same benchmark's keyless page scrape a second
|
|
86
|
-
// look (cached, free when warm) before the cross-source fallback.
|
|
54
|
+
// The API leads, so give the same benchmark's keyless page scrape a second look.
|
|
87
55
|
if (primary.scheme === "aa") {
|
|
88
|
-
const web = await loadAaWeb(
|
|
89
|
-
const added =
|
|
90
|
-
primary.data.intelligence,
|
|
91
|
-
web.
|
|
92
|
-
opts.keys,
|
|
93
|
-
);
|
|
56
|
+
const web = await loadAaWeb();
|
|
57
|
+
const added =
|
|
58
|
+
merge(primary.data.intelligence, web.intelligence, opts.keys) +
|
|
59
|
+
merge(primary.data.tps, web.tps, opts.keys);
|
|
94
60
|
if (added > 0) fills.push(`${added} values from AA web scrape`);
|
|
95
61
|
}
|
|
96
62
|
{
|
|
@@ -1,12 +1,44 @@
|
|
|
1
|
+
import { fetchText } from "#~/data/scrape/index.ts";
|
|
1
2
|
import { loadAaApi } from "#~/data/sources/aa/api.ts";
|
|
3
|
+
import { loadAaWeb } from "#~/data/sources/aa/web.ts";
|
|
4
|
+
import { normalizeKey } from "#~/keys.ts";
|
|
2
5
|
import { loadCc } from "./cc.ts";
|
|
3
|
-
import {
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
6
|
+
import type { AbilityOptions, BenchData, Resolved } from "./types.ts";
|
|
7
|
+
|
|
8
|
+
/** Parse `{ model: score }` or `[{ model|name, score|value }]` JSON. */
|
|
9
|
+
function parseJsonScores(text: string): Map<string, number> {
|
|
10
|
+
const body = JSON.parse(text) as unknown;
|
|
11
|
+
const scores = new Map<string, number>();
|
|
12
|
+
const add = (name: unknown, score: unknown): void => {
|
|
13
|
+
const key = typeof name === "string" ? normalizeKey(name) : "";
|
|
14
|
+
const value = Number(score);
|
|
15
|
+
if (key && Number.isFinite(value)) scores.set(key, value);
|
|
16
|
+
};
|
|
17
|
+
if (Array.isArray(body)) {
|
|
18
|
+
for (const row of body) {
|
|
19
|
+
const record = (row ?? {}) as Record<string, unknown>;
|
|
20
|
+
add(record.model ?? record.name, record.score ?? record.value);
|
|
21
|
+
}
|
|
22
|
+
} else if (body && typeof body === "object") {
|
|
23
|
+
for (const [name, score] of Object.entries(
|
|
24
|
+
body as Record<string, unknown>,
|
|
25
|
+
)) {
|
|
26
|
+
add(name, score);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
return scores;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
async function loadFile(path: string): Promise<Map<string, number>> {
|
|
33
|
+
const file = Bun.file(path);
|
|
34
|
+
if (!(await file.exists()))
|
|
35
|
+
throw new Error(`bench file not found: ${path}`);
|
|
36
|
+
return parseJsonScores(await file.text());
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
async function loadUrl(url: string): Promise<Map<string, number>> {
|
|
40
|
+
return parseJsonScores(await fetchText(url));
|
|
41
|
+
}
|
|
10
42
|
|
|
11
43
|
export async function resolvePrimary(opts: AbilityOptions): Promise<Resolved> {
|
|
12
44
|
const [scheme, arg] = opts.source.includes(":")
|
|
@@ -23,15 +55,12 @@ export async function resolvePrimary(opts: AbilityOptions): Promise<Resolved> {
|
|
|
23
55
|
data: await loadCc(),
|
|
24
56
|
label: "CommandCode Intelligence",
|
|
25
57
|
};
|
|
26
|
-
case "aa-web":
|
|
27
|
-
const data = emptyData();
|
|
28
|
-
data.intelligence = await loadAaWebCached(Boolean(opts.refresh));
|
|
58
|
+
case "aa-web":
|
|
29
59
|
return {
|
|
30
60
|
scheme: "aa-web",
|
|
31
|
-
data,
|
|
61
|
+
data: await loadAaWeb(),
|
|
32
62
|
label: "Artificial Analysis (web, partial)",
|
|
33
63
|
};
|
|
34
|
-
}
|
|
35
64
|
case "aa": {
|
|
36
65
|
const key = opts.key ?? process.env.AA_API_KEY;
|
|
37
66
|
if (!key) {
|
|
@@ -72,15 +101,6 @@ export async function resolvePrimary(opts: AbilityOptions): Promise<Resolved> {
|
|
|
72
101
|
}
|
|
73
102
|
}
|
|
74
103
|
|
|
75
|
-
/**
|
|
76
|
-
* Keyless Artificial Analysis page scrape (partial — only the models AA embeds).
|
|
77
|
-
*/
|
|
78
|
-
export async function loadAaWeb(opts: AbilityOptions): Promise<BenchData> {
|
|
79
|
-
const data = emptyData();
|
|
80
|
-
data.intelligence = await loadAaWebCached(Boolean(opts.refresh));
|
|
81
|
-
return data;
|
|
82
|
-
}
|
|
83
|
-
|
|
84
104
|
/**
|
|
85
105
|
* Artificial Analysis as a gap filler: the full API when a key is available,
|
|
86
106
|
* otherwise the keyless page scrape (partial).
|
|
@@ -96,7 +116,7 @@ export async function loadAaFallback(
|
|
|
96
116
|
// fall through to the scrape
|
|
97
117
|
}
|
|
98
118
|
}
|
|
99
|
-
return { data: await loadAaWeb(
|
|
119
|
+
return { data: await loadAaWeb(), how: "AA web scrape" };
|
|
100
120
|
}
|
|
101
121
|
|
|
102
122
|
export type {
|
package/src/data/bench/types.ts
CHANGED
package/src/data/scrape/tabs.ts
CHANGED
|
@@ -14,14 +14,16 @@ export interface TabTable {
|
|
|
14
14
|
* told apart — the Go and Go Plus catalogs share headers and token rates, and
|
|
15
15
|
* differ only in the monthly-limit values.
|
|
16
16
|
*
|
|
17
|
-
* Labels come from the
|
|
18
|
-
*
|
|
19
|
-
*
|
|
17
|
+
* Labels come from the `[role="tab"]` control, which the document places before
|
|
18
|
+
* the panels: its end-tag fills the map, then the matching panel's table
|
|
19
|
+
* inherits its label. The tab names its panel either through `href="#id"`
|
|
20
|
+
* (Starlight's `<a role="tab">`) or `aria-controls="id"` (the v2 console page's
|
|
21
|
+
* `<button role="tab">`), so read whichever is present.
|
|
20
22
|
*/
|
|
21
23
|
export async function parseTabTables(html: string): Promise<TabTable[]> {
|
|
22
24
|
const labels = new Map<string, string>();
|
|
23
25
|
const tables: TabTable[] = [];
|
|
24
|
-
let
|
|
26
|
+
let panelId: string | null = null;
|
|
25
27
|
let anchorText = "";
|
|
26
28
|
let label: string | null = null;
|
|
27
29
|
let table: Table | null = null;
|
|
@@ -29,14 +31,16 @@ export async function parseTabTables(html: string): Promise<TabTable[]> {
|
|
|
29
31
|
let cell: string | null = null;
|
|
30
32
|
|
|
31
33
|
const rewriter = new HTMLRewriter()
|
|
32
|
-
.on('
|
|
34
|
+
.on('[role="tab"]', {
|
|
33
35
|
element(el) {
|
|
34
|
-
|
|
36
|
+
const href = el.getAttribute("href");
|
|
37
|
+
panelId = href
|
|
38
|
+
? href.replace(/^#/, "")
|
|
39
|
+
: el.getAttribute("aria-controls");
|
|
35
40
|
anchorText = "";
|
|
36
41
|
el.onEndTag(() => {
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
anchorHref = null;
|
|
42
|
+
if (panelId) labels.set(panelId, anchorText.trim());
|
|
43
|
+
panelId = null;
|
|
40
44
|
});
|
|
41
45
|
},
|
|
42
46
|
text(chunk) {
|
|
@@ -1,53 +1,110 @@
|
|
|
1
1
|
import { AA_MODELS_URL } from "#~/constants/sources.ts";
|
|
2
|
+
import { type BenchData, emptyData } from "#~/data/bench/types.ts";
|
|
2
3
|
import { fetchText } from "#~/data/scrape/index.ts";
|
|
3
4
|
import { normalizeKey } from "#~/keys.ts";
|
|
4
5
|
|
|
5
|
-
/**
|
|
6
|
-
interface
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
6
|
+
/** The subset of an AA models-page record this parser reads. */
|
|
7
|
+
interface AaModel {
|
|
8
|
+
slug?: unknown;
|
|
9
|
+
name?: unknown;
|
|
10
|
+
shortName?: unknown;
|
|
11
|
+
intelligenceIndex?: unknown;
|
|
12
|
+
timescaleData?: { medianOutputSpeed?: unknown } | null;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Decode a Next.js page's streamed RSC payload. Each chunk is a JSON string
|
|
17
|
+
* literal inside `self.__next_f.push([1,"…"])`; joining the decoded strings
|
|
18
|
+
* yields the flight text the page renders from, with plain `"initialModels":`
|
|
19
|
+
* keys instead of the `\"`-escaped form seen in the raw HTML.
|
|
20
|
+
*/
|
|
21
|
+
function decodeFlight(html: string): string {
|
|
22
|
+
const chunks: string[] = [];
|
|
23
|
+
const re = /self\.__next_f\.push\(\[1,("(?:[^"\\]|\\.)*")\]\)/g;
|
|
24
|
+
for (const match of html.matchAll(re)) {
|
|
25
|
+
try {
|
|
26
|
+
chunks.push(JSON.parse(match[1] ?? '""') as string);
|
|
27
|
+
} catch {
|
|
28
|
+
// A chunk we cannot read as a JSON string is skipped.
|
|
29
|
+
}
|
|
26
30
|
}
|
|
27
|
-
return
|
|
31
|
+
return chunks.join("");
|
|
28
32
|
}
|
|
29
33
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
34
|
+
/** The JSON array value of `"key"` in `text`, bracket-matched string-aware. */
|
|
35
|
+
function readArray(text: string, key: string): unknown[] | null {
|
|
36
|
+
const at = text.indexOf(`"${key}":`);
|
|
37
|
+
if (at < 0) return null;
|
|
38
|
+
const open = text.indexOf("[", at);
|
|
39
|
+
if (open < 0) return null;
|
|
40
|
+
let depth = 0;
|
|
41
|
+
let inString = false;
|
|
42
|
+
for (let i = open; i < text.length; i++) {
|
|
43
|
+
const char = text[i];
|
|
44
|
+
if (inString) {
|
|
45
|
+
if (char === "\\") i++;
|
|
46
|
+
else if (char === '"') inString = false;
|
|
47
|
+
continue;
|
|
48
|
+
}
|
|
49
|
+
if (char === '"') inString = true;
|
|
50
|
+
else if (char === "[") depth++;
|
|
51
|
+
else if (char === "]") {
|
|
52
|
+
depth--;
|
|
53
|
+
if (depth === 0) {
|
|
54
|
+
try {
|
|
55
|
+
return JSON.parse(text.slice(open, i + 1)) as unknown[];
|
|
56
|
+
} catch {
|
|
57
|
+
return null;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
37
61
|
}
|
|
38
|
-
return
|
|
62
|
+
return null;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** A finite, positive number; `null` for anything else (incl. 0 = unmeasured). */
|
|
66
|
+
function positive(value: unknown): number | null {
|
|
67
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0
|
|
68
|
+
? value
|
|
69
|
+
: null;
|
|
39
70
|
}
|
|
40
71
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
72
|
+
function labelOf(model: AaModel): string {
|
|
73
|
+
if (typeof model.name === "string") return model.name;
|
|
74
|
+
return typeof model.shortName === "string" ? model.shortName : "";
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function keyOf(value: unknown): string {
|
|
78
|
+
return typeof value === "string" ? normalizeKey(value) : "";
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Pure parser over the AA models page. Keyless and live: the page embeds its
|
|
83
|
+
* `initialModels` array in the RSC flight payload, each record carrying the
|
|
84
|
+
* Intelligence Index and (mostly) a median output speed. Only that embedded
|
|
85
|
+
* top-N is covered, so this is a partial source.
|
|
44
86
|
*/
|
|
45
|
-
export
|
|
46
|
-
const
|
|
47
|
-
|
|
87
|
+
export function parseAaWeb(html: string): BenchData {
|
|
88
|
+
const data = emptyData();
|
|
89
|
+
const models = readArray(decodeFlight(html) || html, "initialModels");
|
|
90
|
+
if (!models) return data;
|
|
91
|
+
for (const raw of models) {
|
|
92
|
+
const model = (raw ?? {}) as AaModel;
|
|
93
|
+
// slug is the precise key; the display name is the fallback.
|
|
94
|
+
const keys = [keyOf(model.slug), keyOf(labelOf(model))].filter(Boolean);
|
|
95
|
+
const score = positive(model.intelligenceIndex);
|
|
96
|
+
const speed = positive(model.timescaleData?.medianOutputSpeed);
|
|
97
|
+
for (const key of keys) {
|
|
98
|
+
if (score !== null && !data.intelligence.has(key)) {
|
|
99
|
+
data.intelligence.set(key, score);
|
|
100
|
+
}
|
|
101
|
+
if (speed !== null && !data.tps.has(key)) data.tps.set(key, speed);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
return data;
|
|
48
105
|
}
|
|
49
106
|
|
|
50
|
-
/**
|
|
51
|
-
export function
|
|
52
|
-
return
|
|
107
|
+
/** Fetch and parse the AA models page. Live, per run — nothing cached. */
|
|
108
|
+
export async function loadAaWeb(): Promise<BenchData> {
|
|
109
|
+
return parseAaWeb(await fetchText(AA_MODELS_URL));
|
|
53
110
|
}
|
package/src/data/bench/store.ts
DELETED
|
@@ -1,169 +0,0 @@
|
|
|
1
|
-
import { CACHE_TTL_MS } from "#~/constants/sources.ts";
|
|
2
|
-
import { fetchText } from "#~/data/scrape/index.ts";
|
|
3
|
-
import { normalizeKey } from "#~/keys.ts";
|
|
4
|
-
|
|
5
|
-
function parseJsonScores(text: string): Map<string, number> {
|
|
6
|
-
const body = JSON.parse(text) as unknown;
|
|
7
|
-
const scores = new Map<string, number>();
|
|
8
|
-
const add = (name: unknown, score: unknown): void => {
|
|
9
|
-
const key = typeof name === "string" ? normalizeKey(name) : "";
|
|
10
|
-
const value = Number(score);
|
|
11
|
-
if (key && Number.isFinite(value)) scores.set(key, value);
|
|
12
|
-
};
|
|
13
|
-
if (Array.isArray(body)) {
|
|
14
|
-
for (const row of body) {
|
|
15
|
-
const record = (row ?? {}) as Record<string, unknown>;
|
|
16
|
-
add(record.model ?? record.name, record.score ?? record.value);
|
|
17
|
-
}
|
|
18
|
-
} else if (body && typeof body === "object") {
|
|
19
|
-
for (const [name, score] of Object.entries(
|
|
20
|
-
body as Record<string, unknown>,
|
|
21
|
-
)) {
|
|
22
|
-
add(name, score);
|
|
23
|
-
}
|
|
24
|
-
}
|
|
25
|
-
return scores;
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
export async function loadFile(path: string): Promise<Map<string, number>> {
|
|
29
|
-
const file = Bun.file(path);
|
|
30
|
-
if (!(await file.exists()))
|
|
31
|
-
throw new Error(`bench file not found: ${path}`);
|
|
32
|
-
return parseJsonScores(await file.text());
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
export async function loadUrl(url: string): Promise<Map<string, number>> {
|
|
36
|
-
return parseJsonScores(await fetchText(url));
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
import { mkdir } from "node:fs/promises";
|
|
40
|
-
import { homedir } from "node:os";
|
|
41
|
-
import { join } from "node:path";
|
|
42
|
-
import { loadAaWeb } from "#~/data/sources/aa/web.ts";
|
|
43
|
-
|
|
44
|
-
function cachePath(): string {
|
|
45
|
-
const base = process.env.XDG_CACHE_HOME ?? join(homedir(), ".cache");
|
|
46
|
-
return join(base, "mpc", "ability-aa-web.json");
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
async function readCache(
|
|
50
|
-
refresh: boolean,
|
|
51
|
-
): Promise<Map<string, number> | null> {
|
|
52
|
-
if (refresh) return null;
|
|
53
|
-
try {
|
|
54
|
-
const file = Bun.file(cachePath());
|
|
55
|
-
if (!(await file.exists())) return null;
|
|
56
|
-
const raw = (await file.json()) as {
|
|
57
|
-
fetchedAt?: number;
|
|
58
|
-
scores?: Record<string, number>;
|
|
59
|
-
};
|
|
60
|
-
if (!raw.fetchedAt || Date.now() - raw.fetchedAt > CACHE_TTL_MS)
|
|
61
|
-
return null;
|
|
62
|
-
return new Map(Object.entries(raw.scores ?? {}));
|
|
63
|
-
} catch {
|
|
64
|
-
return null;
|
|
65
|
-
}
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
async function writeCache(scores: Map<string, number>): Promise<void> {
|
|
69
|
-
try {
|
|
70
|
-
const path = cachePath();
|
|
71
|
-
await mkdir(join(path, ".."), { recursive: true });
|
|
72
|
-
await Bun.write(
|
|
73
|
-
path,
|
|
74
|
-
JSON.stringify({
|
|
75
|
-
fetchedAt: Date.now(),
|
|
76
|
-
scores: Object.fromEntries(scores),
|
|
77
|
-
}),
|
|
78
|
-
);
|
|
79
|
-
} catch {
|
|
80
|
-
// Cache is best-effort.
|
|
81
|
-
}
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
/** Keyless AA page scrape, cached on disk. */
|
|
85
|
-
export async function loadAaWebCached(
|
|
86
|
-
refresh: boolean,
|
|
87
|
-
): Promise<Map<string, number>> {
|
|
88
|
-
const cached = await readCache(refresh);
|
|
89
|
-
if (cached) return cached;
|
|
90
|
-
const scores = await loadAaWeb();
|
|
91
|
-
await writeCache(scores);
|
|
92
|
-
return scores;
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
/**
|
|
96
|
-
* The last good benchmark result, whole. The per-source caches above only cover
|
|
97
|
-
* the AA page scrape; this one covers the result mpc actually returned, so a
|
|
98
|
-
* source that is down (or a page that silently dropped a column, as the
|
|
99
|
-
* CommandCode plan pages did with Tok/s) falls back to the scores we already
|
|
100
|
-
* have instead of blanking the ability/tps columns.
|
|
101
|
-
*/
|
|
102
|
-
export interface CachedAbility {
|
|
103
|
-
intelligence: Map<string, number>;
|
|
104
|
-
tps: Map<string, number>;
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
function abilityCachePath(): string {
|
|
108
|
-
const base = process.env.XDG_CACHE_HOME ?? join(homedir(), ".cache");
|
|
109
|
-
return join(base, "mpc", "ability.json");
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
export async function readAbilityCache(
|
|
113
|
-
refresh = false,
|
|
114
|
-
): Promise<CachedAbility | null> {
|
|
115
|
-
if (refresh) return null;
|
|
116
|
-
try {
|
|
117
|
-
const file = Bun.file(abilityCachePath());
|
|
118
|
-
if (!(await file.exists())) return null;
|
|
119
|
-
const raw = (await file.json()) as {
|
|
120
|
-
fetchedAt?: number;
|
|
121
|
-
intelligence?: Record<string, number>;
|
|
122
|
-
tps?: Record<string, number>;
|
|
123
|
-
};
|
|
124
|
-
if (!raw.fetchedAt || Date.now() - raw.fetchedAt > CACHE_TTL_MS)
|
|
125
|
-
return null;
|
|
126
|
-
return {
|
|
127
|
-
intelligence: new Map(Object.entries(raw.intelligence ?? {})),
|
|
128
|
-
tps: new Map(Object.entries(raw.tps ?? {})),
|
|
129
|
-
};
|
|
130
|
-
} catch {
|
|
131
|
-
return null;
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
export async function writeAbilityCache(data: CachedAbility): Promise<void> {
|
|
136
|
-
try {
|
|
137
|
-
const path = abilityCachePath();
|
|
138
|
-
await mkdir(join(path, ".."), { recursive: true });
|
|
139
|
-
await Bun.write(
|
|
140
|
-
path,
|
|
141
|
-
JSON.stringify({
|
|
142
|
-
fetchedAt: Date.now(),
|
|
143
|
-
intelligence: Object.fromEntries(data.intelligence),
|
|
144
|
-
tps: Object.fromEntries(data.tps),
|
|
145
|
-
}),
|
|
146
|
-
);
|
|
147
|
-
} catch {
|
|
148
|
-
// Cache is best-effort.
|
|
149
|
-
}
|
|
150
|
-
}
|
|
151
|
-
|
|
152
|
-
/** Fill keys the live run is missing from `cached`. Returns how many were added. */
|
|
153
|
-
export function fillFromCache(
|
|
154
|
-
data: CachedAbility,
|
|
155
|
-
cached: CachedAbility,
|
|
156
|
-
): number {
|
|
157
|
-
let added = 0;
|
|
158
|
-
for (const [into, from] of [
|
|
159
|
-
[data.intelligence, cached.intelligence],
|
|
160
|
-
[data.tps, cached.tps],
|
|
161
|
-
] as const) {
|
|
162
|
-
for (const [key, value] of from) {
|
|
163
|
-
if (into.has(key)) continue;
|
|
164
|
-
into.set(key, value);
|
|
165
|
-
added++;
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
|
-
return added;
|
|
169
|
-
}
|