@jeffreyjyz/mpc 0.0.0-stage → 0.1.1

package/src/data/bench/resolve.ts

@@ -0,0 +1,107 @@
+import { loadAaApi } from "~/data/sources/aa/api.ts";
+import { loadCc } from "./cc.ts";
+import { loadAaWebCached, loadFile, loadUrl } from "./store.ts";
+import {
+	type AbilityOptions,
+	type BenchData,
+	emptyData,
+	type Resolved,
+} from "./types.ts";
+
+export async function resolvePrimary(opts: AbilityOptions): Promise<Resolved> {
+	const [scheme, arg] = opts.source.includes(":")
+		? [
+				opts.source.split(":")[0] ?? "",
+				opts.source.slice(opts.source.indexOf(":") + 1),
+			]
+		: [opts.source, undefined];
+
+	switch (scheme) {
+		case "cc":
+			return {
+				scheme: "cc",
+				data: await loadCc(),
+				label: "CommandCode Intelligence",
+			};
+		case "aa-web": {
+			const data = emptyData();
+			data.intelligence = await loadAaWebCached(Boolean(opts.refresh));
+			return {
+				scheme: "aa-web",
+				data,
+				label: "Artificial Analysis (web, partial)",
+			};
+		}
+		case "aa": {
+			const key = opts.key ?? process.env.AA_API_KEY;
+			if (!key) {
+				throw new Error(
+					"--bench aa needs a key: set AA_API_KEY or pass --aa-key",
+				);
+			}
+			const data = await loadAaApi(key);
+			if (data.intelligence.size === 0) {
+				throw new Error(
+					"Artificial Analysis returned no scores — check the API key and response shape",
+				);
+			}
+			return {
+				scheme: "aa",
+				data,
+				label: `Artificial Analysis (${data.intelligence.size} models, ${data.tps.size} speed)`,
+			};
+		}
+		case "file":
+			if (!arg) throw new Error("--bench file:<path> needs a path");
+			return {
+				scheme: "file",
+				data: { intelligence: await loadFile(arg), tps: new Map() },
+				label: `bench file ${arg}`,
+			};
+		case "url":
+			if (!arg) throw new Error("--bench url:<url> needs a url");
+			return {
+				scheme: "url",
+				data: { intelligence: await loadUrl(arg), tps: new Map() },
+				label: `bench ${arg}`,
+			};
+		default:
+			throw new Error(
+				`unknown --bench "${opts.source}" (cc | aa | aa-web | file:<path> | url:<url>)`,
+			);
+	}
+}
+
+/**
+ * Keyless Artificial Analysis page scrape (partial — only the models AA embeds).
+ */
+export async function loadAaWeb(opts: AbilityOptions): Promise<BenchData> {
+	const data = emptyData();
+	data.intelligence = await loadAaWebCached(Boolean(opts.refresh));
+	return data;
+}
+
+/**
+ * Artificial Analysis as a gap filler: the full API when a key is available,
+ * otherwise the keyless page scrape (partial).
+ */
+export async function loadAaFallback(
+	opts: AbilityOptions,
+): Promise<{ data: BenchData; how: string }> {
+	const key = opts.key ?? process.env.AA_API_KEY;
+	if (key) {
+		try {
+			return { data: await loadAaApi(key), how: "AA API" };
+		} catch {
+			// fall through to the scrape
+		}
+	}
+	return { data: await loadAaWeb(opts), how: "AA web scrape" };
+}
+
+export type {
+	AbilityOptions,
+	AbilityResult,
+	AbilitySource,
+	BenchData,
+} from "./types.ts";

package/src/data/bench/store.ts

@@ -0,0 +1,169 @@
+import { CACHE_TTL_MS } from "~/constants/sources.ts";
+import { fetchText } from "~/data/scrape/index.ts";
+import { normalizeKey } from "~/keys.ts";
+
+function parseJsonScores(text: string): Map<string, number> {
+	const body = JSON.parse(text) as unknown;
+	const scores = new Map<string, number>();
+	const add = (name: unknown, score: unknown): void => {
+		const key = typeof name === "string" ? normalizeKey(name) : "";
+		const value = Number(score);
+		if (key && Number.isFinite(value)) scores.set(key, value);
+	};
+	if (Array.isArray(body)) {
+		for (const row of body) {
+			const record = (row ?? {}) as Record<string, unknown>;
+			add(record.model ?? record.name, record.score ?? record.value);
+		}
+	} else if (body && typeof body === "object") {
+		for (const [name, score] of Object.entries(
+			body as Record<string, unknown>,
+		)) {
+			add(name, score);
+		}
+	}
+	return scores;
+}
+
+export async function loadFile(path: string): Promise<Map<string, number>> {
+	const file = Bun.file(path);
+	if (!(await file.exists()))
+		throw new Error(`bench file not found: ${path}`);
+	return parseJsonScores(await file.text());
+}
+
+export async function loadUrl(url: string): Promise<Map<string, number>> {
+	return parseJsonScores(await fetchText(url));
+}
+
+import { mkdir } from "node:fs/promises";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { loadAaWeb } from "~/data/sources/aa/web.ts";
+
+function cachePath(): string {
+	const base = process.env.XDG_CACHE_HOME ?? join(homedir(), ".cache");
+	return join(base, "mpc", "ability-aa-web.json");
+}
+
+async function readCache(
+	refresh: boolean,
+): Promise<Map<string, number> | null> {
+	if (refresh) return null;
+	try {
+		const file = Bun.file(cachePath());
+		if (!(await file.exists())) return null;
+		const raw = (await file.json()) as {
+			fetchedAt?: number;
+			scores?: Record<string, number>;
+		};
+		if (!raw.fetchedAt || Date.now() - raw.fetchedAt > CACHE_TTL_MS)
+			return null;
+		return new Map(Object.entries(raw.scores ?? {}));
+	} catch {
+		return null;
+	}
+}
+
+async function writeCache(scores: Map<string, number>): Promise<void> {
+	try {
+		const path = cachePath();
+		await mkdir(join(path, ".."), { recursive: true });
+		await Bun.write(
+			path,
+			JSON.stringify({
+				fetchedAt: Date.now(),
+				scores: Object.fromEntries(scores),
+			}),
+		);
+	} catch {
+		// Cache is best-effort.
+	}
+}
+
+/** Keyless AA page scrape, cached on disk. */
+export async function loadAaWebCached(
+	refresh: boolean,
+): Promise<Map<string, number>> {
+	const cached = await readCache(refresh);
+	if (cached) return cached;
+	const scores = await loadAaWeb();
+	await writeCache(scores);
+	return scores;
+}
+
+/**
+ * The last good benchmark result, whole. The per-source caches above only cover
+ * the AA page scrape; this one covers the result mpc actually returned, so a
+ * source that is down (or a page that silently dropped a column, as the
+ * CommandCode plan pages did with Tok/s) falls back to the scores we already
+ * have instead of blanking the ability/tps columns.
+ */
+export interface CachedAbility {
+	intelligence: Map<string, number>;
+	tps: Map<string, number>;
+}
+
+function abilityCachePath(): string {
+	const base = process.env.XDG_CACHE_HOME ?? join(homedir(), ".cache");
+	return join(base, "mpc", "ability.json");
+}
+
+export async function readAbilityCache(
+	refresh = false,
+): Promise<CachedAbility | null> {
+	if (refresh) return null;
+	try {
+		const file = Bun.file(abilityCachePath());
+		if (!(await file.exists())) return null;
+		const raw = (await file.json()) as {
+			fetchedAt?: number;
+			intelligence?: Record<string, number>;
+			tps?: Record<string, number>;
+		};
+		if (!raw.fetchedAt || Date.now() - raw.fetchedAt > CACHE_TTL_MS)
+			return null;
+		return {
+			intelligence: new Map(Object.entries(raw.intelligence ?? {})),
+			tps: new Map(Object.entries(raw.tps ?? {})),
+		};
+	} catch {
+		return null;
+	}
+}
+
+export async function writeAbilityCache(data: CachedAbility): Promise<void> {
+	try {
+		const path = abilityCachePath();
+		await mkdir(join(path, ".."), { recursive: true });
+		await Bun.write(
+			path,
+			JSON.stringify({
+				fetchedAt: Date.now(),
+				intelligence: Object.fromEntries(data.intelligence),
+				tps: Object.fromEntries(data.tps),
+			}),
+		);
+	} catch {
+		// Cache is best-effort.
+	}
+}
+
+/** Fill keys the live run is missing from `cached`. Returns how many were added. */
+export function fillFromCache(
+	data: CachedAbility,
+	cached: CachedAbility,
+): number {
+	let added = 0;
+	for (const [into, from] of [
+		[data.intelligence, cached.intelligence],
+		[data.tps, cached.tps],
+	] as const) {
+		for (const [key, value] of from) {
+			if (into.has(key)) continue;
+			into.set(key, value);
+			added++;
+		}
+	}
+	return added;
+}

package/src/data/bench/types.ts

@@ -0,0 +1,34 @@
+export type AbilitySource = "cc" | "aa" | "aa-web" | "file" | "url";
+
+/** Benchmark data, keyed by canonical model key. */
+export interface BenchData {
+	intelligence: Map<string, number>;
+	/** Output tokens per second, when the source publishes it. */
+	tps: Map<string, number>;
+}
+
+export interface AbilityResult extends BenchData {
+	label: string;
+	/** Note about coverage limits, shown in the footer. */
+	note?: string;
+}
+
+export interface AbilityOptions {
+	source: string;
+	key?: string;
+	/** Only these model keys matter; fills outside the catalog are ignored. */
+	keys?: Set<string>;
+	/** Fill models the primary source misses from the other sources. */
+	fallback?: boolean;
+	refresh?: boolean;
+}
+
+export interface Resolved {
+	scheme: AbilitySource;
+	data: BenchData;
+	label: string;
+}
+
+export function emptyData(): BenchData {
+	return { intelligence: new Map(), tps: new Map() };
+}

package/src/data/cmduse.ts

@@ -0,0 +1,43 @@
+export interface RunResult {
+	ok: boolean;
+	stdout: string;
+	stderr: string;
+}
+
+/** CommandCode CLI to shell out to; override to test a dev build. */
+export function cmduseBin(): string {
+	return process.env.CMDUSE_BIN || "cmduse";
+}
+
+/**
+ * Run a binary with args. Never throws: a missing or failing binary comes back
+ * as `ok: false` so callers can pick a fallback rather than crash the report.
+ * `extraEnv` is merged over the environment for this child (and inherited by
+ * anything it spawns).
+ */
+export async function runBinary(
+	cmd: string,
+	args: string[],
+	extraEnv?: Record<string, string>,
+): Promise<RunResult> {
+	let proc: Bun.Subprocess<"pipe", "pipe", "pipe">;
+	try {
+		proc = Bun.spawn([cmd, ...args], {
+			stdout: "pipe",
+			stderr: "pipe",
+			env: extraEnv ? { ...process.env, ...extraEnv } : undefined,
+		});
+	} catch (error) {
+		return { ok: false, stdout: "", stderr: String(error) };
+	}
+	const [stdout, stderr, code] = await Promise.all([
+		new Response(proc.stdout).text(),
+		new Response(proc.stderr).text(),
+		proc.exited,
+	]);
+	return { ok: code === 0, stdout, stderr };
+}
+
+export async function runCmduse(args: string[]): Promise<RunResult> {
+	return runBinary(cmduseBin(), args);
+}

package/src/data/scrape/catalog/catalog.ts

@@ -0,0 +1,99 @@
+import {
+	cellText,
+	headerIndex,
+	nameCell,
+	parseMoney,
+	type Table,
+} from "~/data/scrape/tables.ts";
+import { displayName, normalizeKey } from "~/keys.ts";
+import type { CatalogEntry, ModelPricing, ProviderId } from "~/types.ts";
+import { dealIn } from "./deal.ts";
+import { variantScore } from "./variant.ts";
+
+interface CatalogOptions {
+	provider: ProviderId;
+	plan: string;
+	/** Matches the header cell that carries the per-model allowance. */
+	creditHeader?: RegExp;
+	/** Allowance to assume for rate-only tables (no allowance column). */
+	defaultAllowance?: number;
+	/** OpenCode Go only: prefer peak-rate rows over off-peak. */
+	peak?: boolean;
+}
+
+function pricingOf(
+	cells: string[],
+	cols: { in: number; out: number; cache: number; write: number },
+): ModelPricing {
+	return {
+		input: parseMoney(cells[cols.in] ?? "") ?? 0,
+		output: parseMoney(cells[cols.out] ?? "") ?? 0,
+		cacheRead:
+			cols.cache >= 0 ? (parseMoney(cells[cols.cache] ?? "") ?? 0) : 0,
+		cacheWrite:
+			cols.write >= 0 ? parseMoney(cells[cols.write] ?? "") : null,
+	};
+}
+
+/**
+ * Extract a provider catalog from already-parsed tables. Only tables whose
+ * header carries Input + Output + an allowance column qualify, so the wide
+ * "models included" tables are ignored.
+ */
+export function extractCatalog(
+	tables: Table[],
+	options: CatalogOptions,
+): CatalogEntry[] {
+	const {
+		provider,
+		plan,
+		creditHeader,
+		defaultAllowance,
+		peak = false,
+	} = options;
+	const best = new Map<string, { entry: CatalogEntry; score: number }>();
+
+	for (const table of tables) {
+		const header = (table[0] ?? []).map(cellText);
+		if (header.length < 4) continue;
+		const cols = {
+			in: headerIndex(header, /^input/i),
+			out: headerIndex(header, /^output/i),
+			cache: headerIndex(header, /cache(d)?\s*read/i),
+			write: headerIndex(header, /cache\s*write/i),
+		};
+		const creditCol = creditHeader ? headerIndex(header, creditHeader) : -2;
+		if (cols.in < 0 || cols.out < 0) continue;
+		if (creditHeader && creditCol < 0) continue;
+		if (!creditHeader && defaultAllowance === undefined) continue;
+
+		for (const cells of table.slice(1)) {
+			const rawName = nameCell(cells[0] ?? "");
+			const deal = dealIn(cells[0] ?? "");
+			if (!rawName) continue;
+			const allowance = creditHeader
+				? parseMoney(cells[creditCol] ?? "")
+				: (defaultAllowance ?? 0);
+			if (allowance === null) continue;
+			const key = normalizeKey(rawName);
+			if (!key) continue;
+
+			const score = variantScore(rawName, peak);
+			const existing = best.get(key);
+			if (existing && existing.score <= score) continue;
+			best.set(key, {
+				score,
+				entry: {
+					provider,
+					plan,
+					key,
+					name: displayName(rawName),
+					pricing: pricingOf(cells, cols),
+					allowance,
+					...(deal ? { deal } : {}),
+				},
+			});
+		}
+	}
+	return [...best.values()].map((v) => v.entry);
+}

package/src/data/scrape/catalog/deal.ts

@@ -0,0 +1,20 @@
+import { BADGE, BOUNDARY, ENDS } from "~/constants/data.ts";
+import { cellText } from "~/data/scrape/tables.ts";
+import type { Deal } from "~/types.ts";
+
+/**
+ * Read the promotion from a raw model name cell. `nameCell` deliberately keeps
+ * only the first text node — the one that is the name — so the badges that
+ * follow it are dropped at the door unless they are read here.
+ *
+ * Cells look like `Grok 4.7` BOUNDARY `-40%` BOUNDARY `Ends September 27, 2026`
+ * or `Pixel Canary` BOUNDARY `Free`.
+ */
+export function dealIn(cell: string): Deal | undefined {
+	const parts = cell.split(BOUNDARY).map(cellText).filter(Boolean);
+	const rest = parts.slice(1);
+	const badge = rest.find((part) => BADGE.test(part));
+	if (!badge) return undefined;
+	const ends = rest.find((part) => ENDS.test(part));
+	return ends ? { badge, ends } : { badge };
+}

package/src/data/scrape/catalog/numeric.ts

@@ -0,0 +1,30 @@
+import {
+	cellText,
+	headerIndex,
+	nameCell,
+	type Table,
+} from "~/data/scrape/tables.ts";
+import { normalizeKey } from "~/keys.ts";
+
+/**
+ * Pull one numeric column out of any tables that carry it, keyed by model.
+ * Used for benchmark columns like CommandCode's "Intelligence".
+ */
+export function extractNumericColumn(
+	tables: Table[],
+	header: RegExp,
+): Map<string, number> {
+	const scores = new Map<string, number>();
+	for (const table of tables) {
+		const headerCells = (table[0] ?? []).map(cellText);
+		const col = headerIndex(headerCells, header);
+		if (col < 0) continue;
+		for (const cells of table.slice(1)) {
+			const key = normalizeKey(nameCell(cells[0] ?? ""));
+			if (!key || scores.has(key)) continue;
+			const match = (cells[col] ?? "").match(/-?[0-9]+(?:\.[0-9]+)?/);
+			if (match) scores.set(key, Number(match[0]));
+		}
+	}
+	return scores;
+}

package/src/data/scrape/catalog/variant.ts

@@ -0,0 +1,11 @@
+export function variantScore(name: string, peak: boolean): number {
+	const lower = name.toLowerCase();
+	let score = 0;
+	// Prefer the base context tier over the "> 200K" tier.
+	if (lower.includes(">")) score += 2;
+	const isOff = lower.includes("off-peak");
+	const isPeak = !isOff && lower.includes("peak");
+	if (peak) score += isPeak ? 0 : 2;
+	else score += isOff ? 0 : isPeak ? 2 : 1;
+	return score;
+}

package/src/data/scrape/index.ts

@@ -0,0 +1,21 @@
+export { extractCatalog } from "./catalog/catalog.ts";
+export { dealIn } from "./catalog/deal.ts";
+export { extractNumericColumn } from "./catalog/numeric.ts";
+export { parseRoleRows } from "./roleRows.ts";
+export type { Table } from "./tables.ts";
+export {
+	cellText,
+	headerIndex,
+	nameCell,
+	parseMoney,
+	parseTables,
+} from "./tables.ts";
+
+export async function fetchText(url: string): Promise<string> {
+	const res = await fetch(url, {
+		headers: { "user-agent": "mpc/0.1 (+model price compare)" },
+	});
+	if (!res.ok)
+		throw new Error(`GET ${url} -> ${res.status} ${res.statusText}`);
+	return res.text();
+}

package/src/data/scrape/roleRows.ts

@@ -0,0 +1,41 @@
+import { BOUNDARY } from "~/constants/data.ts";
+import type { Table } from "./tables.ts";
+
+/**
+ * Parse `role="row"` div grids (CommandCode's model catalog on some plan
+ * pages) into the same Table shape as parseTables.
+ */
+export async function parseRoleRows(html: string): Promise<Table[]> {
+	let grid: Table | null = null;
+	let row: string[] | null = null;
+	let cell: string | null = null;
+
+	const rewriter = new HTMLRewriter()
+		.on('div[role="row"]', {
+			element(el) {
+				grid ??= [];
+				const start = grid;
+				row = [];
+				el.onEndTag(() => {
+					if (row && start) start.push(row);
+					row = null;
+				});
+			},
+		})
+		.on('div[role="row"] > div', {
+			element(el) {
+				const current = row;
+				cell = "";
+				el.onEndTag(() => {
+					if (current) current.push((cell ?? "").trim());
+					cell = null;
+				});
+			},
+			text(chunk) {
+				if (cell !== null) cell += `${chunk.text}${BOUNDARY}`;
+			},
+		});
+
+	await rewriter.transform(new Response(html)).text();
+	return grid ? [grid] : [];
+}

package/src/data/scrape/tables.ts

@@ -0,0 +1,96 @@
+import { BOUNDARY } from "~/constants/data.ts";
+
+export type Table = string[][];
+
+function clean(text: string): string {
+	return text
+		.replace(/\u00a0/g, " ")
+		.replace(/\s+/g, " ")
+		.trim();
+}
+
+/** Collapse a raw cell (boundary markers included) into plain text. */
+export function cellText(cell: string): string {
+	return clean(cell.split(BOUNDARY).join(" "));
+}
+
+/** The first text node of a cell; deal badges live in later nodes. */
+export function nameCell(cell: string): string {
+	return cellText(cell.split(BOUNDARY)[0] ?? "");
+}
+
+/** Parse every <table> on a page into rows of trimmed cell text. */
+export async function parseTables(html: string): Promise<Table[]> {
+	const tables: Table[] = [];
+	let table: Table | null = null;
+	let row: string[] | null = null;
+	let cell: string | null = null;
+
+	const rewriter = new HTMLRewriter()
+		.on("table", {
+			element(el) {
+				table = [];
+				el.onEndTag(() => {
+					if (table) tables.push(table);
+					table = null;
+				});
+			},
+		})
+		.on("tr", {
+			element(el) {
+				const start = table;
+				row = [];
+				el.onEndTag(() => {
+					if (row && start) start.push(row);
+					row = null;
+				});
+			},
+		})
+		.on("th, td", {
+			element(el) {
+				const current = row;
+				cell = "";
+				el.onEndTag(() => {
+					// BOUNDARY marks a text-node boundary so "$60" + "4x" stays
+					// two tokens instead of collapsing into "$604x".
+					if (current) current.push((cell ?? "").trim());
+					cell = null;
+				});
+			},
+			text(chunk) {
+				if (cell !== null) cell += `${chunk.text}${BOUNDARY}`;
+			},
+		});
+
+	await rewriter.transform(new Response(html)).text();
+	return tables;
+}
+
+/**
+ * Pull USD out of a price/allowance cell. Cells carry BOUNDARY text-node
+ * boundaries; each segment is scanned and the last one carrying a price wins,
+ * which skips trailing multiplier notes like "4x". "Free" is 0, em-dash null.
+ */
+export function parseMoney(cell: string): number | null {
+	let result: number | null = null;
+	for (const segment of cell.split(BOUNDARY)) {
+		const s = segment.replace(/~~/g, "").trim();
+		if (/\bfree\b/i.test(s)) return 0;
+		const matches = [...s.matchAll(/\$\s*([0-9]+(?:\.[0-9]+)?)/g)];
+		if (matches.length > 0) {
+			result = Number(matches[matches.length - 1]?.[1]);
+		}
+	}
+	return result;
+}
+
+/** Match a table header cell, ignoring sort arrows and punctuation. */
+export function headerIndex(header: string[], pattern: RegExp): number {
+	return header.findIndex((h) => {
+		const norm = h
+			.replace(/[^a-z0-9 ]+/gi, " ")
+			.replace(/\s+/g, " ")
+			.trim();
+		return pattern.test(norm);
+	});
+}