@jeffreyjyz/mpc 0.0.0-stage → 0.1.0

package/src/cli/engine/project.ts

@@ -0,0 +1,145 @@
+import { PER_MILLION } from "~/constants/scoring.ts";
+import type { UsageEntry } from "~/data/usage/index.ts";
+import type { CatalogEntry, PlanInfo, ProviderId } from "~/types.ts";
+
+export interface SideProjection {
+	/** Cost of one real request at list rates. */
+	costPerRequest: number;
+	/** What that request costs you on the plan. */
+	payPerRequest: number;
+	/** Plan subscription cost this model's real usage earns, scaled to a month. */
+	monthly: number;
+	/** Plan credits the real usage draws. */
+	creditsDrawn: number;
+	allowance: number;
+	overCap: boolean;
+}
+
+export interface UsageRow {
+	key: string;
+	name: string;
+	requests: number;
+	tokens: {
+		input: number;
+		cacheRead: number;
+		output: number;
+		reasoning: number;
+	};
+	/** List-rate value of the tokens for the period, scaled to a month. */
+	listCost: number;
+	oc?: SideProjection;
+	cc?: SideProjection;
+}
+
+export interface ProjectOptions {
+	/** Treat the logged usage as covering this many months. */
+	months: number;
+}
+
+export interface UsageProjection {
+	rows: UsageRow[];
+	unmatched: string[];
+	ocMonthly: number;
+	ccMonthly: number;
+	ccListCost: number;
+}
+
+/**
+ * cmduse reports per-model totals, so this is the whole period's list cost.
+ *
+ * Same rule as the fixed-workload path (`cost.ts`): reasoning bills at the
+ * output rate *on top of* output, and an unpublished cache-write rate is billed
+ * at the input rate, never free. Diverging here made `--usage`'s `your $` read
+ * low (worst on reasoning-heavy models).
+ */
+function totalCost(entry: CatalogEntry, usage: UsageEntry): number {
+	const p = entry.pricing;
+	return (
+		(usage.tokensIn * p.input +
+			usage.cacheRead * p.cacheRead +
+			(usage.tokensOut + usage.reasoning) * p.output +
+			usage.cacheWrite * (p.cacheWrite ?? p.input)) /
+		PER_MILLION
+	);
+}
+
+function side(
+	entry: CatalogEntry,
+	usage: UsageEntry,
+	plan: PlanInfo,
+	periodCost: number,
+	months: number,
+): SideProjection {
+	const requests = Math.max(usage.requests, 1e-9);
+	const costPerRequest = periodCost / requests;
+	const payPerRequest =
+		entry.allowance > 0
+			? (plan.price * costPerRequest) / entry.allowance
+			: 0;
+	const creditsDrawn = periodCost / months;
+	return {
+		costPerRequest,
+		payPerRequest,
+		monthly: (usage.requests * payPerRequest) / months,
+		creditsDrawn,
+		allowance: entry.allowance,
+		overCap: creditsDrawn > entry.allowance,
+	};
+}
+
+/** Project the user's real usage onto both providers. */
+export function project(
+	usage: UsageEntry[],
+	catalogs: Record<ProviderId, CatalogEntry[]>,
+	plans: Record<ProviderId, PlanInfo>,
+	options: ProjectOptions,
+): UsageProjection {
+	const months = options.months > 0 ? options.months : 1;
+	const byKey = (list: CatalogEntry[]): Map<string, CatalogEntry> =>
+		new Map(list.map((entry) => [entry.key, entry]));
+	const oc = byKey(catalogs["oc-go"]);
+	const cc = byKey(catalogs.cc);
+
+	const rows: UsageRow[] = [];
+	const unmatched: string[] = [];
+	let ocMonthly = 0;
+	let ccMonthly = 0;
+	let ccListCost = 0;
+
+	for (const item of usage) {
+		const ocEntry = oc.get(item.key);
+		const ccEntry = cc.get(item.key);
+		if (!ocEntry && !ccEntry) {
+			unmatched.push(item.name);
+			continue;
+		}
+		const ccCost = ccEntry ? totalCost(ccEntry, item) : 0;
+		const ocCost = ocEntry ? totalCost(ocEntry, item) : 0;
+		const periodCost = ccEntry ? ccCost : ocCost;
+		const row: UsageRow = {
+			key: item.key,
+			name: ccEntry?.name ?? ocEntry?.name ?? item.name,
+			requests: item.requests,
+			tokens: {
+				input: item.tokensIn,
+				cacheRead: item.cacheRead,
+				output: item.tokensOut,
+				reasoning: item.reasoning,
+			},
+			listCost: periodCost / months,
+		};
+		if (ocEntry) {
+			row.oc = side(ocEntry, item, plans["oc-go"], ocCost, months);
+			ocMonthly += row.oc.monthly;
+		}
+		if (ccEntry) {
+			row.cc = side(ccEntry, item, plans.cc, ccCost, months);
+			ccMonthly += row.cc.monthly;
+			ccListCost += row.listCost;
+		}
+		rows.push(row);
+	}
+
+	rows.sort((a, b) => b.listCost - a.listCost);
+	return { rows, unmatched, ocMonthly, ccMonthly, ccListCost };
+}

package/src/cli/engine/rows.ts

@@ -0,0 +1,64 @@
+import { DEFAULT_SCORE } from "~/constants/scoring.ts";
+import type {
+	CatalogEntry,
+	CompareRow,
+	EntryMetrics,
+	PlanInfo,
+	ProviderId,
+	Workload,
+} from "~/types.ts";
+import { buildMetrics } from "./index.ts";
+import type { ScoreConfig } from "./score.ts";
+
+function indexByKey(
+	entries: CatalogEntry[],
+	metrics: EntryMetrics[],
+): Map<string, EntryMetrics> {
+	const byKey = new Map<string, EntryMetrics>();
+	entries.forEach((entry, i) => {
+		const metric = metrics[i];
+		if (metric) byKey.set(entry.key, metric);
+	});
+	return byKey;
+}
+
+export function buildRows(
+	ocEntries: CatalogEntry[],
+	ccEntries: CatalogEntry[],
+	ocPlan: PlanInfo,
+	ccPlan: PlanInfo,
+	workloads: Record<ProviderId, Workload>,
+	ability: Map<string, number> = new Map(),
+	tps: Map<string, number> = new Map(),
+	config: ScoreConfig = DEFAULT_SCORE,
+): CompareRow[] {
+	const plans = new Map<ProviderId, PlanInfo>([
+		["oc-go", ocPlan],
+		["cc", ccPlan],
+	]);
+	// Score every entry from both providers against one shared scale.
+	const all = buildMetrics(
+		[...ocEntries, ...ccEntries],
+		plans,
+		workloads,
+		ability,
+		tps,
+		config,
+	);
+	const ocByKey = indexByKey(ocEntries, all.slice(0, ocEntries.length));
+	const ccByKey = indexByKey(ccEntries, all.slice(ocEntries.length));
+
+	const keys = new Set([...ocByKey.keys(), ...ccByKey.keys()]);
+	const rows: CompareRow[] = [];
+	for (const key of keys) {
+		const ccEntry = ccEntries.find((e) => e.key === key);
+		const ocEntry = ocEntries.find((e) => e.key === key);
+		rows.push({
+			key,
+			name: ccEntry?.name ?? ocEntry?.name ?? key,
+			oc: ocByKey.get(key),
+			cc: ccByKey.get(key),
+		});
+	}
+	return rows;
+}

package/src/cli/engine/score.ts

@@ -0,0 +1,118 @@
+import type { ScaleMode } from "~/cli/options.ts";
+import type { EntryMetrics } from "~/types.ts";
+import { logMinmax, minmax } from "./cost.ts";
+
+/** Everything the scorer can be told to do differently. */
+export interface ScoreConfig {
+	abilityWeight: number;
+	tpsWeight: number;
+	/** ability, tps, volume, cache, output — normalised to sum 1. */
+	valWeights?: number[];
+	scale: ScaleMode;
+	inheritSuffixes?: string[];
+	window?: [number, number];
+}
+
+function normalise(
+	weights: number[] | undefined,
+	fallback: number[],
+): number[] {
+	const values =
+		weights && weights.length === fallback.length ? weights : fallback;
+	const total = values.reduce((a, b) => a + b, 0);
+	return total > 0 ? values.map((v) => v / total) : fallback;
+}
+
+const scale = (values: number[], mode: ScaleMode): number[] =>
+	mode === "linear" ? minmax(values) : logMinmax(values);
+
+/** COST index (inverted for display, so 0 = most requests = best): volume only,
+ * the log-scaled requests/month, min-max normalised across every entry. Cache
+ * and output prices are their own columns and are deliberately not folded in —
+ * "cost" here means "how many requests the plan buys". */
+export function assignIndex(metrics: EntryMetrics[]): void {
+	const priced = metrics.filter((m) => Number.isFinite(m.requestsPerMonth));
+	const volume = minmax(
+		priced.map((m) => Math.log10(Math.max(m.requestsPerMonth, 1))),
+	);
+
+	priced.forEach((m, i) => {
+		m.index = Math.round(100 * (volume[i] ?? 0.5));
+	});
+	for (const m of metrics) {
+		if (!Number.isFinite(m.requestsPerMonth)) m.index = 100;
+		else if (m.requestsPerMonth <= 0) m.index = 0;
+	}
+}
+
+/** VAL: ability / speed / volume / cache / output. Unscored models stay null. */
+export function assignValueIndex(
+	metrics: EntryMetrics[],
+	config: ScoreConfig,
+): void {
+	const weights = normalise(config.valWeights, [
+		config.abilityWeight,
+		config.tpsWeight,
+		(1 - config.abilityWeight - config.tpsWeight) * 0.5,
+		(1 - config.abilityWeight - config.tpsWeight) * 0.25,
+		(1 - config.abilityWeight - config.tpsWeight) * 0.25,
+	]);
+	const [wAbility, wTps, wVolume, wCache, wOutput] = weights;
+	const scored = metrics.filter((m) => m.ability !== null);
+	const priced = metrics.filter((m) => Number.isFinite(m.requestsPerMonth));
+	const speeded = metrics.filter((m) => m.tps !== null);
+
+	const ability = minmax(scored.map((m) => m.ability ?? 0));
+	const speed = scale(
+		speeded.map((m) => m.tps ?? 1),
+		config.scale,
+	);
+	const volume = minmax(
+		priced.map((m) => Math.log10(Math.max(m.requestsPerMonth, 1))),
+	);
+	const cache = scale(
+		priced.map((m) => m.pricing.cacheRead),
+		config.scale,
+	);
+	const output = scale(
+		priced.map((m) => m.pricing.output),
+		config.scale,
+	);
+
+	const of = <T>(items: T[], values: number[]): Map<T, number> =>
+		new Map(items.map((item, i) => [item, values[i] ?? 0.5]));
+	const byVolume = of(priced, volume);
+	const byCache = of(priced, cache);
+	const byOutput = of(priced, output);
+	const byAbility = of(scored, ability);
+	const bySpeed = of(speeded, speed);
+
+	// Unpriced models have unlimited requests, so they sit at the top of the
+	// scale and stay out of the range below.
+	const composites = new Map<EntryMetrics, number>();
+	for (const m of scored) {
+		if (!Number.isFinite(m.requestsPerMonth)) {
+			m.valueIndex = 100;
+			continue;
+		}
+		composites.set(
+			m,
+			100 * (wAbility ?? 0) * (byAbility.get(m) ?? 0.5) +
+				100 * (wTps ?? 0) * (bySpeed.get(m) ?? 0.5) +
+				100 * (wVolume ?? 0) * (byVolume.get(m) ?? 0.5) +
+				100 * (wCache ?? 0) * (1 - (byCache.get(m) ?? 0.5)) +
+				100 * (wOutput ?? 0) * (1 - (byOutput.get(m) ?? 0.5)),
+		);
+	}
+	// The weighted sum alone tops out in the sixties: no entry leads every term
+	// (the volume winner has no ability, the speed winner is mid-pack on volume),
+	// so the raw number reads as "everything is mediocre". Rescale across entries
+	// — the way COST does — so the best reads 100 and the spread is legible.
+	const values = [...composites.values()];
+	const low = values.length > 0 ? Math.min(...values) : 0;
+	const high = values.length > 0 ? Math.max(...values) : 0;
+	for (const [m, value] of composites) {
+		m.valueIndex =
+			high > low ? Math.round((100 * (value - low)) / (high - low)) : 100;
+	}
+}

package/src/cli/flow/check.ts

@@ -0,0 +1,66 @@
+import type { Options } from "~/cli/options.ts";
+import { loadOcGoModelIds } from "~/data/sources/opencode.ts";
+import { normalizeKey } from "~/keys.ts";
+import type { CatalogEntry, CompareRow } from "~/types.ts";
+
+export interface AbilityInfo {
+	label: string;
+	note?: string;
+	intelligence: Map<string, number>;
+	tps: Map<string, number>;
+}
+
+export async function runCheck(
+	_options: Options,
+	ocEntries: CatalogEntry[],
+	ccEntries: CatalogEntry[],
+	rows: CompareRow[],
+	ability: AbilityInfo,
+): Promise<number> {
+	const ocKeys = new Set(ocEntries.map((e) => e.key));
+	const ccKeys = new Set(ccEntries.map((e) => e.key));
+	const onlyOc = [...ocKeys].filter((k) => !ccKeys.has(k));
+	const onlyCc = [...ccKeys].filter((k) => !ocKeys.has(k));
+
+	console.log(`OpenCode entries parsed: ${ocEntries.length}`);
+	console.log(`CommandCode entries:     ${ccEntries.length}`);
+	console.log(
+		`matched models:          ${rows.filter((r) => r.oc && r.cc).length}`,
+	);
+	console.log(
+		`free OpenCode models:    ${ocEntries.filter((e) => e.allowance === 0).length}`,
+	);
+
+	const scored = rows.filter(
+		(r) => r.oc?.ability != null || r.cc?.ability != null,
+	).length;
+	console.log(
+		`ability source:          ${ability.label || "disabled"} (${ability.intelligence.size} scores, ${ability.tps.size} speed, ${scored}/${rows.length} rows scored)`,
+	);
+	if (ability.note) console.log(`ability note:            ${ability.note}`);
+
+	const unscored = rows
+		.filter((r) => (r.oc?.ability ?? r.cc?.ability ?? null) === null)
+		.map((r) => r.key);
+	if (unscored.length > 0) {
+		const head = unscored.slice(0, 20).join(", ");
+		const more =
+			unscored.length > 20 ? `  +${unscored.length - 20} more` : "";
+		console.log(`ability missing (${unscored.length}): ${head}${more}`);
+	}
+
+	const modelIds = await loadOcGoModelIds();
+	console.log(`OpenCode /zen/go/v1/models ids: ${modelIds.length}`);
+	const unpriced = modelIds.filter((id) => !ocKeys.has(normalizeKey(id)));
+	console.log(
+		`OpenCode ids missing from docs table (${unpriced.length}): ${unpriced.join(", ") || "—"}`,
+	);
+
+	console.log(
+		`\nonly in OpenCode (${onlyOc.length}): ${onlyOc.join(", ") || "—"}`,
+	);
+	console.log(
+		`only in CommandCode (${onlyCc.length}): ${onlyCc.join(", ") || "—"}`,
+	);
+	return 0;
+}

package/src/cli/flow/collect.ts

@@ -0,0 +1,76 @@
+import { buildRows } from "~/cli/engine/index.ts";
+import type { ScoreConfig } from "~/cli/engine/score.ts";
+import type { Options } from "~/cli/options.ts";
+import { loadAbility } from "~/data/bench/index.ts";
+import { loadShapes } from "~/data/shape.ts";
+import { loadCcCatalog } from "~/data/sources/cc/catalog.ts";
+import { loadCcPlan } from "~/data/sources/cc/plans.ts";
+import { loadOcGoCatalog, ocGoPlan } from "~/data/sources/opencode.ts";
+import type { ProviderId, Workload } from "~/types.ts";
+
+function scoreConfig(options: Options): ScoreConfig {
+	return {
+		abilityWeight: options.benchWeight,
+		tpsWeight: options.tpsWeight,
+		valWeights: options.valWeights,
+		scale: options.scale,
+		inheritSuffixes: options.inheritSuffixes,
+		window: options.window,
+	};
+}
+
+export async function collect(options: Options) {
+	const [ocEntries, ccEntries, ccPlanInfo, shape] = await Promise.all([
+		loadOcGoCatalog(options.peak),
+		loadCcCatalog(options.ccPlan),
+		loadCcPlan(options.ccPlan),
+		loadShapes(options.shape, options.since),
+	]);
+	// A measured shape prices both sides on the same request, so the comparison
+	// isolates price and allowance from traffic; otherwise both fall back to the
+	// single documented fixed workload.
+	const workload = shape.workload ?? options.workload;
+	const workloads: Record<ProviderId, Workload> = {
+		"oc-go": workload,
+		cc: workload,
+	};
+	// Ability is loaded after the catalogs so fills can be limited to our rows.
+	const keys = new Set(
+		[...ocEntries, ...ccEntries].map((entry) => entry.key),
+	);
+	const ability = options.noAbility
+		? {
+				intelligence: new Map<string, number>(),
+				tps: new Map<string, number>(),
+				label: "",
+				note: undefined as string | undefined,
+			}
+		: await loadAbility({
+				source: options.bench,
+				key: options.aaKey,
+				keys,
+				fallback: !options.noFallback,
+				refresh: options.refresh,
+			});
+	const ocPlanInfo = ocGoPlan(ocEntries);
+	const rows = buildRows(
+		ocEntries,
+		ccEntries,
+		ocPlanInfo,
+		ccPlanInfo,
+		workloads,
+		ability.intelligence,
+		ability.tps,
+		scoreConfig(options),
+	);
+	return {
+		ocEntries,
+		ccEntries,
+		ocPlanInfo,
+		ccPlanInfo,
+		rows,
+		ability,
+		workloads,
+		shapeNote: shape.note,
+	};
+}

package/src/cli/flow/columns.ts

@@ -0,0 +1,32 @@
+import type { Options } from "~/cli/options.ts";
+import { SETS, TIERS, type Tier } from "~/constants/cli.ts";
+
+/**
+ * The tier the flags asked for. The default is the full set — `--fit` used to
+ * be opt-in, but trimming to the terminal is now what plain `mpc` does.
+ */
+function tierOf(options: Options): Tier {
+	const picked = TIERS.filter((tier) => options[tier]);
+	if (picked.length > 1) {
+		throw new Error(
+			`--${picked.join(" and --")} conflict — pick one column preset`,
+		);
+	}
+	return picked[0] ?? "detail";
+}
+
+/** The columns to render: an exact `--columns` list, a config preset, or a tier. */
+export function resolveColumns(options: Options): string[] {
+	if (options.columns) return options.columns;
+	if (options.preset) {
+		const named = options.presets[options.preset];
+		if (!named) throw new Error(`unknown preset "${options.preset}"`);
+		return named;
+	}
+	return SETS[tierOf(options)];
+}
+
+/** Every tier trims to the terminal width except `--detail` and an exact list. */
+export function trimsToWidth(options: Options): boolean {
+	return !options.columns && !options.detail && options.fit;
+}

package/src/cli/flow/sort.ts

@@ -0,0 +1,68 @@
+import type { Metric, Options } from "~/cli/options.ts";
+import type { CompareRow } from "~/types.ts";
+
+export function matches(row: CompareRow, pattern: Options["model"]): boolean {
+	if (!pattern) return true;
+	try {
+		const re = new RegExp(pattern, "i");
+		return re.test(row.name) || re.test(row.key);
+	} catch {
+		return row.name.toLowerCase().includes(pattern.toLowerCase());
+	}
+}
+
+const maxValue = (r: CompareRow): number | null => {
+	const values = [r.oc?.valueIndex, r.cc?.valueIndex].filter(
+		(value): value is number => typeof value === "number",
+	);
+	return values.length === 0 ? null : Math.max(...values);
+};
+const maxIndex = (r: CompareRow): number =>
+	Math.max(r.oc?.index ?? -1, r.cc?.index ?? -1);
+const maxRequests = (r: CompareRow): number =>
+	Math.max(r.oc?.requestsPerMonth ?? -1, r.cc?.requestsPerMonth ?? -1);
+const minPay = (r: CompareRow): number => {
+	const values = [r.oc?.payPerRequest, r.cc?.payPerRequest].filter(
+		(value): value is number => value !== undefined,
+	);
+	return values.length === 0 ? Number.POSITIVE_INFINITY : Math.min(...values);
+};
+
+/** `asc` means "lower is better"; it flips each metric's default direction. */
+export function sortRows(
+	rows: CompareRow[],
+	metric: Metric,
+	asc: boolean,
+): CompareRow[] {
+	switch (metric) {
+		case "name": {
+			const sorted = [...rows].sort((a, b) =>
+				a.name.localeCompare(b.name),
+			);
+			return asc ? sorted : sorted.reverse();
+		}
+		case "val": {
+			const scored = rows
+				.filter((r) => maxValue(r) !== null)
+				.sort(
+					(a, b) =>
+						((maxValue(a) ?? 0) - (maxValue(b) ?? 0)) *
+						(asc ? 1 : -1),
+				);
+			const unscored = rows.filter((r) => maxValue(r) === null);
+			return [...scored, ...unscored];
+		}
+		case "cost":
+			return [...rows].sort(
+				(a, b) => (maxIndex(a) - maxIndex(b)) * (asc ? -1 : 1),
+			);
+		case "perreq":
+			return [...rows].sort(
+				(a, b) => (minPay(a) - minPay(b)) * (asc ? -1 : 1),
+			);
+		case "req":
+			return [...rows].sort(
+				(a, b) => (maxRequests(a) - maxRequests(b)) * (asc ? 1 : -1),
+			);
+	}
+}

package/src/cli/options.ts

@@ -0,0 +1,78 @@
+import { FORMATS, METRICS } from "~/constants/cli.ts";
+import type { Workload } from "~/types.ts";
+
+export type Metric = "val" | "cost" | "perreq" | "req" | "name";
+export type ColorMode = "auto" | "always" | "never";
+export type ScaleMode = "log" | "linear";
+export type Format = "table" | "json" | "csv" | "md";
+
+export interface Options {
+	ccPlan: string;
+	workload: Workload;
+	/** `off` (the default), `measured`, or a path to a saved reqshape JSON. */
+	shape: string;
+	/** Lower bound for the measured shape, passed through to reqshape. */
+	since?: string;
+	metric: Metric;
+	model?: string;
+	only: "both" | "all";
+	columns?: string[];
+	preset?: string;
+	presets: Record<string, string[]>;
+	detail: boolean;
+	minimal: boolean;
+	medium: boolean;
+	width?: number;
+	fit: boolean;
+	bench: string;
+	benchWeight: number;
+	tpsWeight: number;
+	valWeights?: number[];
+	scale: ScaleMode;
+	inheritSuffixes?: string[];
+	window?: [number, number];
+	costThresholds: [number, number];
+	valThresholds: [number, number];
+	benchName?: string;
+	aaKey?: string;
+	noFallback: boolean;
+	refresh: boolean;
+	noAbility: boolean;
+	peak: boolean;
+	asc: boolean;
+	json: boolean;
+	format: Format;
+	colorMode: ColorMode;
+	check: boolean;
+	printConfig: boolean;
+	usage: boolean;
+	usageFile?: string;
+	usageLog?: string;
+	usageDb?: string;
+	usageMonths: number;
+	usageWindow: "period" | "all" | `${number}d`;
+	plugins: string[];
+}
+
+export function parseMetric(value: string | undefined): Metric {
+	if (!METRICS.includes(value as Metric)) {
+		throw new Error(`unknown --metric "${value}"`);
+	}
+	return value as Metric;
+}
+
+export function parseFormat(value: string | undefined): Format {
+	if (!FORMATS.includes(value as Format)) {
+		throw new Error(
+			`unknown --format "${value}" (table | json | csv | md)`,
+		);
+	}
+	return value as Format;
+}
+
+export function parseOnly(value: string | undefined): "both" | "all" {
+	if (value !== "both" && value !== "all") {
+		throw new Error(`--only expects "both" or "all", got "${value}"`);
+	}
+	return value;
+}