@xynogen/pix-data 0.2.2 → 0.2.4

package/README.md

@@ -1,42 +1,57 @@
 # pix-data
 
-Pi coding agent extension — shared model data layer. Fetches and caches the
-[modelgrep](https://modelgrep.com) model catalog to `~/.cache/pi/` on session
-start, so other extensions (model picker, footer, subagent resolver) can read
-context window, pricing, and a coding-focused score/rank synchronously without
-redundant network calls.
+Pi coding agent extension — shared model data layer. Warms two cached
+data sources on session start so other extensions (model picker, footer,
+subagent resolver) can read context window, pricing, and a coding-focused
+score/rank synchronously without redundant network calls:
 
-## Data source
+- **[modelgrep](https://modelgrep.com)** — the model catalog (context window,
+  pricing, modalities, capabilities, raw benchmark fields) used as the
+  authoritative source when present.
+- **[benchlm.ai](https://benchlm.ai)** — a leaderboard of 0–100 coding scores
+  used as a fallback when modelgrep's `artificial_analysis` block is null
+  (currently the common case for the long tail of models).
 
-All data comes from a **single source**: [modelgrep.com](https://modelgrep.com)
-(`/api/v1/models?benchmarked=1&sort=coding`). Free, no API key, ~190 benchmarked
-models with real model ids. modelgrep aggregates benchmark numbers from
-[Artificial Analysis](https://artificialanalysis.ai).
+Both caches live under `~/.cache/pi/` and are shared across every Pi
+extension using the same `DataSource` class — whichever extension loads
+first populates the cache; subsequent extensions read from disk.
 
-- **Context window + pricing** — taken verbatim from modelgrep.
-- **Score** — computed locally from the raw benchmark fields (see below).
-- **Rank** — the model's position once the whole catalog is sorted by that score
-  (best = `#1`). Unscored models sink to the bottom.
+## Data sources
 
-Cached 24h → `~/.cache/pi/modelgrep.json`. On outage the stale cache keeps the
-picker working until it can refresh.
+- **`modelgrep`** — `GET /api/v1/models?sort=coding&order=desc&limit=200`,
+  paginated up to 10 pages (`meta.has_more` / `next_offset`). Free, no API key.
+  modelgrep aggregates benchmark numbers from
+  [Artificial Analysis](https://artificialanalysis.ai). Context window, pricing,
+  and modalities are taken verbatim from the catalog.
+- **`benchlm`** — `GET https://benchlm.ai/api/data/leaderboard`. Free, no API
+  key. Each entry has an `overallScore` (0–100) used as the fallback score
+  when modelgrep's `artificial_analysis` block is null.
+
+Cache files:
+
+- `~/.cache/pi/modelgrep.json` (TTL 24h)
+- `~/.cache/pi/benchlm.json` (TTL 24h)
+
+On outage the stale cache keeps the picker working until it can refresh.
 
 ## Scoring methodology
 
-**Primary score = [Artificial Analysis Intelligence Index](https://artificialanalysis.ai/methodology/intelligence-benchmarking)**
-when available — AA's authoritative composite of 9 independent evals (agents,
-coding, scientific reasoning, general), already weighted toward agentic work.
-It is rescaled to 0–100 (`intelligence / 65 × 100`; the current leader scores
-~65).
-
-**Fallback = a coding-and-agentic heuristic** for the ~84% of models AA has not
-index-scored, computed from the raw benchmarks below, then mapped onto the index
-scale by a least-squares line. Both the heuristic weights *and* the line were
-jointly tuned against the index on the models that carry *both* it and the raw
-benches (`index100 ≈ 120.6·heuristic − 10.6`, deduped n=29, R²=0.901,
-leave-one-out RMSE 6.55pt) — a data calibration, not a guessed penalty. The
-picker exists to choose a model *for coding work in an agent*, so the heuristic
-is weighted toward exactly that:
+The score a model receives is the first of the following that succeeds, in
+order:
+
+1. **Primary = [Artificial Analysis Intelligence Index](https://artificialanalysis.ai/methodology/intelligence-benchmarking)**
+   when present on the modelgrep entry — AA's authoritative composite of 9
+   independent evals (agents, coding, scientific reasoning, general), already
+   weighted toward agentic work. Rescaled to 0–100
+   (`intelligence / 65 × 100`; the current leader scores ~65).
+2. **Heuristic** from modelgrep's raw benchmark fields when the AA index is
+   absent. Weighted blend of the same family of evals AA uses, then mapped onto
+   the index scale by a least-squares line. Both the heuristic weights *and*
+   the line were jointly tuned against the index on the models that carry
+   *both* it and the raw benches (`index100 ≈ 120.6·heuristic − 10.6`, deduped
+   n=29, R²=0.901, leave-one-out RMSE 6.55pt) — a data calibration, not a
+   guessed penalty. The picker exists to choose a model *for coding work in an
+   agent*, so the heuristic is weighted toward exactly that:
 
 | bench | range | measures |
 |---|---|---|
@@ -59,6 +74,12 @@ heuristic = 0.30·coding_score + 0.60·agentic_score + 0.10·reasoning_score
 score     = round(clamp₀₁₀₀(120.6·heuristic − 10.6))   // fitted to the index
 ```
 
+3. **benchlm.ai fallback** — if the model exists in benchlm but modelgrep has
+   no AA index and no raw benches, look up the benchlm `overallScore` (0–100)
+   and use it verbatim. Match strategy (in `lookupBenchlmScore`): exact
+   normalized slug, then prefix overlap either way, then take the
+   highest-scoring match on a tie.
+
 **Why a heuristic at all, and why these raw evals only:** the AA Intelligence
 Index *is* the ideal number — but only ~16% of the catalog has it. For the rest
 we rebuild a comparable score from the same family of raw evals. Crucially we
@@ -89,13 +110,15 @@ place if your priorities differ.
 
 | Export | Description |
 |---|---|
-| `modelgrep` | `DataSource<ModelGrepModel[]>` — the catalog. TTL 24h → `~/.cache/pi/modelgrep.json` |
+| `modelgrep` | `DataSource<ModelGrepModel[]>` — the modelgrep catalog. TTL 24h → `~/.cache/pi/modelgrep.json` |
+| `benchlm` | `DataSource<BenchLMRawEntry[]>` — the benchlm.ai leaderboard (fallback scores). TTL 24h → `~/.cache/pi/benchlm.json` |
 | `DataSource` | Generic cached data source class |
 | `CACHE_DIR` | Resolved cache directory (`~/.cache/pi`) |
 | `buildModelsDevIndex` | Build a lookup `Map` from the catalog (context/cost/modalities) |
 | `lookupInIndex` | Fuzzy-match a router model id against an index |
-| `lookupModelsDev` | Sync lookup by provider + id from in-memory cache |
+| `lookupModelsDev` | Sync lookup by id from in-memory cache (joined on slug) |
 | `lookupBenchmark` | Sync lookup a model by id — returns score + rank + pricing |
+| `benchScoreColor` | Map a 0–100 score to a `success`/`warning`/`error`/`muted` token |
 
 ## Install
 
@@ -105,10 +128,11 @@ pi install npm:@xynogen/pix-data
 
 ## How it works
 
-On session start the extension fires a background fetch (`modelgrep.get()`),
-paginating the API until the full benchmarked catalog is retrieved. If the cache
-is fresh the fetch is skipped. The cache file lives in `~/.cache/pi/` — any Pi
-extension using the same `DataSource` shares it automatically.
+On session start the extension fires two non-blocking fetches in parallel
+(`modelgrep.get()` and `benchlm.get()`) — Pi session start is not gated on
+either. If the cache is fresh both fetches are skipped. The cache files live
+in `~/.cache/pi/` — any Pi extension using the same `DataSource` shares them
+automatically.
 
 ## Full distro
 

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@xynogen/pix-data",
-	"version": "0.2.2",
+	"version": "0.2.4",
 	"description": "Pi extension — shared model data layer (models.dev + BenchLM), cached at ~/.cache/pi",
 	"type": "module",
 	"main": "src/index.ts",

package/src/data.test.ts

@@ -1,5 +1,6 @@
 import { afterEach, beforeEach, describe, expect, it } from "bun:test";
 import {
+	benchlm,
 	buildModelsDevIndex,
 	lookupBenchmark,
 	lookupInIndex,
@@ -227,3 +228,88 @@ describe("modelgrep adapters", () => {
 		expect(lookupBenchmark("nonexistent-model-xyz")).toBeUndefined();
 	});
 });
+
+// ── benchlm fallback (modelgrep AA null → benchlm) ────────────────────────────
+
+describe("benchlm fallback", () => {
+	// modelgrep catalog: every model has null benchmarks (real-world shape today)
+	const catalog: ModelGrepModel[] = [
+		mg("anthropic/claude-opus-4-8", { name: "Claude Opus 4.8" }),
+		mg("minimax/minimax-m3", { name: "MiniMax M3" }),
+		mg("deepseek/deepseek-v4-pro", { name: "DeepSeek V4 Pro" }),
+		mg("qwen/qwen3.7-max", { name: "Qwen3.7 Max" }),
+		mg("ghost/uncataloged", { name: "Ghost" }), // not in benchlm either
+	];
+	// benchlm: real shape (no benchmarks field, just overallScore 0-100)
+	const benchlmEntries = [
+		{ rank: 1, model: "Claude Opus 4.8 (Max)", overallScore: 95 },
+		{ rank: 2, model: "Claude Opus 4.8", overallScore: 93 },
+		{ rank: 25, model: "MiniMax M3", overallScore: 78 },
+		{ rank: 39, model: "DeepSeek V4 Pro", overallScore: 68 },
+		{ rank: 10, model: "Qwen3.7 Max", overallScore: 90 },
+	];
+
+	beforeEach(() => {
+		(modelgrep as unknown as { _mem: ModelGrepModel[] })._mem = catalog;
+		(benchlm as unknown as { _mem: typeof benchlmEntries })._mem =
+			benchlmEntries;
+	});
+	afterEach(() => {
+		(modelgrep as unknown as { _mem: ModelGrepModel[] | null })._mem = null;
+		(benchlm as unknown as { _mem: typeof benchlmEntries | null })._mem = null;
+	});
+
+	it("falls back to benchlm when modelgrep benchmarks are null", () => {
+		const b = lookupBenchmark("claude-opus-4-8");
+		// Two candidates: (Max)=95, base=93 → pick higher
+		expect(b?.overallScore).toBe(95);
+	});
+
+	it("prefers the higher-scoring benchlm variant when multiple match", () => {
+		const b = lookupBenchmark("minimax-m3");
+		expect(b?.overallScore).toBe(78);
+	});
+
+	it("returns null when both modelgrep and benchlm lack the model", () => {
+		const b = lookupBenchmark("uncataloged");
+		expect(b?.overallScore).toBeNull();
+	});
+
+	it("ranks scored models above unscored when only some have benchlm data", () => {
+		// catalog has 5 models, 4 in benchlm → uncataloged sinks to last
+		const b = lookupBenchmark("uncataloged");
+		expect(b?.rank).toBe(5); // 4 scored + 1 unscored at bottom
+	});
+
+	it("normalizes dots and parens: qwen3.7-max ↔ Qwen3.7 Max", () => {
+		const b = lookupBenchmark("qwen3.7-max");
+		expect(b?.overallScore).toBe(90);
+	});
+});
+
+describe("modelgrep AA primary wins over benchlm", () => {
+	const catalog: ModelGrepModel[] = [
+		mg("anthropic/claude-opus-4-8", {
+			bench: { intelligence: 60 }, // AA index: 60/65 → 92
+		}),
+	];
+	const benchlmEntries = [
+		{ rank: 1, model: "Claude Opus 4.8", overallScore: 50 },
+	];
+
+	beforeEach(() => {
+		(modelgrep as unknown as { _mem: ModelGrepModel[] })._mem = catalog;
+		(benchlm as unknown as { _mem: typeof benchlmEntries })._mem =
+			benchlmEntries;
+	});
+	afterEach(() => {
+		(modelgrep as unknown as { _mem: ModelGrepModel[] | null })._mem = null;
+		(benchlm as unknown as { _mem: typeof benchlmEntries | null })._mem = null;
+	});
+
+	it("uses AA intelligence when present, ignores benchlm", () => {
+		const b = lookupBenchmark("claude-opus-4-8");
+		// 60/65 * 100 = 92.23 → 92, not benchlm's 50
+		expect(b?.overallScore).toBe(92);
+	});
+});

package/src/data.ts

@@ -1,9 +1,11 @@
 /**
  * data.ts — shared Pi model data layer
  *
- * Single source of truth, sourced from modelgrep (coding-sorted), cached at
- * ~/.cache/pi/modelgrep.json (TTL 24h). Provides context, cost, modalities,
- * capabilities, coding-percentile score, and rank.
+ * Two data sources, each its own cached DataSource:
+ *   - modelgrep (coding-sorted catalog) — ~/.cache/pi/modelgrep.json (TTL 24h):
+ *     context, cost, modalities, capabilities, coding-percentile score, rank.
+ *   - BenchLM — ~/.cache/pi/benchlm.json: fallback overall score when modelgrep
+ *     has no benchmark for a model (see lookupBenchmark).
  *
  * Cache files are shared across all Pi extensions — whichever extension loads
  * first populates the cache; subsequent extensions read from disk.
@@ -285,6 +287,40 @@ export const modelgrep = new DataSource<ModelGrepModel[]>({
 	empty: [],
 });
 
+// ── BenchLM (fallback coding-score source) ────────────────────────────────────
+// Upstream `benchlm.ai` ships a 0–100 `overallScore` per model with category
+// breakdown (coding/agentic/reasoning/…). Used as a fallback when modelgrep's
+// `benchmarks.artificial_analysis` is null (current state). Same name as
+// before the 4dfb443 swap.
+interface BenchLMCategoryScores {
+	coding?: number | null;
+	agentic?: number | null;
+	reasoning?: number | null;
+}
+
+interface BenchLMRawEntry {
+	rank: number;
+	model: string;
+	creator?: string;
+	overallScore: number | null;
+	categoryScores?: BenchLMCategoryScores;
+}
+
+interface BenchLMResponse {
+	lastUpdated?: string;
+	mode?: string;
+	models?: BenchLMRawEntry[];
+}
+
+export const benchlm = new DataSource<BenchLMRawEntry[]>({
+	label: "benchlm",
+	url: "https://benchlm.ai/api/data/leaderboard",
+	cachePath: join(CACHE_DIR, "benchlm.json"),
+	parse: (raw) => (raw as BenchLMResponse).models ?? [],
+	parseCache: (data) => (data as BenchLMResponse)?.models ?? [],
+	empty: [],
+});
+
 // ── Lookup helpers ─────────────────────────────────────────────────────────────
 
 function normalize(id: string): string {
@@ -443,12 +479,24 @@ function codingScore(
 
 function buildBenchIndex(): Map<string, BenchmarkEntry> {
 	const index = new Map<string, BenchmarkEntry>();
+	// BenchLM lookup table: normalized benchlm name → entry, indexed in source
+	// order (highest score first when ties exist). Built once per call.
+	const benchlmByNorm = new Map<string, BenchLMRawEntry[]>();
+	for (const b of benchlm.getCached()) {
+		const k = normalizeBenchlmName(b.model);
+		if (!k) continue;
+		const arr = benchlmByNorm.get(k) ?? [];
+		arr.push(b);
+		benchlmByNorm.set(k, arr);
+	}
+
 	// Rank by our computed score (desc); unscored sink to the bottom, holding
 	// source order among themselves.
-	const scored = modelgrep.getCached().map((g) => ({
-		g,
-		score: g.benchmarks ? codingScore(g.benchmarks) : null,
-	}));
+	const scored = modelgrep.getCached().map((g) => {
+		const fromAA = g.benchmarks ? codingScore(g.benchmarks) : null;
+		const score = fromAA ?? lookupBenchlmScore(g, benchlmByNorm);
+		return { g, score };
+	});
 	scored.sort((a, b) => (b.score ?? -1) - (a.score ?? -1));
 	scored.forEach(({ g, score }, i) => {
 		const slug = slugOf(g.id);
@@ -466,6 +514,55 @@ function buildBenchIndex(): Map<string, BenchmarkEntry> {
 	return index;
 }
 
+// Normalize a benchlm `model` field (e.g. "Claude Opus 4.8 (Max)") to a slug
+// comparable to modelgrep ids (e.g. "claude-opus-4-8"). Drops parenthesized
+// variants, lowercases, folds . _ space → -, strips leading/trailing dashes.
+function normalizeBenchlmName(name: string): string {
+	return name
+		.replace(/\s*\([^)]*\)\s*/g, " ") // drop "(Max)", "(High)", etc.
+		.toLowerCase()
+		.replace(/[._\s]+/g, "-")
+		.replace(/-+/g, "-")
+		.replace(/^-|-$/g, "");
+}
+
+// Try to find a benchlm score for a modelgrep model. Match strategy:
+//   1. exact normalized match of modelgrep slug
+//   2. prefix overlap (claude-opus-4-8 ↔ claude-opus-4-8-thinking) — benchlm
+//      may list a long-form name; prefer the shortest match on tie (base > variants)
+//   3. if multiple benchlm entries match, return the highest score
+function lookupBenchlmScore(
+	g: ModelGrepModel,
+	benchlmByNorm: Map<string, BenchLMRawEntry[]>,
+): number | null {
+	const slug = slugOf(g.id);
+	const norm = normalize(slug);
+
+	// Collect candidates: exact match + prefix matches (either side).
+	const candidates: BenchLMRawEntry[] = [];
+	const direct = benchlmByNorm.get(norm);
+	if (direct) candidates.push(...direct);
+	for (const [key, entries] of benchlmByNorm) {
+		if (key === norm) continue;
+		if (key.startsWith(norm) || norm.startsWith(key))
+			candidates.push(...entries);
+	}
+	if (candidates.length === 0) return null;
+
+	// Best entry = highest overallScore. Sort by score desc, then by slug
+	// length asc (prefer base name over suffix variants on a tie).
+	const best = [...candidates].sort((a, b) => {
+		const sa = a.overallScore ?? -Infinity;
+		const sb = b.overallScore ?? -Infinity;
+		if (sa !== sb) return sb - sa;
+		return (
+			normalizeBenchlmName(a.model).length -
+			normalizeBenchlmName(b.model).length
+		);
+	})[0];
+	return best.overallScore ?? null;
+}
+
 /** Map a benchmark score (0–100) to a semantic color token. */
 export function benchScoreColor(
 	score: number | null | undefined,

package/src/index.ts

@@ -4,11 +4,12 @@
  * Warms the shared model data cache on session start so other extensions
  * (pix-9router, models picker, footer) can read from ~/.cache/pi/* synchronously.
  *
- * Single non-blocking fetch — Pi session starts immediately.
+ * Two non-blocking fetches (modelgrep catalog + BenchLM scores) — Pi session
+ * starts immediately; consumers read whichever cache file they need.
  */
 
 import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
-import { modelgrep } from "./data.ts";
+import { benchlm, modelgrep } from "./data.ts";
 
 export type {
 	BenchmarkEntry,
@@ -20,6 +21,7 @@ export type {
 // Consumers (pix-core, pix-9router, …) import these instead of duplicating
 // the DataSource implementation and models.dev/BenchLM lookups.
 export {
+	benchlm,
 	benchScoreColor,
 	buildModelsDevIndex,
 	CACHE_DIR,
@@ -33,4 +35,5 @@ export {
 
 export default function (_pi: ExtensionAPI): void {
 	void modelgrep.get();
+	void benchlm.get();
 }