@danypops/jittor 0.8.0 → 0.9.0

package/README.md

@@ -80,9 +80,11 @@ Token-budget thresholds are optional and must be configured by the user; Jittor
 
 ### Benchmark evidence
 
-Jittor can ingest bounded OpenRouter model metadata, p50 latency/throughput ordering, versioned Artificial Analysis benchmark indices, and Design Arena Elo rankings as provenance-bearing evidence without treating OpenRouter as model-scope authority. Enable online ingestion explicitly with `JITTOR_OPENROUTER_BENCHMARKS=1`; it is off by default. OpenRouter model metadata and operational ordering are public; benchmark-index and Design Arena ingestion additionally use `OPENROUTER_API_KEY` from the supervised service environment without retaining it. Snapshots preserve the upstream publisher, normalized model identities, immutable retrieval revisions, source URLs, confidence, license terms, and explicit freshness deadlines. A malformed or oversized refresh leaves the last complete snapshot visible and records only a payload-safe failure state.
+Jittor can ingest bounded OpenRouter model metadata, p50 latency/throughput ordering, and Design Arena Elo rankings as provenance-bearing evidence without treating OpenRouter as model-scope authority. Enable online ingestion explicitly with `JITTOR_OPENROUTER_BENCHMARKS=1`; it is off by default. OpenRouter model metadata and operational ordering are public; Design Arena ingestion additionally uses `OPENROUTER_API_KEY` from the supervised service environment without retaining it. Snapshots preserve the upstream publisher, normalized model identities, immutable retrieval revisions, source URLs, confidence, license terms, and explicit freshness deadlines. A malformed or oversized refresh leaves the last complete snapshot visible and records only a payload-safe failure state.
 
-Design Arena rates models across dozens of arena/category pairs (music, video, text-to-speech, ASCII art, ...); Jittor ingests only the bounded allowlist of categories (`codecategories`, `website`, `uicomponent`, `dataviz`, `svg`) that measure frontend/UI-generation skill relevant to routing a coding agent, tagged into one `design` domain distinct from `coding`. A model with no OpenRouter-reachable identity (proprietary platforms, image/video generators) is skipped rather than fabricated into unroutable evidence.
+Design Arena rates models across dozens of arena/category pairs (music, video, text-to-speech, ASCII art, ...); Jittor ingests only the bounded allowlist of categories (`codecategories`, `website`, `uicomponent`, `dataviz`, `svg`) that measure frontend/UI-generation skill relevant to routing a coding agent, tagged into one `design` domain distinct from `coding`. A model with no OpenRouter-reachable identity (proprietary platforms, image/video generators) is skipped rather than fabricated into unroutable evidence. Kept on the OpenRouter passthrough rather than migrated to a direct integration: Design Arena's own native API requires a manually reviewed application (1-2 business days), unlike Artificial Analysis's instant self-serve signup below.
+
+Jittor also ingests LMArena's own official Hugging Face dataset (`lmarena-ai/leaderboard-dataset`, via the public `datasets-server.huggingface.co` API -- no credential required) for its Code Arena (`webdev`) and Agent Arena human-preference battles, and, when `ARTIFICIAL_ANALYSIS_API_KEY` is configured, Artificial Analysis's own direct API (replaces the former OpenRouter passthrough to the same publisher; adds a `math` domain and measured per-model latency the passthrough never exposed). LMArena's Bradley-Terry/IPS ratings aren't on the same scale as Artificial Analysis's 0-100 indices, so they're tagged under distinct `-arena`-suffixed dimensions (`quality-coding-arena`, `quality-type-planning-arena`) instead of blended into the same average -- stored and queryable on their own, not yet part of the default ranked "quality" score.
 
 Use the authenticated CLI channels independently:
 

package/extension/src/benchmark-tui.ts

@@ -85,7 +85,7 @@ export async function showBenchmarkPanel(
 				context: MODEL_RANKING_DEFAULT_CONTEXT_WEIGHT,
 				reliability: MODEL_RANKING_DEFAULT_RELIABILITY_WEIGHT,
 			},
-			sourceIds: ["openrouter-models", "openrouter-artificial-analysis", "openrouter-design-arena"],
+			sourceIds: ["openrouter-models", "lmarena-hf", "artificial-analysis-direct", "openrouter-design-arena"],
 		}) as ModelRankingResult;
 		if (ctx.mode !== "tui") {
 			ctx.ui.notify(renderBenchmarkView(result, currentIdentity, 100, { fg: (_color, text) => text, bold: (text) => text }).join("\n"), "info");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@danypops/jittor",
-  "version": "0.8.0",
+  "version": "0.9.0",
   "description": "Just-in-Time Token Optimizing Router for Pi",
   "type": "module",
   "keywords": ["pi-package", "llm-router", "token-budget"],

package/src/adapters/artificial-analysis-direct-source.ts

@@ -0,0 +1,80 @@
+import { BENCHMARK_MAX_MODELS_PER_SOURCE, BENCHMARK_REFRESH_INTERVAL_MS, BENCHMARK_SOURCE_MAX_RESPONSE_BYTES } from "../constants.ts";
+import { normalizeModelIdentity, validateBenchmarkObservation, type BenchmarkObservation, type BenchmarkSourceSnapshot } from "../domain/benchmark.ts";
+import type { BenchmarkSource } from "../ports/benchmark-source.ts";
+import { contractRecord } from "../providers/openrouter-contracts.ts";
+
+const SOURCE_ID = "artificial-analysis-direct";
+const ENDPOINT = "https://artificialanalysis.ai/api/v2/data/llms/models";
+
+export type ArtificialAnalysisTransport = (request: Request) => Promise<Response>;
+
+function requiredText(value: unknown, name: string): string {
+	if (typeof value !== "string" || value.length === 0 || value.length > 500) throw new Error(`Artificial Analysis ${name} schema changed`);
+	return value;
+}
+
+function requiredNumber(value: unknown, name: string): number {
+	if (typeof value !== "number" || !Number.isFinite(value)) throw new Error(`Artificial Analysis ${name} schema changed`);
+	return value;
+}
+
+/** Replaces the removed OpenRouter passthrough to the same publisher; adds math_index and measured latency it never exposed. Same dimension names, since it's the same facts. */
+export class ArtificialAnalysisDirectSource implements BenchmarkSource {
+	readonly id = SOURCE_ID;
+
+	constructor(
+		private readonly apiKey: string,
+		private readonly transport: ArtificialAnalysisTransport = fetch,
+		private readonly clock: () => number = Date.now,
+	) {
+		if (apiKey.length === 0) throw new Error("Artificial Analysis API key is required");
+	}
+
+	async fetch(): Promise<BenchmarkSourceSnapshot> {
+		const retrievedAt = this.clock();
+		if (!Number.isSafeInteger(retrievedAt) || retrievedAt <= 0) throw new Error("benchmark retrieval time is invalid");
+		const response = await this.transport(new Request(ENDPOINT, { headers: { "x-api-key": this.apiKey } }));
+		if (!response.ok) throw new Error(`Artificial Analysis benchmarks failed with HTTP ${response.status}`);
+		const text = await response.text();
+		if (new TextEncoder().encode(text).byteLength > BENCHMARK_SOURCE_MAX_RESPONSE_BYTES) throw new Error("Artificial Analysis benchmark response exceeds the size limit");
+		let payload: unknown;
+		try { payload = JSON.parse(text); } catch { throw new Error("Artificial Analysis benchmark response is not valid JSON"); }
+		const root = contractRecord(payload, "benchmark response");
+		if (!Array.isArray(root["data"]) || root["data"].length > BENCHMARK_MAX_MODELS_PER_SOURCE) throw new Error("Artificial Analysis benchmark result count is invalid");
+		const revision = String(retrievedAt);
+		const observations = root["data"].flatMap((value): BenchmarkObservation[] => {
+			const row = contractRecord(value, "benchmark row");
+			const creator = contractRecord(row["model_creator"], "model creator");
+			const evaluations = contractRecord(row["evaluations"], "evaluations");
+			const identity = normalizeModelIdentity(requiredText(creator["slug"], "creator slug"), requiredText(row["slug"], "model slug"), [`artificial-analysis/${requiredText(row["id"], "model id")}`]);
+			const provenance = {
+				sourceId: SOURCE_ID,
+				sourceType: "creator" as const,
+				publisher: "Artificial Analysis",
+				url: "https://artificialanalysis.ai/",
+				revision,
+				publishedAt: retrievedAt,
+				retrievedAt,
+				freshUntil: retrievedAt + BENCHMARK_REFRESH_INTERVAL_MS,
+				license: "Attribution to artificialanalysis.ai required; see their free API terms",
+				confidence: 0.85,
+			};
+			const methodology = { basis: "Artificial Analysis direct API, /data/llms/models" };
+			const observations: BenchmarkObservation[] = [];
+			const index = (field: string, dimension: string): void => {
+				const raw = evaluations[field];
+				if (raw === undefined || raw === null) return;
+				observations.push(validateBenchmarkObservation({ model: identity, dimension, value: requiredNumber(raw, field), unit: "ratio", provenance, methodology }));
+			};
+			index("artificial_analysis_coding_index", "quality-coding");
+			index("artificial_analysis_intelligence_index", "quality-general");
+			index("artificial_analysis_math_index", "quality-math");
+			const ttft = row["median_time_to_first_token_seconds"];
+			if (typeof ttft === "number" && Number.isFinite(ttft)) {
+				observations.push(validateBenchmarkObservation({ model: identity, dimension: "latency", value: ttft * 1_000, unit: "milliseconds", provenance, methodology }));
+			}
+			return observations;
+		});
+		return { sourceId: this.id, snapshotId: `${this.id}:${revision}`, retrievedAt, observations };
+	}
+}

package/src/adapters/lmarena-hf-source.ts

@@ -0,0 +1,111 @@
+import { BENCHMARK_MAX_MODELS_PER_SOURCE, BENCHMARK_REFRESH_INTERVAL_MS, BENCHMARK_SOURCE_MAX_RESPONSE_BYTES } from "../constants.ts";
+import { normalizeModelIdentity, validateBenchmarkObservation, type BenchmarkObservation, type BenchmarkSourceSnapshot } from "../domain/benchmark.ts";
+import type { BenchmarkSource } from "../ports/benchmark-source.ts";
+import { contractRecord } from "../providers/openrouter-contracts.ts";
+
+const SOURCE_ID = "lmarena-hf";
+const BASE_URL = "https://datasets-server.huggingface.co/rows";
+/** The server's own per-page cap; observed directly against the live API. */
+const HF_ROWS_PER_PAGE = 100;
+
+export type LmArenaTransport = (request: Request) => Promise<Response>;
+
+/**
+ * Limited to webdev (Code Arena) and agent (Agent Arena); text/text_style_control's "coding"
+ * category exists but its "latest" split spans tens of thousands of rows, unbounded compared to
+ * these two's few hundred.
+ *
+ * Dimensions are distinct from AA's (quality-coding-arena, not quality-coding) because the
+ * scales don't match: Bradley-Terry rating (~1200-1700) and IPS score (~0) vs AA's 0-100 ratio.
+ * withEvidence() averages same-named dimensions with no unit conversion, so sharing a name would
+ * blend incompatible scales into a meaningless number. Stored and queryable on their own,
+ * intentionally not folded into models.rank's blended "quality" component yet.
+ */
+const ARENAS: ReadonlyArray<{ config: string; dimension: string; unit: "ratio" }> = [
+	{ config: "webdev", dimension: "quality-coding-arena", unit: "ratio" },
+	{ config: "agent", dimension: "quality-type-planning-arena", unit: "ratio" },
+];
+
+const THINKING_SUFFIX = /\s*\((?:high|xhigh|low|medium|max|fast|thinking|codex-harness)\)\s*$/i;
+
+function requiredText(value: unknown, name: string): string {
+	if (typeof value !== "string" || value.length === 0 || value.length > 500) throw new Error(`LMArena benchmark ${name} schema changed`);
+	return value;
+}
+
+function requiredNumber(value: unknown, name: string): number {
+	if (typeof value !== "number" || !Number.isFinite(value)) throw new Error(`LMArena benchmark ${name} schema changed`);
+	return value;
+}
+
+/**
+ * Best-effort: LMArena has no machine identity field, just a display name and org slug. A wrong
+ * guess is safe, not a correctness bug -- identity only ever matches exactly against real
+ * candidates the caller already supplied, so a bad slug or an unreleased codename ("Inkling")
+ * is just inert evidence, never misattributed evidence.
+ */
+function bestEffortIdentity(organization: string, displayName: string): { provider: string; model: string; aliases: string[] } {
+	const base = displayName.replace(THINKING_SUFFIX, "").trim();
+	const slug = base.toLowerCase().replace(/[^a-z0-9.]+/g, "-").replace(/^-+|-+$/g, "");
+	return { provider: organization, model: slug, aliases: [displayName.toLowerCase()] };
+}
+
+export class LmArenaHfSource implements BenchmarkSource {
+	readonly id = SOURCE_ID;
+
+	constructor(
+		private readonly transport: LmArenaTransport = fetch,
+		private readonly clock: () => number = Date.now,
+	) {}
+
+	async fetch(): Promise<BenchmarkSourceSnapshot> {
+		const retrievedAt = this.clock();
+		if (!Number.isSafeInteger(retrievedAt) || retrievedAt <= 0) throw new Error("benchmark retrieval time is invalid");
+		const perArena = await Promise.all(ARENAS.map((arena) => this.fetchArena(arena, retrievedAt)));
+		const observations = perArena.flatMap((page) => page.observations).slice(0, BENCHMARK_MAX_MODELS_PER_SOURCE * ARENAS.length);
+		const asOf = perArena.map((page) => page.asOf).sort().at(-1) ?? "unknown";
+		return { sourceId: this.id, snapshotId: `${this.id}:${asOf}`, retrievedAt, observations };
+	}
+
+	private async fetchArena(arena: { config: string; dimension: string; unit: "ratio" }, retrievedAt: number): Promise<{ asOf: string; observations: BenchmarkObservation[] }> {
+		const url = `${BASE_URL}?dataset=lmarena-ai%2Fleaderboard-dataset&config=${arena.config}&split=latest&length=${HF_ROWS_PER_PAGE}`;
+		const response = await this.transport(new Request(url));
+		if (!response.ok) throw new Error(`LMArena benchmark fetch failed with HTTP ${response.status}`);
+		const text = await response.text();
+		if (new TextEncoder().encode(text).byteLength > BENCHMARK_SOURCE_MAX_RESPONSE_BYTES) throw new Error("LMArena benchmark response exceeds the size limit");
+		let payload: unknown;
+		try { payload = JSON.parse(text); } catch { throw new Error("LMArena benchmark response is not valid JSON"); }
+		const root = contractRecord(payload, "benchmark response");
+		if (!Array.isArray(root["rows"]) || root["rows"].length > HF_ROWS_PER_PAGE) throw new Error("LMArena benchmark row count is invalid");
+		let asOf = "";
+		const observations = root["rows"].flatMap((entry): BenchmarkObservation[] => {
+			const wrapper = contractRecord(entry, "benchmark row wrapper");
+			const row = contractRecord(wrapper["row"], "benchmark row");
+			const publishDate = requiredText(row["leaderboard_publish_date"], "publish date");
+			if (publishDate > asOf) asOf = publishDate;
+			// Skip finer per-category splits, if any exist for this config -- out of scope for now.
+			if (requiredText(row["category"], "category") !== "overall") return [];
+			const scoreValue = row["rating"] ?? row["score"];
+			const value = requiredNumber(scoreValue, "score");
+			const organization = requiredText(row["organization"], "organization");
+			const displayName = requiredText(row["model_name"], "model name");
+			const guessed = bestEffortIdentity(organization, displayName);
+			const identity = normalizeModelIdentity(guessed.provider, guessed.model, guessed.aliases);
+			const provenance = {
+				sourceId: SOURCE_ID,
+				sourceType: "preference" as const,
+				publisher: "LMArena",
+				url: "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
+				revision: `${arena.config}:${publishDate}`,
+				publishedAt: Date.parse(publishDate) || null,
+				retrievedAt,
+				freshUntil: retrievedAt + BENCHMARK_REFRESH_INTERVAL_MS,
+				license: "See lmarena-ai/leaderboard-dataset on Hugging Face for per-model license terms",
+				confidence: 0.6,
+			};
+			const methodology = { basis: "LMArena human-preference battles", arena: arena.config, displayName, publishDate };
+			return [validateBenchmarkObservation({ model: identity, dimension: arena.dimension, value, unit: arena.unit, provenance, methodology })];
+		});
+		return { asOf, observations };
+	}
+}

package/src/daemon.ts

@@ -3,9 +3,10 @@ import { MAINTENANCE_INTERVAL_MS, TELEMETRY_POLL_INTERVAL_MS } from "./constants
 import { DEFAULT_POLICY, UNCONFIGURED_ROUTE } from "./config.ts";
 import { SQLiteMetricStore } from "./adapters/sqlite-metric-store.ts";
 import { MetricBenchmarkStore } from "./adapters/metric-benchmark-store.ts";
-import { OpenRouterBenchmarkIndexSource } from "./adapters/openrouter-benchmark-index-source.ts";
 import { OpenRouterBenchmarkSource } from "./adapters/openrouter-benchmark-source.ts";
 import { OpenRouterDesignArenaSource } from "./adapters/openrouter-design-arena-source.ts";
+import { LmArenaHfSource } from "./adapters/lmarena-hf-source.ts";
+import { ArtificialAnalysisDirectSource } from "./adapters/artificial-analysis-direct-source.ts";
 import { openJittorDb } from "./db.ts";
 import { BenchmarkCatalog } from "./domain/benchmark.ts";
 import { EvidenceModelRanker } from "./domain/model-ranking-service.ts";
@@ -26,13 +27,15 @@ export function reportMaintenanceFailure(event: string, error: unknown): void {
 	logEvent("error", event, { message: error instanceof Error ? error.message : String(error) });
 }
 
+// Flag name predates non-OpenRouter sources; kept as the one "opt into online benchmark
+// ingestion" toggle rather than adding a second flag for the same decision.
 export function benchmarkSourcesFromEnvironment(env: Record<string, string | undefined> = process.env): BenchmarkSource[] {
 	if (env["JITTOR_OPENROUTER_BENCHMARKS"] !== "1") return [];
-	const sources: BenchmarkSource[] = [new OpenRouterBenchmarkSource()];
-	if (env["OPENROUTER_API_KEY"]) {
-		sources.push(new OpenRouterBenchmarkIndexSource(env["OPENROUTER_API_KEY"]));
-		sources.push(new OpenRouterDesignArenaSource(env["OPENROUTER_API_KEY"]));
-	}
+	const sources: BenchmarkSource[] = [new OpenRouterBenchmarkSource(), new LmArenaHfSource()];
+	// Design Arena's own API needs manual approval, unlike Artificial Analysis's instant signup --
+	// no direct alternative exists yet, so the OpenRouter passthrough stays.
+	if (env["OPENROUTER_API_KEY"]) sources.push(new OpenRouterDesignArenaSource(env["OPENROUTER_API_KEY"]));
+	if (env["ARTIFICIAL_ANALYSIS_API_KEY"]) sources.push(new ArtificialAnalysisDirectSource(env["ARTIFICIAL_ANALYSIS_API_KEY"]));
 	return sources;
 }
 

package/src/domain/model-observation.ts

@@ -16,7 +16,7 @@ import type { MetricObservation, MetricUnit, StoredMetricObservation } from "./m
  * usage carries no distinguishing signal for that axis; a run can score coding on domain and
  * research on type simultaneously (e.g. reading a file, then searching the web in one turn).
  */
-export const TASK_DOMAINS = ["coding", "design", "general"] as const;
+export const TASK_DOMAINS = ["coding", "design", "math", "general"] as const;
 export type ModelTaskDomain = typeof TASK_DOMAINS[number];
 export const TASK_TYPES = ["research", "planning", "general"] as const;
 export type ModelTaskType = typeof TASK_TYPES[number];

package/src/adapters/openrouter-benchmark-index-source.ts

@@ -1,94 +0,0 @@
-import {
-	BENCHMARK_MAX_MODELS_PER_SOURCE,
-	BENCHMARK_REFRESH_INTERVAL_MS,
-	BENCHMARK_SOURCE_MAX_RESPONSE_BYTES,
-} from "../constants.ts";
-import { normalizeModelIdentity, validateBenchmarkObservation, type BenchmarkObservation, type BenchmarkSourceSnapshot } from "../domain/benchmark.ts";
-import type { BenchmarkSource } from "../ports/benchmark-source.ts";
-import { contractRecord } from "../providers/openrouter-contracts.ts";
-import type { OpenRouterBenchmarkTransport } from "./openrouter-benchmark-source.ts";
-
-const SOURCE_ID = "openrouter-artificial-analysis";
-const ENDPOINT = `https://openrouter.ai/api/v1/benchmarks?source=artificial-analysis&task_type=coding&max_results=${BENCHMARK_MAX_MODELS_PER_SOURCE}`;
-
-function requiredText(value: unknown, name: string): string {
-	if (typeof value !== "string" || value.length === 0 || value.length > 500) throw new Error(`OpenRouter benchmark ${name} schema changed`);
-	return value;
-}
-
-function requiredNumber(value: unknown, name: string): number {
-	if (typeof value !== "number" || !Number.isFinite(value)) throw new Error(`OpenRouter benchmark ${name} schema changed`);
-	return value;
-}
-
-function price(value: unknown, name: string): number {
-	const parsed = Number(requiredText(value, name));
-	if (!Number.isFinite(parsed) || parsed < 0) throw new Error(`OpenRouter benchmark ${name} schema changed`);
-	return parsed;
-}
-
-export class OpenRouterBenchmarkIndexSource implements BenchmarkSource {
-	readonly id = SOURCE_ID;
-
-	constructor(
-		private readonly apiKey: string,
-		private readonly transport: OpenRouterBenchmarkTransport = fetch,
-		private readonly clock: () => number = Date.now,
-	) {
-		if (apiKey.length === 0) throw new Error("OpenRouter API key is required for benchmark indices");
-	}
-
-	async fetch(): Promise<BenchmarkSourceSnapshot> {
-		const response = await this.transport(new Request(ENDPOINT, { headers: { authorization: `Bearer ${this.apiKey}` } }));
-		if (!response.ok) throw new Error(`OpenRouter benchmarks failed with HTTP ${response.status}`);
-		const text = await response.text();
-		if (new TextEncoder().encode(text).byteLength > BENCHMARK_SOURCE_MAX_RESPONSE_BYTES) throw new Error("OpenRouter benchmark response exceeds the size limit");
-		let payload: unknown;
-		try { payload = JSON.parse(text); } catch { throw new Error("OpenRouter benchmark response is not valid JSON"); }
-		const root = contractRecord(payload, "benchmark response");
-		const meta = contractRecord(root["meta"], "benchmark metadata");
-		if (!Array.isArray(root["data"]) || root["data"].length === 0 || root["data"].length > BENCHMARK_MAX_MODELS_PER_SOURCE) throw new Error("OpenRouter benchmark result count is invalid");
-		if (requiredText(meta["source"], "source") !== "artificial-analysis") throw new Error("OpenRouter benchmark source schema changed");
-		const version = requiredText(meta["version"], "version");
-		const asOf = requiredText(meta["as_of"], "as-of date");
-		const publishedAt = Date.parse(asOf);
-		if (!Number.isSafeInteger(publishedAt) || publishedAt <= 0) throw new Error("OpenRouter benchmark publication date schema changed");
-		const upstreamUrl = new URL(requiredText(meta["source_url"], "source URL"));
-		if (upstreamUrl.protocol !== "https:") throw new Error("OpenRouter benchmark source URL must use HTTPS");
-		const retrievedAt = this.clock();
-		if (!Number.isSafeInteger(retrievedAt) || retrievedAt <= 0) throw new Error("benchmark retrieval time is invalid");
-		const revision = `${version}:${asOf}`;
-		const observations = root["data"].flatMap((value): BenchmarkObservation[] => {
-			const row = contractRecord(value, "benchmark row");
-			if (requiredText(row["source"], "row source") !== "artificial-analysis") throw new Error("OpenRouter benchmark row source schema changed");
-			const permaslug = requiredText(row["model_permaslug"], "model permaslug");
-			const separator = permaslug.indexOf("/");
-			if (separator <= 0 || separator === permaslug.length - 1) throw new Error("OpenRouter benchmark model identity schema changed");
-			const model = normalizeModelIdentity(permaslug.slice(0, separator), permaslug.slice(separator + 1), [`openrouter/${permaslug}`]);
-			const pricing = contractRecord(row["pricing"], "benchmark pricing");
-			const provenance = {
-				sourceId: SOURCE_ID,
-				sourceType: "independent" as const,
-				publisher: "Artificial Analysis via OpenRouter",
-				url: ENDPOINT,
-				revision,
-				publishedAt,
-				retrievedAt,
-				freshUntil: retrievedAt + BENCHMARK_REFRESH_INTERVAL_MS,
-				license: "OpenRouter API terms; upstream terms apply",
-				confidence: 0.8,
-			};
-			const common = { model, provenance };
-			const methodology = { basis: "Artificial Analysis index via OpenRouter", upstreamUrl: upstreamUrl.toString(), version, asOf };
-			return [
-				validateBenchmarkObservation({ ...common, dimension: "quality-coding", value: requiredNumber(row["coding_index"], "coding index"), unit: "ratio", methodology }),
-				validateBenchmarkObservation({ ...common, dimension: "quality-general", value: requiredNumber(row["intelligence_index"], "intelligence index"), unit: "ratio", methodology }),
-				// agentic_index measures tool-use/agentic execution style, an activity (type), not a subject-matter domain.
-				validateBenchmarkObservation({ ...common, dimension: "quality-type-planning", value: requiredNumber(row["agentic_index"], "agentic index"), unit: "ratio", methodology }),
-				validateBenchmarkObservation({ ...common, dimension: "price-input", value: price(pricing["prompt"], "prompt pricing"), unit: "usd", methodology: { ...methodology, basis: "OpenRouter USD per input token" } }),
-				validateBenchmarkObservation({ ...common, dimension: "price-output", value: price(pricing["completion"], "completion pricing"), unit: "usd", methodology: { ...methodology, basis: "OpenRouter USD per output token" } }),
-			];
-		});
-		return { sourceId: this.id, snapshotId: `${this.id}:${revision}`, retrievedAt, observations };
-	}
-}