@danypops/jittor 0.7.0 → 0.8.0

package/README.md

@@ -80,7 +80,9 @@ Token-budget thresholds are optional and must be configured by the user; Jittor
 
 ### Benchmark evidence
 
-Jittor can ingest bounded OpenRouter model metadata, p50 latency/throughput ordering, and versioned Artificial Analysis benchmark indices as provenance-bearing evidence without treating OpenRouter as model-scope authority. Enable online ingestion explicitly with `JITTOR_OPENROUTER_BENCHMARKS=1`; it is off by default. OpenRouter model metadata and operational ordering are public; benchmark-index ingestion additionally uses `OPENROUTER_API_KEY` from the supervised service environment without retaining it. Snapshots preserve the upstream publisher, normalized model identities, immutable retrieval revisions, source URLs, confidence, license terms, and explicit freshness deadlines. A malformed or oversized refresh leaves the last complete snapshot visible and records only a payload-safe failure state.
+Jittor can ingest bounded OpenRouter model metadata, p50 latency/throughput ordering, versioned Artificial Analysis benchmark indices, and Design Arena Elo rankings as provenance-bearing evidence without treating OpenRouter as model-scope authority. Enable online ingestion explicitly with `JITTOR_OPENROUTER_BENCHMARKS=1`; it is off by default. OpenRouter model metadata and operational ordering are public; benchmark-index and Design Arena ingestion additionally use `OPENROUTER_API_KEY` from the supervised service environment without retaining it. Snapshots preserve the upstream publisher, normalized model identities, immutable retrieval revisions, source URLs, confidence, license terms, and explicit freshness deadlines. A malformed or oversized refresh leaves the last complete snapshot visible and records only a payload-safe failure state.
+
+Design Arena rates models across dozens of arena/category pairs (music, video, text-to-speech, ASCII art, ...); Jittor ingests only the bounded allowlist of categories (`codecategories`, `website`, `uicomponent`, `dataviz`, `svg`) that measure frontend/UI-generation skill relevant to routing a coding agent, tagged into one `design` domain distinct from `coding`. A model with no OpenRouter-reachable identity (proprietary platforms, image/video generators) is skipped rather than fabricated into unroutable evidence.
 
 Use the authenticated CLI channels independently:
 

package/extension/src/benchmark-tui.ts

@@ -85,7 +85,7 @@ export async function showBenchmarkPanel(
 				context: MODEL_RANKING_DEFAULT_CONTEXT_WEIGHT,
 				reliability: MODEL_RANKING_DEFAULT_RELIABILITY_WEIGHT,
 			},
-			sourceIds: ["openrouter-models", "openrouter-artificial-analysis"],
+			sourceIds: ["openrouter-models", "openrouter-artificial-analysis", "openrouter-design-arena"],
 		}) as ModelRankingResult;
 		if (ctx.mode !== "tui") {
 			ctx.ui.notify(renderBenchmarkView(result, currentIdentity, 100, { fg: (_color, text) => text, bold: (text) => text }).join("\n"), "info");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@danypops/jittor",
-  "version": "0.7.0",
+  "version": "0.8.0",
   "description": "Just-in-Time Token Optimizing Router for Pi",
   "type": "module",
   "keywords": ["pi-package", "llm-router", "token-budget"],

package/src/adapters/openrouter-design-arena-source.ts

@@ -0,0 +1,105 @@
+import {
+	BENCHMARK_MAX_MODELS_PER_SOURCE,
+	BENCHMARK_REFRESH_INTERVAL_MS,
+	BENCHMARK_SOURCE_MAX_RESPONSE_BYTES,
+} from "../constants.ts";
+import { normalizeModelIdentity, validateBenchmarkObservation, type BenchmarkObservation, type BenchmarkSourceSnapshot } from "../domain/benchmark.ts";
+import type { BenchmarkSource } from "../ports/benchmark-source.ts";
+import { contractRecord } from "../providers/openrouter-contracts.ts";
+import type { OpenRouterBenchmarkTransport } from "./openrouter-benchmark-source.ts";
+
+const SOURCE_ID = "openrouter-design-arena";
+const BASE_URL = `https://openrouter.ai/api/v1/benchmarks?source=design-arena&max_results=${BENCHMARK_MAX_MODELS_PER_SOURCE}`;
+
+/**
+ * Design Arena publishes rankings across dozens of arena/category pairs (music, video, TTS,
+ * ASCII art, ...) -- most of them have nothing to do with a text coding agent's model choice.
+ * This is a deliberate, bounded allowlist of the categories that measure frontend/UI-generation
+ * skill, the one facet of "design" quality relevant to routing a coding agent. Extending this
+ * list is a data curation decision (does the category's skill actually predict something a
+ * coding agent needs), not a mechanical one.
+ */
+const RELEVANT_CATEGORIES = ["codecategories", "website", "uicomponent", "dataviz", "svg"] as const;
+
+function requiredText(value: unknown, name: string): string {
+	if (typeof value !== "string" || value.length === 0 || value.length > 500) throw new Error(`OpenRouter Design Arena benchmark ${name} schema changed`);
+	return value;
+}
+
+function requiredNumber(value: unknown, name: string): number {
+	if (typeof value !== "number" || !Number.isFinite(value)) throw new Error(`OpenRouter Design Arena benchmark ${name} schema changed`);
+	return value;
+}
+
+function identityFromOpenRouterId(openRouterId: string): { provider: string; model: string } {
+	const separator = openRouterId.indexOf("/");
+	if (separator <= 0 || separator === openRouterId.length - 1) throw new Error("OpenRouter Design Arena model identity schema changed");
+	return { provider: openRouterId.slice(0, separator), model: openRouterId.slice(separator + 1) };
+}
+
+export class OpenRouterDesignArenaSource implements BenchmarkSource {
+	readonly id = SOURCE_ID;
+
+	constructor(
+		private readonly apiKey: string,
+		private readonly transport: OpenRouterBenchmarkTransport = fetch,
+		private readonly clock: () => number = Date.now,
+	) {
+		if (apiKey.length === 0) throw new Error("OpenRouter API key is required for Design Arena benchmark evidence");
+	}
+
+	async fetch(): Promise<BenchmarkSourceSnapshot> {
+		const retrievedAt = this.clock();
+		if (!Number.isSafeInteger(retrievedAt) || retrievedAt <= 0) throw new Error("benchmark retrieval time is invalid");
+		const perCategory = await Promise.all(RELEVANT_CATEGORIES.map((category) => this.fetchCategory(category, retrievedAt)));
+		const revisions = new Set(perCategory.map((page) => page.revision));
+		const observations = perCategory.flatMap((page) => page.observations);
+		return { sourceId: this.id, snapshotId: `${this.id}:${[...revisions].sort().join(",")}`, retrievedAt, observations };
+	}
+
+	private async fetchCategory(category: string, retrievedAt: number): Promise<{ revision: string; observations: BenchmarkObservation[] }> {
+		const url = `${BASE_URL}&task_type=${category}`;
+		const response = await this.transport(new Request(url, { headers: { authorization: `Bearer ${this.apiKey}` } }));
+		if (!response.ok) throw new Error(`OpenRouter Design Arena benchmarks failed with HTTP ${response.status}`);
+		const text = await response.text();
+		if (new TextEncoder().encode(text).byteLength > BENCHMARK_SOURCE_MAX_RESPONSE_BYTES) throw new Error("OpenRouter Design Arena benchmark response exceeds the size limit");
+		let payload: unknown;
+		try { payload = JSON.parse(text); } catch { throw new Error("OpenRouter Design Arena benchmark response is not valid JSON"); }
+		const root = contractRecord(payload, "benchmark response");
+		const meta = contractRecord(root["meta"], "benchmark metadata");
+		if (!Array.isArray(root["data"]) || root["data"].length > BENCHMARK_MAX_MODELS_PER_SOURCE) throw new Error("OpenRouter Design Arena benchmark result count is invalid");
+		if (requiredText(meta["source"], "source") !== "design-arena") throw new Error("OpenRouter Design Arena benchmark source schema changed");
+		const asOf = requiredText(meta["as_of"], "as-of date");
+		const publishedAt = Date.parse(asOf);
+		if (!Number.isSafeInteger(publishedAt) || publishedAt <= 0) throw new Error("OpenRouter Design Arena benchmark publication date schema changed");
+		const revision = `${category}:${asOf}`;
+		const observations = root["data"].flatMap((value): BenchmarkObservation[] => {
+			const row = contractRecord(value, "benchmark row");
+			if (requiredText(row["source"], "row source") !== "design-arena") throw new Error("OpenRouter Design Arena benchmark row source schema changed");
+			// A null open_router_id means the model isn't reachable through OpenRouter at all
+			// (proprietary platform, image/video generator, deprecated model) -- not evidence
+			// Jittor can ever route against, so the row is skipped rather than rejected.
+			const openRouterId = row["open_router_id"];
+			if (openRouterId === null) return [];
+			const { provider, model } = identityFromOpenRouterId(requiredText(openRouterId, "open_router_id"));
+			const identity = normalizeModelIdentity(provider, model, [`openrouter/${provider}/${model}`]);
+			const provenance = {
+				sourceId: SOURCE_ID,
+				sourceType: "independent" as const,
+				publisher: "Design Arena via OpenRouter",
+				url: BASE_URL,
+				revision,
+				publishedAt,
+				retrievedAt,
+				freshUntil: retrievedAt + BENCHMARK_REFRESH_INTERVAL_MS,
+				license: "OpenRouter API terms; Design Arena terms apply",
+				confidence: 0.7,
+			};
+			const methodology = { basis: "Design Arena Elo rating via OpenRouter", category, asOf };
+			return [validateBenchmarkObservation({
+				model: identity, dimension: "quality-design", value: requiredNumber(row["elo"], "elo rating"), unit: "elo", provenance, methodology,
+			})];
+		});
+		return { revision, observations };
+	}
+}

package/src/daemon.ts

@@ -5,6 +5,7 @@ import { SQLiteMetricStore } from "./adapters/sqlite-metric-store.ts";
 import { MetricBenchmarkStore } from "./adapters/metric-benchmark-store.ts";
 import { OpenRouterBenchmarkIndexSource } from "./adapters/openrouter-benchmark-index-source.ts";
 import { OpenRouterBenchmarkSource } from "./adapters/openrouter-benchmark-source.ts";
+import { OpenRouterDesignArenaSource } from "./adapters/openrouter-design-arena-source.ts";
 import { openJittorDb } from "./db.ts";
 import { BenchmarkCatalog } from "./domain/benchmark.ts";
 import { EvidenceModelRanker } from "./domain/model-ranking-service.ts";
@@ -28,7 +29,10 @@ export function reportMaintenanceFailure(event: string, error: unknown): void {
 export function benchmarkSourcesFromEnvironment(env: Record<string, string | undefined> = process.env): BenchmarkSource[] {
 	if (env["JITTOR_OPENROUTER_BENCHMARKS"] !== "1") return [];
 	const sources: BenchmarkSource[] = [new OpenRouterBenchmarkSource()];
-	if (env["OPENROUTER_API_KEY"]) sources.push(new OpenRouterBenchmarkIndexSource(env["OPENROUTER_API_KEY"]));
+	if (env["OPENROUTER_API_KEY"]) {
+		sources.push(new OpenRouterBenchmarkIndexSource(env["OPENROUTER_API_KEY"]));
+		sources.push(new OpenRouterDesignArenaSource(env["OPENROUTER_API_KEY"]));
+	}
 	return sources;
 }
 

package/src/domain/metric.ts

@@ -1,6 +1,6 @@
 import { METRIC_ATTRIBUTES_MAX_DEPTH, METRIC_ATTRIBUTES_MAX_SERIALIZED_CHARACTERS, METRIC_IDENTITY_MAX_CHARACTERS } from "../constants.ts";
 
-export const METRIC_UNITS = ["ratio", "usd", "tokens", "tokens-per-second", "requests", "milliseconds", "count"] as const;
+export const METRIC_UNITS = ["ratio", "usd", "tokens", "tokens-per-second", "requests", "milliseconds", "count", "elo"] as const;
 export type MetricUnit = typeof METRIC_UNITS[number];
 
 export interface MetricObservation {

package/src/domain/model-observation.ts

@@ -16,7 +16,7 @@ import type { MetricObservation, MetricUnit, StoredMetricObservation } from "./m
  * usage carries no distinguishing signal for that axis; a run can score coding on domain and
  * research on type simultaneously (e.g. reading a file, then searching the web in one turn).
  */
-export const TASK_DOMAINS = ["coding", "general"] as const;
+export const TASK_DOMAINS = ["coding", "design", "general"] as const;
 export type ModelTaskDomain = typeof TASK_DOMAINS[number];
 export const TASK_TYPES = ["research", "planning", "general"] as const;
 export type ModelTaskType = typeof TASK_TYPES[number];