@danypops/jittor 0.5.0 → 0.6.0

package/src/daemon.ts

@@ -1,9 +1,15 @@
 import { LOOPBACK_HOST, MAINTENANCE_INTERVAL_MS, TELEMETRY_POLL_INTERVAL_MS } from "./constants.ts";
 import { DEFAULT_POLICY, UNCONFIGURED_ROUTE } from "./config.ts";
 import { SQLiteMetricStore } from "./adapters/sqlite-metric-store.ts";
+import { MetricBenchmarkStore } from "./adapters/metric-benchmark-store.ts";
+import { OpenRouterBenchmarkIndexSource } from "./adapters/openrouter-benchmark-index-source.ts";
+import { OpenRouterBenchmarkSource } from "./adapters/openrouter-benchmark-source.ts";
 import { openJittorDb } from "./db.ts";
+import { BenchmarkCatalog } from "./domain/benchmark.ts";
+import { EvidenceModelRanker } from "./domain/model-ranking-service.ts";
 import { createApp, JittorService } from "./service.ts";
 import { JittorRouter } from "./router.ts";
+import type { BenchmarkSource } from "./ports/benchmark-source.ts";
 import type { TelemetrySource } from "./ports/telemetry-source.ts";
 import { CodexTelemetrySource, OpenRouterTelemetrySource } from "./providers/telemetry-sources.ts";
 import {
@@ -20,6 +26,13 @@ export interface RunningDaemon {
 	stop(): Promise<void>;
 }
 
+export function benchmarkSourcesFromEnvironment(env: Record<string, string | undefined> = process.env): BenchmarkSource[] {
+	if (env["JITTOR_OPENROUTER_BENCHMARKS"] !== "1") return [];
+	const sources: BenchmarkSource[] = [new OpenRouterBenchmarkSource()];
+	if (env["OPENROUTER_API_KEY"]) sources.push(new OpenRouterBenchmarkIndexSource(env["OPENROUTER_API_KEY"]));
+	return sources;
+}
+
 export function telemetrySourcesFromEnvironment(env: Record<string, string | undefined> = process.env): TelemetrySource[] {
 	const sources: TelemetrySource[] = [];
 	const codexAuthFile = env["JITTOR_CODEX_AUTH_FILE"];
@@ -36,6 +49,10 @@ export function startDaemon(
 	const token = ensureAuthToken(paths);
 	const metrics = new SQLiteMetricStore(openJittorDb(paths.database));
 	const sources = telemetrySourcesFromEnvironment(env);
+	const benchmarkSources = benchmarkSourcesFromEnvironment(env);
+	const benchmarkStore = new MetricBenchmarkStore(metrics);
+	const benchmarks = new BenchmarkCatalog(benchmarkStore, benchmarkSources);
+	const modelRanker = new EvidenceModelRanker(benchmarkStore, metrics);
 	const router = new JittorRouter({
 		metrics,
 		sources,
@@ -43,7 +60,7 @@ export function startDaemon(
 		routes: [],
 		currentRoute: UNCONFIGURED_ROUTE,
 	});
-	const service = new JittorService(metrics, router);
+	const service = new JittorService(metrics, router, benchmarks, modelRanker);
 	const app = createApp({ service, token });
 	const server = Bun.serve({
 		hostname: LOOPBACK_HOST,
@@ -57,9 +74,13 @@ export function startDaemon(
 		throw new Error("Jittor daemon failed to bind a loopback port");
 	}
 	writeDaemonHandle(paths, { host: LOOPBACK_HOST, port, pid: process.pid });
-	const maintenance = setInterval(() => { void service.execute("service.checkpoint", {}); }, MAINTENANCE_INTERVAL_MS);
+	const maintenance = setInterval(() => {
+		void service.execute("service.checkpoint", {});
+		void benchmarks.refresh();
+	}, MAINTENANCE_INTERVAL_MS);
 	const poll = setInterval(() => { void router.poll(); }, TELEMETRY_POLL_INTERVAL_MS);
 	if (sources.length > 0) void router.poll();
+	if (benchmarkSources.length > 0) void benchmarks.refresh();
 	let stopped = false;
 	return {
 		host: LOOPBACK_HOST,

package/src/domain/benchmark.ts

@@ -0,0 +1,264 @@
+import {
+	BENCHMARK_DEFAULT_QUERY_LIMIT,
+	BENCHMARK_IDENTITY_MAX_CHARACTERS,
+	BENCHMARK_MAX_OBSERVATIONS_PER_SNAPSHOT,
+	BENCHMARK_MAX_QUERY_LIMIT,
+	BENCHMARK_MAX_TEXT_CHARACTERS,
+	BENCHMARK_REFRESH_INTERVAL_MS,
+} from "../constants.ts";
+import { METRIC_UNITS, type MetricUnit } from "./metric.ts";
+import type { BenchmarkController } from "../ports/benchmark-controller.ts";
+import type { BenchmarkSource as BenchmarkSourcePort } from "../ports/benchmark-source.ts";
+import type { BenchmarkStore } from "../ports/benchmark-store.ts";
+
+export type BenchmarkSourceType = "creator" | "marketplace" | "independent" | "operational" | "preference" | "local";
+
+export interface ModelIdentity {
+	provider: string;
+	model: string;
+	version: string | null;
+	canonical: string;
+	aliases: string[];
+}
+
+export interface BenchmarkProvenance {
+	sourceId: string;
+	sourceType: BenchmarkSourceType;
+	publisher: string;
+	url: string;
+	revision: string;
+	publishedAt: number | null;
+	retrievedAt: number;
+	freshUntil: number;
+	license: string;
+	confidence: number;
+}
+
+export interface BenchmarkObservation {
+	model: ModelIdentity;
+	dimension: string;
+	value: number;
+	unit: MetricUnit;
+	provenance: BenchmarkProvenance;
+	methodology: Record<string, string | number | boolean | null | string[]>;
+}
+
+export interface BenchmarkSourceSnapshot {
+	sourceId: string;
+	snapshotId: string;
+	retrievedAt: number;
+	observations: BenchmarkObservation[];
+}
+
+export interface BenchmarkSnapshot extends BenchmarkSourceSnapshot {
+	publishedAt: number;
+}
+
+export interface BenchmarkSourceStatus {
+	id: string;
+	ok: boolean | null;
+	hasEvidence: boolean;
+	lastAttemptAt: number | null;
+	lastSuccessAt: number | null;
+	observations: number;
+	error?: "source refresh failed";
+}
+
+export interface BenchmarkRefreshResult {
+	observedAt: number;
+	sources: BenchmarkSourceStatus[];
+}
+
+export interface BenchmarkQuery {
+	sourceId: string;
+	model?: string;
+	dimension?: string;
+	limit?: number;
+}
+
+export interface BenchmarkQueryResult extends BenchmarkSnapshot {
+	completeness: "complete" | "truncated";
+	freshness: "fresh" | "stale";
+	freshUntil: number;
+}
+
+export interface BenchmarkCatalogOptions {
+	clock?: () => number;
+	refreshIntervalMs?: number;
+}
+
+export type BenchmarkSource = BenchmarkSourcePort;
+
+const VERSION_SUFFIX = /-(\d{4}-\d{2}-\d{2})$/;
+const SOURCE_TYPES = new Set<BenchmarkSourceType>(["creator", "marketplace", "independent", "operational", "preference", "local"]);
+
+function boundedText(value: unknown, name: string, maximum = BENCHMARK_MAX_TEXT_CHARACTERS): string {
+	if (typeof value !== "string" || value.trim().length === 0) throw new Error(`${name} is required`);
+	const normalized = value.trim();
+	if (normalized.length > maximum) throw new Error(`${name} exceeds the length limit`);
+	if (/\p{Cc}/u.test(normalized)) throw new Error(`${name} contains control characters`);
+	return normalized;
+}
+
+function identityPart(value: string, name: string, allowPath = false): string {
+	const normalized = boundedText(value, name, BENCHMARK_IDENTITY_MAX_CHARACTERS).toLowerCase();
+	const pattern = allowPath ? /^[a-z0-9][a-z0-9._:+/-]*$/ : /^[a-z0-9][a-z0-9._:+-]*$/;
+	if (!pattern.test(normalized) || normalized.includes("//")) throw new Error(`${name} is invalid`);
+	return normalized;
+}
+
+export function normalizeModelIdentity(provider: string, model: string, aliases: string[] = []): ModelIdentity {
+	const normalizedProvider = identityPart(provider, "provider");
+	const normalizedModel = identityPart(model, "model", true);
+	const version = VERSION_SUFFIX.exec(normalizedModel)?.[1] ?? null;
+	const canonical = `${normalizedProvider}/${normalizedModel}`;
+	const normalizedAliases = [...new Set(aliases.map((alias) => boundedText(alias, "alias", BENCHMARK_IDENTITY_MAX_CHARACTERS).toLowerCase()))]
+		.filter((alias) => alias !== canonical)
+		.sort();
+	return { provider: normalizedProvider, model: normalizedModel, version, canonical, aliases: normalizedAliases };
+}
+
+function validateIdentity(value: unknown): ModelIdentity {
+	if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error("model identity is required");
+	const input = value as Record<string, unknown>;
+	if (!Array.isArray(input["aliases"]) || !input["aliases"].every((alias) => typeof alias === "string")) throw new Error("model aliases are invalid");
+	const normalized = normalizeModelIdentity(String(input["provider"] ?? ""), String(input["model"] ?? ""), input["aliases"] as string[]);
+	if (input["canonical"] !== normalized.canonical || input["version"] !== normalized.version) throw new Error("model identity is not normalized");
+	return normalized;
+}
+
+function validateTimestamp(value: unknown, name: string, nullable = false): number | null {
+	if (nullable && value === null) return null;
+	if (!Number.isSafeInteger(value) || (value as number) <= 0) throw new Error(`${name} must be a positive integer timestamp`);
+	return value as number;
+}
+
+function validateProvenance(value: unknown): BenchmarkProvenance {
+	if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error("provenance is required");
+	const input = value as Record<string, unknown>;
+	const sourceId = identityPart(String(input["sourceId"] ?? ""), "source id");
+	if (!SOURCE_TYPES.has(input["sourceType"] as BenchmarkSourceType)) throw new Error("source type is invalid");
+	let url: URL;
+	try { url = new URL(boundedText(input["url"], "source URL")); } catch { throw new Error("source URL is invalid"); }
+	if (url.protocol !== "https:") throw new Error("source URL must use HTTPS");
+	const confidence = input["confidence"];
+	if (typeof confidence !== "number" || !Number.isFinite(confidence) || confidence < 0 || confidence > 1) throw new Error("confidence must be between zero and one");
+	const retrievedAt = validateTimestamp(input["retrievedAt"], "retrieval time") as number;
+	const freshUntil = validateTimestamp(input["freshUntil"], "freshness deadline") as number;
+	if (freshUntil < retrievedAt) throw new Error("freshness deadline precedes retrieval");
+	return {
+		sourceId,
+		sourceType: input["sourceType"] as BenchmarkSourceType,
+		publisher: boundedText(input["publisher"], "publisher"),
+		url: url.toString(),
+		revision: boundedText(input["revision"], "revision"),
+		publishedAt: validateTimestamp(input["publishedAt"], "publication time", true),
+		retrievedAt,
+		freshUntil,
+		license: boundedText(input["license"], "license"),
+		confidence,
+	};
+}
+
+function validateMethodology(value: unknown): BenchmarkObservation["methodology"] {
+	if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error("methodology is required");
+	const serialized = JSON.stringify(value);
+	if (serialized.length > BENCHMARK_MAX_TEXT_CHARACTERS) throw new Error("methodology exceeds the size limit");
+	return structuredClone(value as BenchmarkObservation["methodology"]);
+}
+
+export function validateBenchmarkObservation(value: unknown): BenchmarkObservation {
+	if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error("benchmark observation must be an object");
+	const input = value as Record<string, unknown>;
+	const numericValue = input["value"];
+	if (typeof numericValue !== "number" || !Number.isFinite(numericValue)) throw new Error("benchmark value must be finite");
+	if (!METRIC_UNITS.includes(input["unit"] as MetricUnit)) throw new Error("benchmark unit is not supported");
+	return {
+		model: validateIdentity(input["model"]),
+		dimension: identityPart(String(input["dimension"] ?? ""), "dimension"),
+		value: numericValue,
+		unit: input["unit"] as MetricUnit,
+		provenance: validateProvenance(input["provenance"]),
+		methodology: validateMethodology(input["methodology"]),
+	};
+}
+
+function validateSourceSnapshot(value: BenchmarkSourceSnapshot, expectedSourceId: string): BenchmarkSourceSnapshot {
+	const sourceId = identityPart(value.sourceId, "source id");
+	if (sourceId !== expectedSourceId) throw new Error("source snapshot identity mismatch");
+	const snapshotId = boundedText(value.snapshotId, "snapshot id", BENCHMARK_IDENTITY_MAX_CHARACTERS);
+	const retrievedAt = validateTimestamp(value.retrievedAt, "retrieval time") as number;
+	if (!Array.isArray(value.observations) || value.observations.length > BENCHMARK_MAX_OBSERVATIONS_PER_SNAPSHOT) throw new Error("source snapshot exceeds the observation limit");
+	const observations = value.observations.map(validateBenchmarkObservation);
+	if (observations.some((observation) => observation.provenance.sourceId !== sourceId || observation.provenance.retrievedAt !== retrievedAt)) {
+		throw new Error("source snapshot provenance mismatch");
+	}
+	return { sourceId, snapshotId, retrievedAt, observations };
+}
+
+export class BenchmarkCatalog implements BenchmarkController {
+	private readonly clock: () => number;
+	private readonly refreshIntervalMs: number;
+	private readonly states = new Map<string, BenchmarkSourceStatus>();
+
+	constructor(
+		private readonly store: BenchmarkStore,
+		private readonly sources: BenchmarkSource[],
+		options: BenchmarkCatalogOptions = {},
+	) {
+		this.clock = options.clock ?? Date.now;
+		this.refreshIntervalMs = options.refreshIntervalMs ?? BENCHMARK_REFRESH_INTERVAL_MS;
+		for (const source of sources) {
+			const evidence = store.latest(source.id);
+			this.states.set(source.id, {
+				id: source.id,
+				ok: null,
+				hasEvidence: evidence !== null,
+				lastAttemptAt: null,
+				lastSuccessAt: evidence?.retrievedAt ?? null,
+				observations: evidence?.observations.length ?? 0,
+			});
+		}
+	}
+
+	async refresh(force = false): Promise<BenchmarkRefreshResult> {
+		const now = this.clock();
+		await Promise.all(this.sources.map(async (source) => {
+			const prior = this.states.get(source.id)!;
+			if (!force && prior.lastAttemptAt !== null && now - prior.lastAttemptAt < this.refreshIntervalMs) return;
+			this.states.set(source.id, { ...prior, lastAttemptAt: now });
+			try {
+				const snapshot = validateSourceSnapshot(await source.fetch(), source.id);
+				const published = this.store.publish(snapshot.sourceId, snapshot.snapshotId, snapshot.observations);
+				this.states.set(source.id, { id: source.id, ok: true, hasEvidence: true, lastAttemptAt: now, lastSuccessAt: published.retrievedAt, observations: published.observations.length });
+			} catch {
+				const evidence = this.store.latest(source.id);
+				this.states.set(source.id, { id: source.id, ok: false, hasEvidence: evidence !== null, lastAttemptAt: now, lastSuccessAt: evidence?.retrievedAt ?? prior.lastSuccessAt, observations: evidence?.observations.length ?? prior.observations, error: "source refresh failed" });
+			}
+		}));
+		return { observedAt: now, sources: this.status().sources };
+	}
+
+	status(): BenchmarkRefreshResult {
+		return { observedAt: this.clock(), sources: this.sources.map((source) => structuredClone(this.states.get(source.id)!)) };
+	}
+
+	query(input: BenchmarkQuery): BenchmarkQueryResult {
+		const sourceId = identityPart(input.sourceId, "source id");
+		const snapshot = this.store.latest(sourceId);
+		if (!snapshot) throw new Error("benchmark evidence is not available for the source");
+		const requestedLimit = Number.isFinite(input.limit) ? Math.floor(input.limit!) : BENCHMARK_DEFAULT_QUERY_LIMIT;
+		const limit = Math.max(1, Math.min(BENCHMARK_MAX_QUERY_LIMIT, requestedLimit));
+		const model = input.model?.trim().toLowerCase();
+		const dimension = input.dimension?.trim().toLowerCase();
+		const matched = snapshot.observations.filter((observation) => (!model || observation.model.canonical === model || observation.model.aliases.includes(model)) && (!dimension || observation.dimension === dimension));
+		const freshUntil = Math.min(...snapshot.observations.map((observation) => observation.provenance.freshUntil));
+		return {
+			...structuredClone(snapshot),
+			observations: structuredClone(matched.slice(0, limit)),
+			completeness: matched.length > limit ? "truncated" : "complete",
+			freshness: this.clock() <= freshUntil ? "fresh" : "stale",
+			freshUntil,
+		};
+	}
+}

package/src/domain/context-telemetry.ts

@@ -1,4 +1,6 @@
 import {
+	COMPACTION_DURATION_ESTIMATE_MAX_SAMPLES,
+	COMPACTION_DURATION_ESTIMATE_MIN_SAMPLES,
 	CONTEXT_OBSERVATION_MAX_AGE_MS,
 	CONTEXT_OBSERVATION_MAX_CHARACTERS,
 	MILLISECONDS_PER_HOUR,
@@ -278,3 +280,32 @@ export function assessContextTelemetry(
 		},
 	};
 }
+
+export interface CompactionDurationEstimate {
+	ms: number | null;
+	confidence: "cold-start" | "learned";
+	sampleSize: number;
+	observedAt: number;
+}
+
+/**
+ * Learns a bounded duration estimate from the most recent completed Pi compactions so the drain
+ * animation can show an approximate time-to-completion instead of a fixed-rate guess. Reads only
+ * the numeric `compaction-duration` value already recorded content-free by CompactionTelemetry
+ * (never transcript content, credentials, or attributes) and is bounded to the caller-provided
+ * rows — callers must query with `limit: COMPACTION_DURATION_ESTIMATE_MAX_SAMPLES` so retention is
+ * bounded at the query layer, not just here. Below COMPACTION_DURATION_ESTIMATE_MIN_SAMPLES samples
+ * the estimate stays explicit cold-start uncertainty rather than a guess from too little evidence.
+ */
+export function estimateCompactionDuration(compactions: StoredMetricObservation[], now = Date.now()): CompactionDurationEstimate {
+	const durations = compactions
+		.filter((row) => row.source === "pi-context" && row.scope === "compaction" && row.metric === "compaction-duration")
+		.sort((left, right) => right.observedAt - left.observedAt || right.id - left.id)
+		.slice(0, COMPACTION_DURATION_ESTIMATE_MAX_SAMPLES)
+		.flatMap((row) => typeof row.value === "number" && Number.isFinite(row.value) && row.value >= 0 ? [row.value] : []);
+	if (durations.length < COMPACTION_DURATION_ESTIMATE_MIN_SAMPLES) {
+		return { ms: null, confidence: "cold-start", sampleSize: durations.length, observedAt: now };
+	}
+	const median = percentile(durations, 0.5);
+	return { ms: median === null ? null : Math.round(median), confidence: "learned", sampleSize: durations.length, observedAt: now };
+}

package/src/domain/metric.ts

@@ -1,4 +1,6 @@
-export const METRIC_UNITS = ["ratio", "usd", "tokens", "requests", "milliseconds", "count"] as const;
+import { METRIC_ATTRIBUTES_MAX_DEPTH, METRIC_ATTRIBUTES_MAX_SERIALIZED_CHARACTERS, METRIC_IDENTITY_MAX_CHARACTERS } from "../constants.ts";
+
+export const METRIC_UNITS = ["ratio", "usd", "tokens", "tokens-per-second", "requests", "milliseconds", "count"] as const;
 export type MetricUnit = typeof METRIC_UNITS[number];
 
 export interface MetricObservation {
@@ -26,11 +28,52 @@ export interface MetricQuery {
 	order?: "asc" | "desc";
 }
 
+const SENSITIVE_ATTRIBUTE_KEYS = new Set([
+	"accesstoken",
+	"refreshtoken",
+	"authorization",
+	"apikey",
+	"secret",
+	"password",
+	"cookie",
+	"credential",
+	"otpseed",
+]);
+
+function assertCredentialSafeAttributes(value: unknown, depth = 0): void {
+	if (depth > METRIC_ATTRIBUTES_MAX_DEPTH) throw new Error("attributes exceed the nesting depth limit");
+	if (Array.isArray(value)) {
+		for (const item of value) assertCredentialSafeAttributes(item, depth + 1);
+		return;
+	}
+	if (typeof value !== "object" || value === null) return;
+	for (const [key, nested] of Object.entries(value)) {
+		const normalized = key.toLowerCase().replace(/[^a-z0-9]/g, "");
+		if (SENSITIVE_ATTRIBUTE_KEYS.has(normalized)) throw new Error("attributes contain a sensitive field");
+		assertCredentialSafeAttributes(nested, depth + 1);
+	}
+}
+
+function validateAttributes(value: unknown): Record<string, unknown> {
+	if (value === undefined) return {};
+	if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error("attributes must be an object");
+	let serialized: string;
+	try {
+		serialized = JSON.stringify(value);
+	} catch {
+		throw new Error("attributes must be JSON serializable");
+	}
+	if (serialized.length > METRIC_ATTRIBUTES_MAX_SERIALIZED_CHARACTERS) throw new Error("attributes exceed the serialized size limit");
+	assertCredentialSafeAttributes(value);
+	return value as Record<string, unknown>;
+}
+
 export function validateMetricObservation(value: unknown): MetricObservation {
 	if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error("metric observation must be an object");
 	const input = value as Record<string, unknown>;
 	for (const key of ["source", "scope", "metric", "unit"] as const) {
 		if (typeof input[key] !== "string" || input[key].trim().length === 0) throw new Error(`${key} is required`);
+		if (input[key].length > METRIC_IDENTITY_MAX_CHARACTERS) throw new Error(`${key} exceeds the length limit`);
 	}
 	if (!METRIC_UNITS.includes(input["unit"] as MetricUnit)) throw new Error("unit is not supported");
 	if (input["value"] !== null && (typeof input["value"] !== "number" || !Number.isFinite(input["value"]))) {
@@ -39,9 +82,7 @@ export function validateMetricObservation(value: unknown): MetricObservation {
 	if (typeof input["observedAt"] !== "number" || !Number.isSafeInteger(input["observedAt"]) || input["observedAt"] < 0) {
 		throw new Error("observedAt must be a non-negative integer timestamp");
 	}
-	if (input["attributes"] !== undefined && (typeof input["attributes"] !== "object" || input["attributes"] === null || Array.isArray(input["attributes"]))) {
-		throw new Error("attributes must be an object");
-	}
+	const attributes = validateAttributes(input["attributes"]);
 	return {
 		source: input["source"] as string,
 		scope: input["scope"] as string,
@@ -49,6 +90,6 @@ export function validateMetricObservation(value: unknown): MetricObservation {
 		value: input["value"] as number | null,
 		unit: input["unit"] as MetricUnit,
 		observedAt: input["observedAt"] as number,
-		attributes: (input["attributes"] as Record<string, unknown> | undefined) ?? {},
+		attributes,
 	};
 }

package/src/domain/model-observation.ts

@@ -0,0 +1,181 @@
+import {
+	MODEL_AGGREGATE_MAX_GROUPS,
+	MODEL_AGGREGATE_MAX_ROWS,
+	MODEL_OBSERVATION_FRESH_MS,
+	MODEL_OBSERVATION_IDENTITY_MAX_CHARACTERS,
+} from "../constants.ts";
+import { normalizeModelIdentity } from "./benchmark.ts";
+import type { MetricObservation, MetricUnit, StoredMetricObservation } from "./metric.ts";
+
+export const TASK_CLASSES = ["coding", "research", "planning", "general"] as const;
+export type ModelTaskClass = typeof TASK_CLASSES[number];
+export type ExplicitOutcome = "accepted" | "rejected" | "unknown";
+
+export interface ModelRunObservation {
+	runId: string;
+	provider: string;
+	model: string;
+	thinking: string;
+	taskClass: ModelTaskClass;
+	startedAt: number;
+	firstTokenAt: number | null;
+	completedAt: number;
+	inputTokens: number;
+	outputTokens: number;
+	cacheReadTokens: number;
+	cacheWriteTokens: number;
+	costUsd: number;
+	providerResponses: number;
+	toolCalls: number;
+	toolFailures: number;
+	stopReason: "stop" | "length" | "toolUse" | "error" | "aborted" | "unknown";
+	explicitOutcome: ExplicitOutcome;
+}
+
+export interface ModelMetricAggregate {
+	provider: string;
+	model: string;
+	thinking: string;
+	taskClass: ModelTaskClass;
+	dimension: string;
+	unit: MetricUnit;
+	sampleSize: number;
+	median: number;
+	p90: number;
+	medianAbsoluteDeviation: number;
+	latestAt: number;
+	freshness: "fresh" | "stale";
+	confidence: number;
+}
+
+export interface ModelAggregateOptions {
+	now?: number;
+	freshForMs?: number;
+}
+
+const ALLOWED_FIELDS = new Set<keyof ModelRunObservation>([
+	"runId", "provider", "model", "thinking", "taskClass", "startedAt", "firstTokenAt", "completedAt",
+	"inputTokens", "outputTokens", "cacheReadTokens", "cacheWriteTokens", "costUsd", "providerResponses",
+	"toolCalls", "toolFailures", "stopReason", "explicitOutcome",
+]);
+const STOP_REASONS = new Set<ModelRunObservation["stopReason"]>(["stop", "length", "toolUse", "error", "aborted", "unknown"]);
+const OUTCOMES = new Set<ExplicitOutcome>(["accepted", "rejected", "unknown"]);
+
+function text(value: unknown, name: string): string {
+	if (typeof value !== "string" || value.length === 0 || value.length > MODEL_OBSERVATION_IDENTITY_MAX_CHARACTERS || /\p{Cc}/u.test(value)) throw new Error(`${name} is invalid`);
+	return value;
+}
+
+function nonNegative(value: unknown, name: string, integer = false): number {
+	if (typeof value !== "number" || !Number.isFinite(value) || value < 0 || (integer && !Number.isSafeInteger(value))) throw new Error(`${name} is invalid`);
+	return value;
+}
+
+export function validateModelRunObservation(value: unknown): ModelRunObservation {
+	if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error("model run observation must be an object");
+	const input = value as Record<string, unknown>;
+	for (const key of Object.keys(input)) if (!ALLOWED_FIELDS.has(key as keyof ModelRunObservation)) throw new Error(`unsupported field: ${key}`);
+	const identity = normalizeModelIdentity(text(input["provider"], "provider"), text(input["model"], "model"));
+	const startedAt = nonNegative(input["startedAt"], "start time", true);
+	const completedAt = nonNegative(input["completedAt"], "completion time", true);
+	const firstTokenAt = input["firstTokenAt"] === null ? null : nonNegative(input["firstTokenAt"], "first-token time", true);
+	if (completedAt < startedAt || (firstTokenAt !== null && (firstTokenAt < startedAt || firstTokenAt > completedAt))) throw new Error("model run timestamps are not ordered");
+	if (!TASK_CLASSES.includes(input["taskClass"] as ModelTaskClass)) throw new Error("task class is invalid");
+	if (!STOP_REASONS.has(input["stopReason"] as ModelRunObservation["stopReason"])) throw new Error("stop reason is invalid");
+	if (!OUTCOMES.has(input["explicitOutcome"] as ExplicitOutcome)) throw new Error("explicit outcome is invalid");
+	const providerResponses = nonNegative(input["providerResponses"], "provider response count", true);
+	const toolCalls = nonNegative(input["toolCalls"], "tool call count", true);
+	const toolFailures = nonNegative(input["toolFailures"], "tool failure count", true);
+	if (providerResponses < 1 || toolFailures > toolCalls) throw new Error("model run counters are inconsistent");
+	return {
+		runId: text(input["runId"], "run id"), provider: identity.provider, model: identity.model,
+		thinking: text(input["thinking"], "thinking level"), taskClass: input["taskClass"] as ModelTaskClass,
+		startedAt, firstTokenAt, completedAt,
+		inputTokens: nonNegative(input["inputTokens"], "input tokens"),
+		outputTokens: nonNegative(input["outputTokens"], "output tokens"),
+		cacheReadTokens: nonNegative(input["cacheReadTokens"], "cache read tokens"),
+		cacheWriteTokens: nonNegative(input["cacheWriteTokens"], "cache write tokens"),
+		costUsd: nonNegative(input["costUsd"], "cost"), providerResponses, toolCalls, toolFailures,
+		stopReason: input["stopReason"] as ModelRunObservation["stopReason"], explicitOutcome: input["explicitOutcome"] as ExplicitOutcome,
+	};
+}
+
+export function classifyTaskFromTools(toolNames: string[]): ModelTaskClass {
+	const names = new Set(toolNames.slice(0, 100).map((name) => name.toLowerCase()));
+	if (["edit", "write", "read", "bash", "grep", "find", "ls"].some((name) => names.has(name))) return "coding";
+	if (["web_fetch", "web_search"].some((name) => names.has(name))) return "research";
+	if (["tasks", "papyrus_create", "papyrus_graph"].some((name) => names.has(name))) return "planning";
+	return "general";
+}
+
+export function modelRunMetrics(value: ModelRunObservation): MetricObservation[] {
+	const run = validateModelRunObservation(value);
+	const scope = `${run.provider}/${run.model}`;
+	const attributes = { provider: run.provider, model: run.model, thinking: run.thinking, taskClass: run.taskClass, runId: run.runId };
+	const metric = (name: string, amount: number, unit: MetricUnit): MetricObservation => ({ source: "local-model", scope, metric: name, value: amount, unit, observedAt: run.completedAt, attributes });
+	const wallMs = run.completedAt - run.startedAt;
+	const totalInput = run.inputTokens + run.cacheReadTokens;
+	const metrics: MetricObservation[] = [];
+	if (run.firstTokenAt !== null) metrics.push(metric("ttft", run.firstTokenAt - run.startedAt, "milliseconds"));
+	metrics.push(
+		metric("wall-latency", wallMs, "milliseconds"),
+		metric("output-throughput", wallMs === 0 ? 0 : run.outputTokens / (wallMs / 1_000), "tokens-per-second"),
+		metric("input-tokens", run.inputTokens, "tokens"),
+		metric("output-tokens", run.outputTokens, "tokens"),
+		metric("cache-read-tokens", run.cacheReadTokens, "tokens"),
+		metric("cache-write-tokens", run.cacheWriteTokens, "tokens"),
+		metric("cache-read-ratio", totalInput === 0 ? 0 : run.cacheReadTokens / totalInput, "ratio"),
+		metric("cost", run.costUsd, "usd"),
+		metric("provider-responses", run.providerResponses, "count"),
+		metric("retry-count", Math.max(0, run.providerResponses - 1), "count"),
+		metric("tool-calls", run.toolCalls, "count"),
+		metric("tool-failures", run.toolFailures, "count"),
+		metric("failure", run.stopReason === "error" ? 1 : 0, "ratio"),
+	);
+	if (run.explicitOutcome !== "unknown") metrics.push(metric("outcome-accepted", run.explicitOutcome === "accepted" ? 1 : 0, "ratio"));
+	return metrics;
+}
+
+function percentile(sorted: number[], fraction: number): number {
+	return sorted[Math.max(0, Math.ceil(sorted.length * fraction) - 1)]!;
+}
+
+function median(sorted: number[]): number {
+	const middle = Math.floor(sorted.length / 2);
+	return sorted.length % 2 === 0 ? (sorted[middle - 1]! + sorted[middle]!) / 2 : sorted[middle]!;
+}
+
+export function aggregateModelMetrics(input: StoredMetricObservation[], options: ModelAggregateOptions = {}): ModelMetricAggregate[] {
+	const now = options.now ?? Date.now();
+	const freshForMs = options.freshForMs ?? MODEL_OBSERVATION_FRESH_MS;
+	if (!Number.isSafeInteger(now) || now < 0 || !Number.isSafeInteger(freshForMs) || freshForMs <= 0) throw new Error("aggregate time bounds are invalid");
+	const groups = new Map<string, StoredMetricObservation[]>();
+	for (const row of input.slice(0, MODEL_AGGREGATE_MAX_ROWS)) {
+		if (row.source !== "local-model" || typeof row.value !== "number" || !Number.isFinite(row.value)) continue;
+		const provider = row.attributes["provider"];
+		const model = row.attributes["model"];
+		const thinking = row.attributes["thinking"];
+		const taskClass = row.attributes["taskClass"];
+		if (typeof provider !== "string" || typeof model !== "string" || typeof thinking !== "string" || !TASK_CLASSES.includes(taskClass as ModelTaskClass)) continue;
+		const key = JSON.stringify([provider, model, thinking, taskClass, row.metric, row.unit]);
+		if (!groups.has(key) && groups.size >= MODEL_AGGREGATE_MAX_GROUPS) continue;
+		const rows = groups.get(key) ?? [];
+		rows.push(row);
+		groups.set(key, rows);
+	}
+	return [...groups.entries()].map(([key, rows]) => {
+		const [provider, model, thinking, taskClass, dimension, unit] = JSON.parse(key) as [string, string, string, ModelTaskClass, string, MetricUnit];
+		const values = rows.map((row) => row.value as number).sort((left, right) => left - right);
+		const center = median(values);
+		const deviations = values.map((value) => Math.abs(value - center)).sort((left, right) => left - right);
+		const latestAt = Math.max(...rows.map((row) => row.observedAt));
+		const age = Math.max(0, now - latestAt);
+		const recency = Math.max(0, 1 - (age / freshForMs));
+		return {
+			provider, model, thinking, taskClass, dimension, unit,
+			sampleSize: values.length, median: center, p90: percentile(values, 0.9), medianAbsoluteDeviation: median(deviations), latestAt,
+			freshness: age <= freshForMs ? "fresh" as const : "stale" as const,
+			confidence: Math.min(1, Math.sqrt(values.length / 20)) * recency,
+		};
+	}).sort((left, right) => left.provider.localeCompare(right.provider) || left.model.localeCompare(right.model) || left.dimension.localeCompare(right.dimension));
+}

package/src/domain/model-ranking-service.ts

@@ -0,0 +1,40 @@
+import { MODEL_AGGREGATE_MAX_ROWS, MODEL_OBSERVATION_FRESH_MS, MODEL_RANKING_MAX_SOURCES } from "../constants.ts";
+import type { BenchmarkStore } from "../ports/benchmark-store.ts";
+import type { MetricStore } from "../ports/metric-store.ts";
+import { aggregateModelMetrics } from "./model-observation.ts";
+import { rankModelCandidates, type ModelCandidate, type ModelRankingResult, type ScopeAuthority, type UtilityWeights } from "./model-ranking.ts";
+import type { ModelTaskClass } from "./model-observation.ts";
+
+export interface ModelRecommendationInput {
+	candidates: ModelCandidate[];
+	scopeAuthority: ScopeAuthority;
+	taskClass: ModelTaskClass;
+	budgetPressure: number;
+	weights: UtilityWeights;
+	sourceIds: string[];
+}
+
+export interface ModelRanker {
+	rank(input: ModelRecommendationInput): ModelRankingResult;
+}
+
+export class EvidenceModelRanker implements ModelRanker {
+	constructor(
+		private readonly benchmarks: BenchmarkStore,
+		private readonly metrics: MetricStore,
+		private readonly clock: () => number = Date.now,
+	) {}
+
+	rank(input: ModelRecommendationInput): ModelRankingResult {
+		if (!Array.isArray(input.sourceIds) || input.sourceIds.length > MODEL_RANKING_MAX_SOURCES || !input.sourceIds.every((sourceId) => typeof sourceId === "string" && sourceId.length > 0 && sourceId.length <= 160)) {
+			throw new Error("benchmark source selection is invalid");
+		}
+		const sourceIds = [...new Set(input.sourceIds)];
+		const { sourceIds: _sourceIds, ...rankingInput } = input;
+		const externalEvidence = sourceIds.flatMap((sourceId) => this.benchmarks.latest(sourceId)?.observations ?? []);
+		const localRows = this.metrics.query({ source: "local-model", order: "desc", limit: MODEL_AGGREGATE_MAX_ROWS });
+		const now = this.clock();
+		const localEvidence = aggregateModelMetrics(localRows, { now, freshForMs: MODEL_OBSERVATION_FRESH_MS });
+		return rankModelCandidates({ ...rankingInput, externalEvidence, localEvidence, now });
+	}
+}