@contractspec/lib.provider-ranking 0.1.1

package/dist/in-memory-store.js

@@ -0,0 +1,93 @@
+// @bun
+// src/in-memory-store.ts
+class InMemoryProviderRankingStore {
+  benchmarkResults = new Map;
+  modelRankings = new Map;
+  ingestionRuns = new Map;
+  async upsertBenchmarkResult(result) {
+    this.benchmarkResults.set(result.id, result);
+  }
+  async getBenchmarkResult(id) {
+    return this.benchmarkResults.get(id) ?? null;
+  }
+  async listBenchmarkResults(query) {
+    let results = Array.from(this.benchmarkResults.values());
+    if (query.source) {
+      results = results.filter((r) => r.source === query.source);
+    }
+    if (query.modelId) {
+      results = results.filter((r) => r.modelId === query.modelId);
+    }
+    if (query.dimension) {
+      results = results.filter((r) => r.dimension === query.dimension);
+    }
+    if (query.providerKey) {
+      results = results.filter((r) => r.providerKey === query.providerKey);
+    }
+    const total = results.length;
+    const offset = query.offset ?? 0;
+    const limit = query.limit ?? 50;
+    results = results.slice(offset, offset + limit);
+    const nextOffset = offset + results.length < total ? offset + results.length : undefined;
+    return { results, total, nextOffset };
+  }
+  async upsertModelRanking(ranking) {
+    this.modelRankings.set(ranking.modelId, ranking);
+  }
+  async getModelRanking(modelId) {
+    return this.modelRankings.get(modelId) ?? null;
+  }
+  async listModelRankings(query) {
+    let rankings = Array.from(this.modelRankings.values());
+    if (query.providerKey) {
+      rankings = rankings.filter((r) => r.providerKey === query.providerKey);
+    }
+    if (query.dimension) {
+      const dim = query.dimension;
+      rankings.sort((a, b) => {
+        const scoreA = a.dimensionScores[dim]?.score ?? -1;
+        const scoreB = b.dimensionScores[dim]?.score ?? -1;
+        return scoreB - scoreA;
+      });
+    } else {
+      rankings.sort((a, b) => a.rank - b.rank);
+    }
+    const total = rankings.length;
+    const offset = query.offset ?? 0;
+    const limit = query.limit ?? 50;
+    rankings = rankings.slice(offset, offset + limit);
+    const nextOffset = offset + rankings.length < total ? offset + rankings.length : undefined;
+    return { rankings, total, nextOffset };
+  }
+  async getModelProfile(modelId) {
+    const ranking = this.modelRankings.get(modelId);
+    if (!ranking)
+      return null;
+    const benchmarkResults = Array.from(this.benchmarkResults.values()).filter((r) => r.modelId === modelId);
+    return {
+      modelId,
+      providerKey: ranking.providerKey,
+      displayName: modelId,
+      contextWindow: 0,
+      costPerMillion: null,
+      capabilities: [],
+      ranking,
+      benchmarkResults
+    };
+  }
+  async createIngestionRun(run) {
+    this.ingestionRuns.set(run.id, run);
+  }
+  async updateIngestionRun(id, update) {
+    const existing = this.ingestionRuns.get(id);
+    if (existing) {
+      this.ingestionRuns.set(id, { ...existing, ...update });
+    }
+  }
+  async getIngestionRun(id) {
+    return this.ingestionRuns.get(id) ?? null;
+  }
+}
+export {
+  InMemoryProviderRankingStore
+};

package/dist/index.d.ts

@@ -0,0 +1,4 @@
+export type { BenchmarkDimension, BenchmarkSource, BenchmarkResult, DimensionScore, ModelRanking, ModelProfile, BenchmarkResultQuery, BenchmarkResultListResult, RankingQuery, RankingListResult, IngestionRun, DimensionWeightConfig, } from './types';
+export { BENCHMARK_DIMENSIONS } from './types';
+export type { ProviderRankingStore } from './store';
+export { InMemoryProviderRankingStore } from './in-memory-store';

package/dist/index.js

@@ -0,0 +1,106 @@
+// @bun
+// src/in-memory-store.ts
+class InMemoryProviderRankingStore {
+  benchmarkResults = new Map;
+  modelRankings = new Map;
+  ingestionRuns = new Map;
+  async upsertBenchmarkResult(result) {
+    this.benchmarkResults.set(result.id, result);
+  }
+  async getBenchmarkResult(id) {
+    return this.benchmarkResults.get(id) ?? null;
+  }
+  async listBenchmarkResults(query) {
+    let results = Array.from(this.benchmarkResults.values());
+    if (query.source) {
+      results = results.filter((r) => r.source === query.source);
+    }
+    if (query.modelId) {
+      results = results.filter((r) => r.modelId === query.modelId);
+    }
+    if (query.dimension) {
+      results = results.filter((r) => r.dimension === query.dimension);
+    }
+    if (query.providerKey) {
+      results = results.filter((r) => r.providerKey === query.providerKey);
+    }
+    const total = results.length;
+    const offset = query.offset ?? 0;
+    const limit = query.limit ?? 50;
+    results = results.slice(offset, offset + limit);
+    const nextOffset = offset + results.length < total ? offset + results.length : undefined;
+    return { results, total, nextOffset };
+  }
+  async upsertModelRanking(ranking) {
+    this.modelRankings.set(ranking.modelId, ranking);
+  }
+  async getModelRanking(modelId) {
+    return this.modelRankings.get(modelId) ?? null;
+  }
+  async listModelRankings(query) {
+    let rankings = Array.from(this.modelRankings.values());
+    if (query.providerKey) {
+      rankings = rankings.filter((r) => r.providerKey === query.providerKey);
+    }
+    if (query.dimension) {
+      const dim = query.dimension;
+      rankings.sort((a, b) => {
+        const scoreA = a.dimensionScores[dim]?.score ?? -1;
+        const scoreB = b.dimensionScores[dim]?.score ?? -1;
+        return scoreB - scoreA;
+      });
+    } else {
+      rankings.sort((a, b) => a.rank - b.rank);
+    }
+    const total = rankings.length;
+    const offset = query.offset ?? 0;
+    const limit = query.limit ?? 50;
+    rankings = rankings.slice(offset, offset + limit);
+    const nextOffset = offset + rankings.length < total ? offset + rankings.length : undefined;
+    return { rankings, total, nextOffset };
+  }
+  async getModelProfile(modelId) {
+    const ranking = this.modelRankings.get(modelId);
+    if (!ranking)
+      return null;
+    const benchmarkResults = Array.from(this.benchmarkResults.values()).filter((r) => r.modelId === modelId);
+    return {
+      modelId,
+      providerKey: ranking.providerKey,
+      displayName: modelId,
+      contextWindow: 0,
+      costPerMillion: null,
+      capabilities: [],
+      ranking,
+      benchmarkResults
+    };
+  }
+  async createIngestionRun(run) {
+    this.ingestionRuns.set(run.id, run);
+  }
+  async updateIngestionRun(id, update) {
+    const existing = this.ingestionRuns.get(id);
+    if (existing) {
+      this.ingestionRuns.set(id, { ...existing, ...update });
+    }
+  }
+  async getIngestionRun(id) {
+    return this.ingestionRuns.get(id) ?? null;
+  }
+}
+
+// src/types.ts
+var BENCHMARK_DIMENSIONS = [
+  "coding",
+  "reasoning",
+  "agentic",
+  "cost",
+  "latency",
+  "context",
+  "safety",
+  "custom"
+];
+export {
+  InMemoryProviderRankingStore,
+  BENCHMARK_DIMENSIONS
+};

package/dist/ingesters/artificial-analysis.d.ts

@@ -0,0 +1,8 @@
+import type { BenchmarkIngester } from './types';
+/**
+ * Ingests Artificial Analysis data covering quality, speed, and cost.
+ *
+ * Produces results across multiple dimensions: reasoning (quality),
+ * latency (speed/TTFT), cost (pricing), and context (window size).
+ */
+export declare const artificialAnalysisIngester: BenchmarkIngester;

package/dist/ingesters/artificial-analysis.js

@@ -0,0 +1,150 @@
+// @bun
+// src/ingesters/fetch-utils.ts
+async function fetchWithRetry(url, options) {
+  const fetchFn = options?.fetch ?? globalThis.fetch;
+  const maxRetries = options?.maxRetries ?? 2;
+  const baseDelay = options?.baseDelayMs ?? 500;
+  let lastError;
+  for (let attempt = 0;attempt <= maxRetries; attempt++) {
+    try {
+      const response = await fetchFn(url);
+      if (response.ok)
+        return response;
+      if (response.status >= 500 && attempt < maxRetries) {
+        await sleep(baseDelay * Math.pow(2, attempt));
+        continue;
+      }
+      throw new Error(`Fetch failed: ${response.status} ${response.statusText} (${url})`);
+    } catch (error) {
+      lastError = error instanceof Error ? error : new Error(String(error));
+      if (attempt < maxRetries) {
+        await sleep(baseDelay * Math.pow(2, attempt));
+      }
+    }
+  }
+  throw lastError ?? new Error(`Fetch failed after ${maxRetries + 1} attempts: ${url}`);
+}
+function parseJsonSafe(text, label) {
+  try {
+    return JSON.parse(text);
+  } catch {
+    throw new Error(`Failed to parse JSON response from ${label}: ${text.slice(0, 200)}`);
+  }
+}
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+// src/ingesters/artificial-analysis.ts
+var DEFAULT_AA_URL = "https://artificialanalysis.ai/api/models";
+var artificialAnalysisIngester = {
+  source: "artificial-analysis",
+  displayName: "Artificial Analysis",
+  description: "Quality, speed, and cost benchmarks from Artificial Analysis.",
+  async ingest(options) {
+    const url = options?.sourceUrl ?? DEFAULT_AA_URL;
+    const response = await fetchWithRetry(url, { fetch: options?.fetch });
+    const text = await response.text();
+    const data = parseJsonSafe(text, "Artificial Analysis");
+    const now = new Date;
+    const results = [];
+    const dims = options?.dimensions ? new Set(options.dimensions) : null;
+    let entries = data.filter((e) => e.model_id && e.provider);
+    if (options?.modelFilter?.length) {
+      const filterSet = new Set(options.modelFilter);
+      entries = entries.filter((e) => filterSet.has(e.model_id));
+    }
+    for (const entry of entries) {
+      const baseId = `artificial-analysis:${entry.model_id}`;
+      if (entry.quality_score != null && (!dims || dims.has("reasoning"))) {
+        results.push({
+          id: `${baseId}:reasoning`,
+          modelId: entry.model_id,
+          providerKey: entry.provider.toLowerCase(),
+          source: "artificial-analysis",
+          dimension: "reasoning",
+          score: Math.max(0, Math.min(100, entry.quality_score)),
+          rawScore: entry.quality_score,
+          metadata: { model_name: entry.model_name },
+          measuredAt: now,
+          ingestedAt: now
+        });
+      }
+      if ((entry.tokens_per_second != null || entry.ttft_ms != null) && (!dims || dims.has("latency"))) {
+        const latencyScore = computeLatencyScore(entry.tokens_per_second, entry.ttft_ms);
+        results.push({
+          id: `${baseId}:latency`,
+          modelId: entry.model_id,
+          providerKey: entry.provider.toLowerCase(),
+          source: "artificial-analysis",
+          dimension: "latency",
+          score: latencyScore,
+          rawScore: {
+            tokens_per_second: entry.tokens_per_second,
+            ttft_ms: entry.ttft_ms
+          },
+          metadata: { model_name: entry.model_name },
+          measuredAt: now,
+          ingestedAt: now
+        });
+      }
+      if ((entry.price_per_million_input_tokens != null || entry.price_per_million_output_tokens != null) && (!dims || dims.has("cost"))) {
+        const costScore = computeCostScore(entry.price_per_million_input_tokens, entry.price_per_million_output_tokens);
+        results.push({
+          id: `${baseId}:cost`,
+          modelId: entry.model_id,
+          providerKey: entry.provider.toLowerCase(),
+          source: "artificial-analysis",
+          dimension: "cost",
+          score: costScore,
+          rawScore: {
+            input: entry.price_per_million_input_tokens,
+            output: entry.price_per_million_output_tokens
+          },
+          metadata: { model_name: entry.model_name },
+          measuredAt: now,
+          ingestedAt: now
+        });
+      }
+      if (entry.context_window != null && (!dims || dims.has("context"))) {
+        const contextScore = computeContextScore(entry.context_window);
+        results.push({
+          id: `${baseId}:context`,
+          modelId: entry.model_id,
+          providerKey: entry.provider.toLowerCase(),
+          source: "artificial-analysis",
+          dimension: "context",
+          score: contextScore,
+          rawScore: entry.context_window,
+          metadata: { model_name: entry.model_name },
+          measuredAt: now,
+          ingestedAt: now
+        });
+      }
+    }
+    return options?.maxResults ? results.slice(0, options.maxResults) : results;
+  }
+};
+function computeLatencyScore(tokensPerSec, ttftMs) {
+  let score = 50;
+  if (tokensPerSec != null) {
+    score = Math.min(100, tokensPerSec / 200 * 100);
+  }
+  if (ttftMs != null) {
+    const ttftPenalty = Math.max(0, Math.min(30, (ttftMs - 200) / 100 * 10));
+    score = Math.max(0, score - ttftPenalty);
+  }
+  return Math.round(score * 100) / 100;
+}
+function computeCostScore(inputCost, outputCost) {
+  const avgCost = ((inputCost ?? 0) + (outputCost ?? 0)) / 2;
+  const score = Math.max(0, 100 - avgCost / 30 * 100);
+  return Math.round(score * 100) / 100;
+}
+function computeContextScore(contextWindow) {
+  const score = Math.min(100, contextWindow / 1e6 * 100);
+  return Math.round(score * 100) / 100;
+}
+export {
+  artificialAnalysisIngester
+};

package/dist/ingesters/chatbot-arena.d.ts

@@ -0,0 +1,8 @@
+import type { BenchmarkIngester } from './types';
+/**
+ * Ingests Chatbot Arena (LMSYS) Elo ratings.
+ *
+ * Maps Elo ratings to the "reasoning" dimension since Arena
+ * measures general conversational/reasoning ability.
+ */
+export declare const chatbotArenaIngester: BenchmarkIngester;

package/dist/ingesters/chatbot-arena.js

@@ -0,0 +1,143 @@
+// @bun
+// src/ingesters/fetch-utils.ts
+async function fetchWithRetry(url, options) {
+  const fetchFn = options?.fetch ?? globalThis.fetch;
+  const maxRetries = options?.maxRetries ?? 2;
+  const baseDelay = options?.baseDelayMs ?? 500;
+  let lastError;
+  for (let attempt = 0;attempt <= maxRetries; attempt++) {
+    try {
+      const response = await fetchFn(url);
+      if (response.ok)
+        return response;
+      if (response.status >= 500 && attempt < maxRetries) {
+        await sleep(baseDelay * Math.pow(2, attempt));
+        continue;
+      }
+      throw new Error(`Fetch failed: ${response.status} ${response.statusText} (${url})`);
+    } catch (error) {
+      lastError = error instanceof Error ? error : new Error(String(error));
+      if (attempt < maxRetries) {
+        await sleep(baseDelay * Math.pow(2, attempt));
+      }
+    }
+  }
+  throw lastError ?? new Error(`Fetch failed after ${maxRetries + 1} attempts: ${url}`);
+}
+function parseJsonSafe(text, label) {
+  try {
+    return JSON.parse(text);
+  } catch {
+    throw new Error(`Failed to parse JSON response from ${label}: ${text.slice(0, 200)}`);
+  }
+}
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+// src/scoring/normalizer.ts
+var SOURCE_NORMALIZATION = {
+  "chatbot-arena": { min: 800, max: 1400, invertScale: false },
+  "swe-bench": { min: 0, max: 100, invertScale: false },
+  "human-eval": { min: 0, max: 100, invertScale: false },
+  mmlu: { min: 0, max: 100, invertScale: false },
+  gpqa: { min: 0, max: 100, invertScale: false },
+  arc: { min: 0, max: 100, invertScale: false },
+  truthfulqa: { min: 0, max: 100, invertScale: false },
+  "tau-bench": { min: 0, max: 100, invertScale: false },
+  "artificial-analysis": { min: 0, max: 100, invertScale: false }
+};
+function normalizeScore(rawScore, source, configOverride) {
+  const config = configOverride ?? SOURCE_NORMALIZATION[source];
+  if (!config) {
+    return Math.max(0, Math.min(100, rawScore));
+  }
+  const { min, max, invertScale } = config;
+  const range = max - min;
+  if (range === 0)
+    return 50;
+  let normalized = (rawScore - min) / range * 100;
+  if (invertScale) {
+    normalized = 100 - normalized;
+  }
+  return Math.max(0, Math.min(100, normalized));
+}
+function normalizeBenchmarkResults(results) {
+  return results.map((result) => ({
+    ...result,
+    score: normalizeScore(typeof result.rawScore === "number" ? result.rawScore : result.score, result.source)
+  }));
+}
+
+// src/ingesters/chatbot-arena.ts
+var DEFAULT_ARENA_URL = "https://huggingface.co/spaces/lmsys/chatbot-arena-leaderboard/resolve/main/results.json";
+var chatbotArenaIngester = {
+  source: "chatbot-arena",
+  displayName: "Chatbot Arena (LMSYS)",
+  description: "Elo ratings from the LMSYS Chatbot Arena human preference leaderboard.",
+  async ingest(options) {
+    if (options?.dimensions?.length && !options.dimensions.includes("reasoning")) {
+      return [];
+    }
+    const url = options?.sourceUrl ?? DEFAULT_ARENA_URL;
+    const response = await fetchWithRetry(url, { fetch: options?.fetch });
+    const text = await response.text();
+    const data = parseJsonSafe(text, "Chatbot Arena");
+    const now = new Date;
+    let entries = data.filter((entry) => entry["Arena Elo rating"] != null && entry.Model);
+    if (options?.modelFilter?.length) {
+      const filterSet = new Set(options.modelFilter);
+      entries = entries.filter((e) => filterSet.has(e.key ?? e.Model));
+    }
+    if (options?.maxResults) {
+      entries = entries.slice(0, options.maxResults);
+    }
+    let results = entries.map((entry) => {
+      const elo = entry["Arena Elo rating"];
+      const modelId = entry.key ?? entry.Model.toLowerCase().replace(/\s+/g, "-");
+      const org = entry.Organization?.toLowerCase() ?? "unknown";
+      return {
+        id: `chatbot-arena:${modelId}:reasoning`,
+        modelId,
+        providerKey: mapOrganizationToProvider(org),
+        source: "chatbot-arena",
+        dimension: "reasoning",
+        score: normalizeScore(elo, "chatbot-arena"),
+        rawScore: elo,
+        metadata: {
+          organization: entry.Organization,
+          license: entry.License
+        },
+        measuredAt: now,
+        ingestedAt: now
+      };
+    });
+    const { fromDate, toDate } = options ?? {};
+    if (fromDate) {
+      results = results.filter((r) => r.measuredAt >= fromDate);
+    }
+    if (toDate) {
+      results = results.filter((r) => r.measuredAt <= toDate);
+    }
+    return results;
+  }
+};
+function mapOrganizationToProvider(org) {
+  const normalized = org.toLowerCase();
+  if (normalized.includes("openai"))
+    return "openai";
+  if (normalized.includes("anthropic"))
+    return "anthropic";
+  if (normalized.includes("google") || normalized.includes("deepmind"))
+    return "gemini";
+  if (normalized.includes("mistral"))
+    return "mistral";
+  if (normalized.includes("meta"))
+    return "meta";
+  if (normalized.includes("cohere"))
+    return "cohere";
+  return org;
+}
+export {
+  chatbotArenaIngester
+};

package/dist/ingesters/fetch-utils.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Fetch helper with retry and exponential backoff for ingester robustness.
+ */
+interface FetchWithRetryOptions {
+    fetch?: typeof globalThis.fetch;
+    maxRetries?: number;
+    baseDelayMs?: number;
+}
+export declare function fetchWithRetry(url: string, options?: FetchWithRetryOptions): Promise<Response>;
+export declare function parseJsonSafe<T>(text: string, label: string): T;
+export {};

package/dist/ingesters/fetch-utils.js

@@ -0,0 +1,40 @@
+// @bun
+// src/ingesters/fetch-utils.ts
+async function fetchWithRetry(url, options) {
+  const fetchFn = options?.fetch ?? globalThis.fetch;
+  const maxRetries = options?.maxRetries ?? 2;
+  const baseDelay = options?.baseDelayMs ?? 500;
+  let lastError;
+  for (let attempt = 0;attempt <= maxRetries; attempt++) {
+    try {
+      const response = await fetchFn(url);
+      if (response.ok)
+        return response;
+      if (response.status >= 500 && attempt < maxRetries) {
+        await sleep(baseDelay * Math.pow(2, attempt));
+        continue;
+      }
+      throw new Error(`Fetch failed: ${response.status} ${response.statusText} (${url})`);
+    } catch (error) {
+      lastError = error instanceof Error ? error : new Error(String(error));
+      if (attempt < maxRetries) {
+        await sleep(baseDelay * Math.pow(2, attempt));
+      }
+    }
+  }
+  throw lastError ?? new Error(`Fetch failed after ${maxRetries + 1} attempts: ${url}`);
+}
+function parseJsonSafe(text, label) {
+  try {
+    return JSON.parse(text);
+  } catch {
+    throw new Error(`Failed to parse JSON response from ${label}: ${text.slice(0, 200)}`);
+  }
+}
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+export {
+  parseJsonSafe,
+  fetchWithRetry
+};

package/dist/ingesters/index.d.ts

@@ -0,0 +1,7 @@
+export type { BenchmarkIngester, IngesterOptions } from './types';
+export { chatbotArenaIngester } from './chatbot-arena';
+export { artificialAnalysisIngester } from './artificial-analysis';
+export { sweBenchIngester } from './swe-bench';
+export { openLlmLeaderboardIngester } from './open-llm-leaderboard';
+export { IngesterRegistry, createDefaultIngesterRegistry } from './registry';
+export { fetchWithRetry, parseJsonSafe } from './fetch-utils';