@contractspec/lib.provider-ranking 0.1.1

package/README.md

@@ -0,0 +1,44 @@
+# @contractspec/lib.provider-ranking
+
+Website: https://contractspec.io/
+
+**AI provider ranking: benchmark ingestion, scoring, and model comparison.**
+
+Ingests benchmark data from multiple sources (Chatbot Arena, SWE-bench, Artificial Analysis, Open LLM Leaderboard), normalizes scores to a 0-100 scale, and computes composite rankings across dimensions like coding, reasoning, cost, and latency.
+
+## Installation
+
+```bash
+bun add @contractspec/lib.provider-ranking
+```
+
+## Exports
+
+- `.` -- Core types, store interface, and in-memory store
+- `./types` -- `BenchmarkResult`, `ModelRanking`, `ModelProfile`, `BenchmarkDimension`, `DimensionWeightConfig`
+- `./store` -- `ProviderRankingStore` interface
+- `./in-memory-store` -- `InMemoryProviderRankingStore` class
+- `./scoring` -- `computeModelRankings()`, `normalizeScore()`, `DEFAULT_DIMENSION_WEIGHTS`
+- `./ingesters` -- `chatbotArenaIngester`, `sweBenchIngester`, `artificialAnalysisIngester`, `IngesterRegistry`
+- `./eval` -- `EvalRunner`, `EvalSuite`, `EvalCase` for custom evaluation
+
+## Usage
+
+```ts
+import { InMemoryProviderRankingStore } from "@contractspec/lib.provider-ranking/in-memory-store";
+import { createDefaultIngesterRegistry } from "@contractspec/lib.provider-ranking/ingesters";
+import { computeModelRankings } from "@contractspec/lib.provider-ranking/scoring";
+
+const store = new InMemoryProviderRankingStore();
+const registry = createDefaultIngesterRegistry();
+
+const ingester = registry.get("swe-bench");
+const results = await ingester.ingest();
+
+for (const result of results) {
+  await store.addBenchmarkResult(result);
+}
+
+const rankings = computeModelRankings(await store.listBenchmarkResults({}));
+console.log(rankings);
+```

package/dist/browser/eval/index.js

@@ -0,0 +1,101 @@
+// src/eval/runner.ts
+class EvalRunner {
+  adapter;
+  options;
+  constructor(adapter, options = {}) {
+    this.adapter = adapter;
+    this.options = options;
+  }
+  async run(suite, modelId, providerKey) {
+    const runId = `eval-${suite.key}-${modelId}-${Date.now()}`;
+    const startedAt = new Date;
+    const concurrency = this.options.maxConcurrency ?? 5;
+    const caseResults = await this.runCasesWithConcurrency(suite.cases, suite.defaultGrader, concurrency);
+    const passedCases = caseResults.filter((r) => r.passed).length;
+    const averageScore = caseResults.length > 0 ? caseResults.reduce((sum, r) => sum + r.score, 0) / caseResults.length : 0;
+    const averageLatencyMs = caseResults.length > 0 ? caseResults.reduce((sum, r) => sum + r.latencyMs, 0) / caseResults.length : 0;
+    return {
+      runId,
+      evalSuiteKey: suite.key,
+      modelId,
+      providerKey,
+      totalCases: suite.cases.length,
+      passedCases,
+      averageScore: Math.round(averageScore * 100) / 100,
+      averageLatencyMs: Math.round(averageLatencyMs),
+      caseResults,
+      startedAt,
+      completedAt: new Date
+    };
+  }
+  async runCasesWithConcurrency(cases, defaultGrader, concurrency) {
+    const results = [];
+    const queue = [...cases];
+    const workers = Array.from({ length: Math.min(concurrency, queue.length) }, async () => {
+      while (queue.length > 0) {
+        const evalCase = queue.shift();
+        if (!evalCase)
+          break;
+        const result = await this.runSingleCase(evalCase, defaultGrader);
+        results.push(result);
+      }
+    });
+    await Promise.all(workers);
+    return results;
+  }
+  async runSingleCase(evalCase, defaultGrader) {
+    try {
+      const { text, latencyMs } = await this.adapter.chat(evalCase.prompt);
+      const { passed, score } = this.grade(evalCase, text, defaultGrader);
+      return {
+        caseId: evalCase.id,
+        passed,
+        score,
+        response: text,
+        latencyMs
+      };
+    } catch (error) {
+      return {
+        caseId: evalCase.id,
+        passed: false,
+        score: 0,
+        response: "",
+        latencyMs: 0,
+        error: error instanceof Error ? error.message : String(error)
+      };
+    }
+  }
+  grade(evalCase, response, defaultGrader) {
+    const grader = evalCase.graderKey ?? defaultGrader;
+    switch (grader) {
+      case "exact":
+        if (!evalCase.expectedOutput)
+          return { passed: true, score: 1 };
+        return {
+          passed: response.trim() === evalCase.expectedOutput.trim(),
+          score: response.trim() === evalCase.expectedOutput.trim() ? 1 : 0
+        };
+      case "contains":
+        if (!evalCase.expectedOutput)
+          return { passed: true, score: 1 };
+        return {
+          passed: response.includes(evalCase.expectedOutput),
+          score: response.includes(evalCase.expectedOutput) ? 1 : 0
+        };
+      case "regex": {
+        if (!evalCase.expectedPattern)
+          return { passed: true, score: 1 };
+        const regex = new RegExp(evalCase.expectedPattern);
+        const matches = regex.test(response);
+        return { passed: matches, score: matches ? 1 : 0 };
+      }
+      case "llm-judge":
+        return { passed: true, score: 0.5 };
+      default:
+        return { passed: true, score: 0.5 };
+    }
+  }
+}
+export {
+  EvalRunner
+};

package/dist/browser/eval/runner.js

@@ -0,0 +1,101 @@
+// src/eval/runner.ts
+class EvalRunner {
+  adapter;
+  options;
+  constructor(adapter, options = {}) {
+    this.adapter = adapter;
+    this.options = options;
+  }
+  async run(suite, modelId, providerKey) {
+    const runId = `eval-${suite.key}-${modelId}-${Date.now()}`;
+    const startedAt = new Date;
+    const concurrency = this.options.maxConcurrency ?? 5;
+    const caseResults = await this.runCasesWithConcurrency(suite.cases, suite.defaultGrader, concurrency);
+    const passedCases = caseResults.filter((r) => r.passed).length;
+    const averageScore = caseResults.length > 0 ? caseResults.reduce((sum, r) => sum + r.score, 0) / caseResults.length : 0;
+    const averageLatencyMs = caseResults.length > 0 ? caseResults.reduce((sum, r) => sum + r.latencyMs, 0) / caseResults.length : 0;
+    return {
+      runId,
+      evalSuiteKey: suite.key,
+      modelId,
+      providerKey,
+      totalCases: suite.cases.length,
+      passedCases,
+      averageScore: Math.round(averageScore * 100) / 100,
+      averageLatencyMs: Math.round(averageLatencyMs),
+      caseResults,
+      startedAt,
+      completedAt: new Date
+    };
+  }
+  async runCasesWithConcurrency(cases, defaultGrader, concurrency) {
+    const results = [];
+    const queue = [...cases];
+    const workers = Array.from({ length: Math.min(concurrency, queue.length) }, async () => {
+      while (queue.length > 0) {
+        const evalCase = queue.shift();
+        if (!evalCase)
+          break;
+        const result = await this.runSingleCase(evalCase, defaultGrader);
+        results.push(result);
+      }
+    });
+    await Promise.all(workers);
+    return results;
+  }
+  async runSingleCase(evalCase, defaultGrader) {
+    try {
+      const { text, latencyMs } = await this.adapter.chat(evalCase.prompt);
+      const { passed, score } = this.grade(evalCase, text, defaultGrader);
+      return {
+        caseId: evalCase.id,
+        passed,
+        score,
+        response: text,
+        latencyMs
+      };
+    } catch (error) {
+      return {
+        caseId: evalCase.id,
+        passed: false,
+        score: 0,
+        response: "",
+        latencyMs: 0,
+        error: error instanceof Error ? error.message : String(error)
+      };
+    }
+  }
+  grade(evalCase, response, defaultGrader) {
+    const grader = evalCase.graderKey ?? defaultGrader;
+    switch (grader) {
+      case "exact":
+        if (!evalCase.expectedOutput)
+          return { passed: true, score: 1 };
+        return {
+          passed: response.trim() === evalCase.expectedOutput.trim(),
+          score: response.trim() === evalCase.expectedOutput.trim() ? 1 : 0
+        };
+      case "contains":
+        if (!evalCase.expectedOutput)
+          return { passed: true, score: 1 };
+        return {
+          passed: response.includes(evalCase.expectedOutput),
+          score: response.includes(evalCase.expectedOutput) ? 1 : 0
+        };
+      case "regex": {
+        if (!evalCase.expectedPattern)
+          return { passed: true, score: 1 };
+        const regex = new RegExp(evalCase.expectedPattern);
+        const matches = regex.test(response);
+        return { passed: matches, score: matches ? 1 : 0 };
+      }
+      case "llm-judge":
+        return { passed: true, score: 0.5 };
+      default:
+        return { passed: true, score: 0.5 };
+    }
+  }
+}
+export {
+  EvalRunner
+};

package/dist/browser/in-memory-store.js

@@ -0,0 +1,92 @@
+// src/in-memory-store.ts
+class InMemoryProviderRankingStore {
+  benchmarkResults = new Map;
+  modelRankings = new Map;
+  ingestionRuns = new Map;
+  async upsertBenchmarkResult(result) {
+    this.benchmarkResults.set(result.id, result);
+  }
+  async getBenchmarkResult(id) {
+    return this.benchmarkResults.get(id) ?? null;
+  }
+  async listBenchmarkResults(query) {
+    let results = Array.from(this.benchmarkResults.values());
+    if (query.source) {
+      results = results.filter((r) => r.source === query.source);
+    }
+    if (query.modelId) {
+      results = results.filter((r) => r.modelId === query.modelId);
+    }
+    if (query.dimension) {
+      results = results.filter((r) => r.dimension === query.dimension);
+    }
+    if (query.providerKey) {
+      results = results.filter((r) => r.providerKey === query.providerKey);
+    }
+    const total = results.length;
+    const offset = query.offset ?? 0;
+    const limit = query.limit ?? 50;
+    results = results.slice(offset, offset + limit);
+    const nextOffset = offset + results.length < total ? offset + results.length : undefined;
+    return { results, total, nextOffset };
+  }
+  async upsertModelRanking(ranking) {
+    this.modelRankings.set(ranking.modelId, ranking);
+  }
+  async getModelRanking(modelId) {
+    return this.modelRankings.get(modelId) ?? null;
+  }
+  async listModelRankings(query) {
+    let rankings = Array.from(this.modelRankings.values());
+    if (query.providerKey) {
+      rankings = rankings.filter((r) => r.providerKey === query.providerKey);
+    }
+    if (query.dimension) {
+      const dim = query.dimension;
+      rankings.sort((a, b) => {
+        const scoreA = a.dimensionScores[dim]?.score ?? -1;
+        const scoreB = b.dimensionScores[dim]?.score ?? -1;
+        return scoreB - scoreA;
+      });
+    } else {
+      rankings.sort((a, b) => a.rank - b.rank);
+    }
+    const total = rankings.length;
+    const offset = query.offset ?? 0;
+    const limit = query.limit ?? 50;
+    rankings = rankings.slice(offset, offset + limit);
+    const nextOffset = offset + rankings.length < total ? offset + rankings.length : undefined;
+    return { rankings, total, nextOffset };
+  }
+  async getModelProfile(modelId) {
+    const ranking = this.modelRankings.get(modelId);
+    if (!ranking)
+      return null;
+    const benchmarkResults = Array.from(this.benchmarkResults.values()).filter((r) => r.modelId === modelId);
+    return {
+      modelId,
+      providerKey: ranking.providerKey,
+      displayName: modelId,
+      contextWindow: 0,
+      costPerMillion: null,
+      capabilities: [],
+      ranking,
+      benchmarkResults
+    };
+  }
+  async createIngestionRun(run) {
+    this.ingestionRuns.set(run.id, run);
+  }
+  async updateIngestionRun(id, update) {
+    const existing = this.ingestionRuns.get(id);
+    if (existing) {
+      this.ingestionRuns.set(id, { ...existing, ...update });
+    }
+  }
+  async getIngestionRun(id) {
+    return this.ingestionRuns.get(id) ?? null;
+  }
+}
+export {
+  InMemoryProviderRankingStore
+};

package/dist/browser/index.js

@@ -0,0 +1,105 @@
+// src/in-memory-store.ts
+class InMemoryProviderRankingStore {
+  benchmarkResults = new Map;
+  modelRankings = new Map;
+  ingestionRuns = new Map;
+  async upsertBenchmarkResult(result) {
+    this.benchmarkResults.set(result.id, result);
+  }
+  async getBenchmarkResult(id) {
+    return this.benchmarkResults.get(id) ?? null;
+  }
+  async listBenchmarkResults(query) {
+    let results = Array.from(this.benchmarkResults.values());
+    if (query.source) {
+      results = results.filter((r) => r.source === query.source);
+    }
+    if (query.modelId) {
+      results = results.filter((r) => r.modelId === query.modelId);
+    }
+    if (query.dimension) {
+      results = results.filter((r) => r.dimension === query.dimension);
+    }
+    if (query.providerKey) {
+      results = results.filter((r) => r.providerKey === query.providerKey);
+    }
+    const total = results.length;
+    const offset = query.offset ?? 0;
+    const limit = query.limit ?? 50;
+    results = results.slice(offset, offset + limit);
+    const nextOffset = offset + results.length < total ? offset + results.length : undefined;
+    return { results, total, nextOffset };
+  }
+  async upsertModelRanking(ranking) {
+    this.modelRankings.set(ranking.modelId, ranking);
+  }
+  async getModelRanking(modelId) {
+    return this.modelRankings.get(modelId) ?? null;
+  }
+  async listModelRankings(query) {
+    let rankings = Array.from(this.modelRankings.values());
+    if (query.providerKey) {
+      rankings = rankings.filter((r) => r.providerKey === query.providerKey);
+    }
+    if (query.dimension) {
+      const dim = query.dimension;
+      rankings.sort((a, b) => {
+        const scoreA = a.dimensionScores[dim]?.score ?? -1;
+        const scoreB = b.dimensionScores[dim]?.score ?? -1;
+        return scoreB - scoreA;
+      });
+    } else {
+      rankings.sort((a, b) => a.rank - b.rank);
+    }
+    const total = rankings.length;
+    const offset = query.offset ?? 0;
+    const limit = query.limit ?? 50;
+    rankings = rankings.slice(offset, offset + limit);
+    const nextOffset = offset + rankings.length < total ? offset + rankings.length : undefined;
+    return { rankings, total, nextOffset };
+  }
+  async getModelProfile(modelId) {
+    const ranking = this.modelRankings.get(modelId);
+    if (!ranking)
+      return null;
+    const benchmarkResults = Array.from(this.benchmarkResults.values()).filter((r) => r.modelId === modelId);
+    return {
+      modelId,
+      providerKey: ranking.providerKey,
+      displayName: modelId,
+      contextWindow: 0,
+      costPerMillion: null,
+      capabilities: [],
+      ranking,
+      benchmarkResults
+    };
+  }
+  async createIngestionRun(run) {
+    this.ingestionRuns.set(run.id, run);
+  }
+  async updateIngestionRun(id, update) {
+    const existing = this.ingestionRuns.get(id);
+    if (existing) {
+      this.ingestionRuns.set(id, { ...existing, ...update });
+    }
+  }
+  async getIngestionRun(id) {
+    return this.ingestionRuns.get(id) ?? null;
+  }
+}
+
+// src/types.ts
+var BENCHMARK_DIMENSIONS = [
+  "coding",
+  "reasoning",
+  "agentic",
+  "cost",
+  "latency",
+  "context",
+  "safety",
+  "custom"
+];
+export {
+  InMemoryProviderRankingStore,
+  BENCHMARK_DIMENSIONS
+};

package/dist/browser/ingesters/artificial-analysis.js

@@ -0,0 +1,149 @@
+// src/ingesters/fetch-utils.ts
+async function fetchWithRetry(url, options) {
+  const fetchFn = options?.fetch ?? globalThis.fetch;
+  const maxRetries = options?.maxRetries ?? 2;
+  const baseDelay = options?.baseDelayMs ?? 500;
+  let lastError;
+  for (let attempt = 0;attempt <= maxRetries; attempt++) {
+    try {
+      const response = await fetchFn(url);
+      if (response.ok)
+        return response;
+      if (response.status >= 500 && attempt < maxRetries) {
+        await sleep(baseDelay * Math.pow(2, attempt));
+        continue;
+      }
+      throw new Error(`Fetch failed: ${response.status} ${response.statusText} (${url})`);
+    } catch (error) {
+      lastError = error instanceof Error ? error : new Error(String(error));
+      if (attempt < maxRetries) {
+        await sleep(baseDelay * Math.pow(2, attempt));
+      }
+    }
+  }
+  throw lastError ?? new Error(`Fetch failed after ${maxRetries + 1} attempts: ${url}`);
+}
+function parseJsonSafe(text, label) {
+  try {
+    return JSON.parse(text);
+  } catch {
+    throw new Error(`Failed to parse JSON response from ${label}: ${text.slice(0, 200)}`);
+  }
+}
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+// src/ingesters/artificial-analysis.ts
+var DEFAULT_AA_URL = "https://artificialanalysis.ai/api/models";
+var artificialAnalysisIngester = {
+  source: "artificial-analysis",
+  displayName: "Artificial Analysis",
+  description: "Quality, speed, and cost benchmarks from Artificial Analysis.",
+  async ingest(options) {
+    const url = options?.sourceUrl ?? DEFAULT_AA_URL;
+    const response = await fetchWithRetry(url, { fetch: options?.fetch });
+    const text = await response.text();
+    const data = parseJsonSafe(text, "Artificial Analysis");
+    const now = new Date;
+    const results = [];
+    const dims = options?.dimensions ? new Set(options.dimensions) : null;
+    let entries = data.filter((e) => e.model_id && e.provider);
+    if (options?.modelFilter?.length) {
+      const filterSet = new Set(options.modelFilter);
+      entries = entries.filter((e) => filterSet.has(e.model_id));
+    }
+    for (const entry of entries) {
+      const baseId = `artificial-analysis:${entry.model_id}`;
+      if (entry.quality_score != null && (!dims || dims.has("reasoning"))) {
+        results.push({
+          id: `${baseId}:reasoning`,
+          modelId: entry.model_id,
+          providerKey: entry.provider.toLowerCase(),
+          source: "artificial-analysis",
+          dimension: "reasoning",
+          score: Math.max(0, Math.min(100, entry.quality_score)),
+          rawScore: entry.quality_score,
+          metadata: { model_name: entry.model_name },
+          measuredAt: now,
+          ingestedAt: now
+        });
+      }
+      if ((entry.tokens_per_second != null || entry.ttft_ms != null) && (!dims || dims.has("latency"))) {
+        const latencyScore = computeLatencyScore(entry.tokens_per_second, entry.ttft_ms);
+        results.push({
+          id: `${baseId}:latency`,
+          modelId: entry.model_id,
+          providerKey: entry.provider.toLowerCase(),
+          source: "artificial-analysis",
+          dimension: "latency",
+          score: latencyScore,
+          rawScore: {
+            tokens_per_second: entry.tokens_per_second,
+            ttft_ms: entry.ttft_ms
+          },
+          metadata: { model_name: entry.model_name },
+          measuredAt: now,
+          ingestedAt: now
+        });
+      }
+      if ((entry.price_per_million_input_tokens != null || entry.price_per_million_output_tokens != null) && (!dims || dims.has("cost"))) {
+        const costScore = computeCostScore(entry.price_per_million_input_tokens, entry.price_per_million_output_tokens);
+        results.push({
+          id: `${baseId}:cost`,
+          modelId: entry.model_id,
+          providerKey: entry.provider.toLowerCase(),
+          source: "artificial-analysis",
+          dimension: "cost",
+          score: costScore,
+          rawScore: {
+            input: entry.price_per_million_input_tokens,
+            output: entry.price_per_million_output_tokens
+          },
+          metadata: { model_name: entry.model_name },
+          measuredAt: now,
+          ingestedAt: now
+        });
+      }
+      if (entry.context_window != null && (!dims || dims.has("context"))) {
+        const contextScore = computeContextScore(entry.context_window);
+        results.push({
+          id: `${baseId}:context`,
+          modelId: entry.model_id,
+          providerKey: entry.provider.toLowerCase(),
+          source: "artificial-analysis",
+          dimension: "context",
+          score: contextScore,
+          rawScore: entry.context_window,
+          metadata: { model_name: entry.model_name },
+          measuredAt: now,
+          ingestedAt: now
+        });
+      }
+    }
+    return options?.maxResults ? results.slice(0, options.maxResults) : results;
+  }
+};
+function computeLatencyScore(tokensPerSec, ttftMs) {
+  let score = 50;
+  if (tokensPerSec != null) {
+    score = Math.min(100, tokensPerSec / 200 * 100);
+  }
+  if (ttftMs != null) {
+    const ttftPenalty = Math.max(0, Math.min(30, (ttftMs - 200) / 100 * 10));
+    score = Math.max(0, score - ttftPenalty);
+  }
+  return Math.round(score * 100) / 100;
+}
+function computeCostScore(inputCost, outputCost) {
+  const avgCost = ((inputCost ?? 0) + (outputCost ?? 0)) / 2;
+  const score = Math.max(0, 100 - avgCost / 30 * 100);
+  return Math.round(score * 100) / 100;
+}
+function computeContextScore(contextWindow) {
+  const score = Math.min(100, contextWindow / 1e6 * 100);
+  return Math.round(score * 100) / 100;
+}
+export {
+  artificialAnalysisIngester
+};