@contractspec/lib.provider-ranking 0.1.1

package/dist/browser/scoring/composite-scorer.js

@@ -0,0 +1,122 @@
+// src/scoring/dimension-weights.ts
+var DEFAULT_DIMENSION_WEIGHTS = [
+  { dimension: "coding", weight: 20 },
+  { dimension: "reasoning", weight: 20 },
+  { dimension: "agentic", weight: 15 },
+  { dimension: "cost", weight: 10 },
+  { dimension: "latency", weight: 10 },
+  { dimension: "context", weight: 10 },
+  { dimension: "safety", weight: 10 },
+  { dimension: "custom", weight: 5 }
+];
+function getWeightMap(overrides) {
+  const map = new Map;
+  for (const w of DEFAULT_DIMENSION_WEIGHTS) {
+    map.set(w.dimension, w.weight);
+  }
+  if (overrides) {
+    for (const w of overrides) {
+      map.set(w.dimension, w.weight);
+    }
+  }
+  return map;
+}
+function normalizeWeights(weights, activeDimensions) {
+  const totalWeight = activeDimensions.reduce((sum, dim) => sum + (weights.get(dim) ?? 0), 0);
+  if (totalWeight === 0)
+    return new Map;
+  const normalized = new Map;
+  for (const dim of activeDimensions) {
+    const raw = weights.get(dim) ?? 0;
+    normalized.set(dim, raw / totalWeight);
+  }
+  return normalized;
+}
+
+// src/scoring/composite-scorer.ts
+function computeModelRankings(results, options, existingRankings) {
+  const byModel = groupByModel(results);
+  const weights = getWeightMap(options?.weightOverrides);
+  const unsorted = [];
+  for (const [modelId, modelResults] of byModel) {
+    const providerKey = modelResults[0]?.providerKey ?? "unknown";
+    const dimensionScores = computeDimensionScores(modelResults);
+    const activeDimensions = Object.keys(dimensionScores);
+    const normalizedWeights = normalizeWeights(weights, activeDimensions);
+    let compositeScore = 0;
+    for (const dim of activeDimensions) {
+      const dimScore = dimensionScores[dim];
+      const weight = normalizedWeights.get(dim) ?? 0;
+      if (dimScore) {
+        compositeScore += dimScore.score * weight;
+      }
+    }
+    const previousRank = existingRankings?.get(modelId)?.rank ?? null;
+    unsorted.push({
+      modelId,
+      providerKey,
+      compositeScore: Math.round(compositeScore * 100) / 100,
+      dimensionScores,
+      rank: 0,
+      previousRank,
+      updatedAt: new Date
+    });
+  }
+  unsorted.sort((a, b) => b.compositeScore - a.compositeScore);
+  return unsorted.map((ranking, index) => ({
+    ...ranking,
+    rank: index + 1
+  }));
+}
+function groupByModel(results) {
+  const map = new Map;
+  for (const result of results) {
+    const existing = map.get(result.modelId);
+    if (existing) {
+      existing.push(result);
+    } else {
+      map.set(result.modelId, [result]);
+    }
+  }
+  return map;
+}
+function computeDimensionScores(results) {
+  const byDimension = new Map;
+  for (const result of results) {
+    const existing = byDimension.get(result.dimension);
+    if (existing) {
+      existing.push(result);
+    } else {
+      byDimension.set(result.dimension, [result]);
+    }
+  }
+  const scores = {};
+  for (const [dimension, dimResults] of byDimension) {
+    const avgScore = dimResults.reduce((sum, r) => sum + r.score, 0) / dimResults.length;
+    const sources = [...new Set(dimResults.map((r) => r.source))];
+    const recencyFactor = computeRecencyFactor(dimResults);
+    const sourceDiversity = Math.min(sources.length / 3, 1);
+    const confidence = Math.round((recencyFactor * 0.5 + sourceDiversity * 0.5) * 100) / 100;
+    scores[dimension] = {
+      score: Math.round(avgScore * 100) / 100,
+      confidence,
+      sources
+    };
+  }
+  return scores;
+}
+function computeRecencyFactor(results) {
+  if (results.length === 0)
+    return 0;
+  const now = Date.now();
+  const mostRecent = Math.max(...results.map((r) => r.measuredAt.getTime()));
+  const daysSinceMostRecent = (now - mostRecent) / (1000 * 60 * 60 * 24);
+  if (daysSinceMostRecent <= 30)
+    return 1;
+  if (daysSinceMostRecent >= 180)
+    return 0.3;
+  return 1 - (daysSinceMostRecent - 30) / (180 - 30) * 0.7;
+}
+export {
+  computeModelRankings
+};

package/dist/browser/scoring/dimension-weights.js

@@ -0,0 +1,39 @@
+// src/scoring/dimension-weights.ts
+var DEFAULT_DIMENSION_WEIGHTS = [
+  { dimension: "coding", weight: 20 },
+  { dimension: "reasoning", weight: 20 },
+  { dimension: "agentic", weight: 15 },
+  { dimension: "cost", weight: 10 },
+  { dimension: "latency", weight: 10 },
+  { dimension: "context", weight: 10 },
+  { dimension: "safety", weight: 10 },
+  { dimension: "custom", weight: 5 }
+];
+function getWeightMap(overrides) {
+  const map = new Map;
+  for (const w of DEFAULT_DIMENSION_WEIGHTS) {
+    map.set(w.dimension, w.weight);
+  }
+  if (overrides) {
+    for (const w of overrides) {
+      map.set(w.dimension, w.weight);
+    }
+  }
+  return map;
+}
+function normalizeWeights(weights, activeDimensions) {
+  const totalWeight = activeDimensions.reduce((sum, dim) => sum + (weights.get(dim) ?? 0), 0);
+  if (totalWeight === 0)
+    return new Map;
+  const normalized = new Map;
+  for (const dim of activeDimensions) {
+    const raw = weights.get(dim) ?? 0;
+    normalized.set(dim, raw / totalWeight);
+  }
+  return normalized;
+}
+export {
+  normalizeWeights,
+  getWeightMap,
+  DEFAULT_DIMENSION_WEIGHTS
+};

package/dist/browser/scoring/index.js

@@ -0,0 +1,161 @@
+// src/scoring/normalizer.ts
+var SOURCE_NORMALIZATION = {
+  "chatbot-arena": { min: 800, max: 1400, invertScale: false },
+  "swe-bench": { min: 0, max: 100, invertScale: false },
+  "human-eval": { min: 0, max: 100, invertScale: false },
+  mmlu: { min: 0, max: 100, invertScale: false },
+  gpqa: { min: 0, max: 100, invertScale: false },
+  arc: { min: 0, max: 100, invertScale: false },
+  truthfulqa: { min: 0, max: 100, invertScale: false },
+  "tau-bench": { min: 0, max: 100, invertScale: false },
+  "artificial-analysis": { min: 0, max: 100, invertScale: false }
+};
+function normalizeScore(rawScore, source, configOverride) {
+  const config = configOverride ?? SOURCE_NORMALIZATION[source];
+  if (!config) {
+    return Math.max(0, Math.min(100, rawScore));
+  }
+  const { min, max, invertScale } = config;
+  const range = max - min;
+  if (range === 0)
+    return 50;
+  let normalized = (rawScore - min) / range * 100;
+  if (invertScale) {
+    normalized = 100 - normalized;
+  }
+  return Math.max(0, Math.min(100, normalized));
+}
+function normalizeBenchmarkResults(results) {
+  return results.map((result) => ({
+    ...result,
+    score: normalizeScore(typeof result.rawScore === "number" ? result.rawScore : result.score, result.source)
+  }));
+}
+
+// src/scoring/dimension-weights.ts
+var DEFAULT_DIMENSION_WEIGHTS = [
+  { dimension: "coding", weight: 20 },
+  { dimension: "reasoning", weight: 20 },
+  { dimension: "agentic", weight: 15 },
+  { dimension: "cost", weight: 10 },
+  { dimension: "latency", weight: 10 },
+  { dimension: "context", weight: 10 },
+  { dimension: "safety", weight: 10 },
+  { dimension: "custom", weight: 5 }
+];
+function getWeightMap(overrides) {
+  const map = new Map;
+  for (const w of DEFAULT_DIMENSION_WEIGHTS) {
+    map.set(w.dimension, w.weight);
+  }
+  if (overrides) {
+    for (const w of overrides) {
+      map.set(w.dimension, w.weight);
+    }
+  }
+  return map;
+}
+function normalizeWeights(weights, activeDimensions) {
+  const totalWeight = activeDimensions.reduce((sum, dim) => sum + (weights.get(dim) ?? 0), 0);
+  if (totalWeight === 0)
+    return new Map;
+  const normalized = new Map;
+  for (const dim of activeDimensions) {
+    const raw = weights.get(dim) ?? 0;
+    normalized.set(dim, raw / totalWeight);
+  }
+  return normalized;
+}
+
+// src/scoring/composite-scorer.ts
+function computeModelRankings(results, options, existingRankings) {
+  const byModel = groupByModel(results);
+  const weights = getWeightMap(options?.weightOverrides);
+  const unsorted = [];
+  for (const [modelId, modelResults] of byModel) {
+    const providerKey = modelResults[0]?.providerKey ?? "unknown";
+    const dimensionScores = computeDimensionScores(modelResults);
+    const activeDimensions = Object.keys(dimensionScores);
+    const normalizedWeights = normalizeWeights(weights, activeDimensions);
+    let compositeScore = 0;
+    for (const dim of activeDimensions) {
+      const dimScore = dimensionScores[dim];
+      const weight = normalizedWeights.get(dim) ?? 0;
+      if (dimScore) {
+        compositeScore += dimScore.score * weight;
+      }
+    }
+    const previousRank = existingRankings?.get(modelId)?.rank ?? null;
+    unsorted.push({
+      modelId,
+      providerKey,
+      compositeScore: Math.round(compositeScore * 100) / 100,
+      dimensionScores,
+      rank: 0,
+      previousRank,
+      updatedAt: new Date
+    });
+  }
+  unsorted.sort((a, b) => b.compositeScore - a.compositeScore);
+  return unsorted.map((ranking, index) => ({
+    ...ranking,
+    rank: index + 1
+  }));
+}
+function groupByModel(results) {
+  const map = new Map;
+  for (const result of results) {
+    const existing = map.get(result.modelId);
+    if (existing) {
+      existing.push(result);
+    } else {
+      map.set(result.modelId, [result]);
+    }
+  }
+  return map;
+}
+function computeDimensionScores(results) {
+  const byDimension = new Map;
+  for (const result of results) {
+    const existing = byDimension.get(result.dimension);
+    if (existing) {
+      existing.push(result);
+    } else {
+      byDimension.set(result.dimension, [result]);
+    }
+  }
+  const scores = {};
+  for (const [dimension, dimResults] of byDimension) {
+    const avgScore = dimResults.reduce((sum, r) => sum + r.score, 0) / dimResults.length;
+    const sources = [...new Set(dimResults.map((r) => r.source))];
+    const recencyFactor = computeRecencyFactor(dimResults);
+    const sourceDiversity = Math.min(sources.length / 3, 1);
+    const confidence = Math.round((recencyFactor * 0.5 + sourceDiversity * 0.5) * 100) / 100;
+    scores[dimension] = {
+      score: Math.round(avgScore * 100) / 100,
+      confidence,
+      sources
+    };
+  }
+  return scores;
+}
+function computeRecencyFactor(results) {
+  if (results.length === 0)
+    return 0;
+  const now = Date.now();
+  const mostRecent = Math.max(...results.map((r) => r.measuredAt.getTime()));
+  const daysSinceMostRecent = (now - mostRecent) / (1000 * 60 * 60 * 24);
+  if (daysSinceMostRecent <= 30)
+    return 1;
+  if (daysSinceMostRecent >= 180)
+    return 0.3;
+  return 1 - (daysSinceMostRecent - 30) / (180 - 30) * 0.7;
+}
+export {
+  normalizeWeights,
+  normalizeScore,
+  normalizeBenchmarkResults,
+  getWeightMap,
+  computeModelRankings,
+  DEFAULT_DIMENSION_WEIGHTS
+};

package/dist/browser/scoring/normalizer.js

@@ -0,0 +1,37 @@
+// src/scoring/normalizer.ts
+var SOURCE_NORMALIZATION = {
+  "chatbot-arena": { min: 800, max: 1400, invertScale: false },
+  "swe-bench": { min: 0, max: 100, invertScale: false },
+  "human-eval": { min: 0, max: 100, invertScale: false },
+  mmlu: { min: 0, max: 100, invertScale: false },
+  gpqa: { min: 0, max: 100, invertScale: false },
+  arc: { min: 0, max: 100, invertScale: false },
+  truthfulqa: { min: 0, max: 100, invertScale: false },
+  "tau-bench": { min: 0, max: 100, invertScale: false },
+  "artificial-analysis": { min: 0, max: 100, invertScale: false }
+};
+function normalizeScore(rawScore, source, configOverride) {
+  const config = configOverride ?? SOURCE_NORMALIZATION[source];
+  if (!config) {
+    return Math.max(0, Math.min(100, rawScore));
+  }
+  const { min, max, invertScale } = config;
+  const range = max - min;
+  if (range === 0)
+    return 50;
+  let normalized = (rawScore - min) / range * 100;
+  if (invertScale) {
+    normalized = 100 - normalized;
+  }
+  return Math.max(0, Math.min(100, normalized));
+}
+function normalizeBenchmarkResults(results) {
+  return results.map((result) => ({
+    ...result,
+    score: normalizeScore(typeof result.rawScore === "number" ? result.rawScore : result.score, result.source)
+  }));
+}
+export {
+  normalizeScore,
+  normalizeBenchmarkResults
+};

package/dist/browser/types.js

@@ -0,0 +1,14 @@
+// src/types.ts
+var BENCHMARK_DIMENSIONS = [
+  "coding",
+  "reasoning",
+  "agentic",
+  "cost",
+  "latency",
+  "context",
+  "safety",
+  "custom"
+];
+export {
+  BENCHMARK_DIMENSIONS
+};

package/dist/eval/index.d.ts

@@ -0,0 +1,2 @@
+export type { EvalCase, EvalSuite, EvalCaseResult, EvalRunResult, EvalLLMAdapter, } from './types';
+export { EvalRunner } from './runner';

package/dist/eval/index.js

@@ -0,0 +1,102 @@
+// @bun
+// src/eval/runner.ts
+class EvalRunner {
+  adapter;
+  options;
+  constructor(adapter, options = {}) {
+    this.adapter = adapter;
+    this.options = options;
+  }
+  async run(suite, modelId, providerKey) {
+    const runId = `eval-${suite.key}-${modelId}-${Date.now()}`;
+    const startedAt = new Date;
+    const concurrency = this.options.maxConcurrency ?? 5;
+    const caseResults = await this.runCasesWithConcurrency(suite.cases, suite.defaultGrader, concurrency);
+    const passedCases = caseResults.filter((r) => r.passed).length;
+    const averageScore = caseResults.length > 0 ? caseResults.reduce((sum, r) => sum + r.score, 0) / caseResults.length : 0;
+    const averageLatencyMs = caseResults.length > 0 ? caseResults.reduce((sum, r) => sum + r.latencyMs, 0) / caseResults.length : 0;
+    return {
+      runId,
+      evalSuiteKey: suite.key,
+      modelId,
+      providerKey,
+      totalCases: suite.cases.length,
+      passedCases,
+      averageScore: Math.round(averageScore * 100) / 100,
+      averageLatencyMs: Math.round(averageLatencyMs),
+      caseResults,
+      startedAt,
+      completedAt: new Date
+    };
+  }
+  async runCasesWithConcurrency(cases, defaultGrader, concurrency) {
+    const results = [];
+    const queue = [...cases];
+    const workers = Array.from({ length: Math.min(concurrency, queue.length) }, async () => {
+      while (queue.length > 0) {
+        const evalCase = queue.shift();
+        if (!evalCase)
+          break;
+        const result = await this.runSingleCase(evalCase, defaultGrader);
+        results.push(result);
+      }
+    });
+    await Promise.all(workers);
+    return results;
+  }
+  async runSingleCase(evalCase, defaultGrader) {
+    try {
+      const { text, latencyMs } = await this.adapter.chat(evalCase.prompt);
+      const { passed, score } = this.grade(evalCase, text, defaultGrader);
+      return {
+        caseId: evalCase.id,
+        passed,
+        score,
+        response: text,
+        latencyMs
+      };
+    } catch (error) {
+      return {
+        caseId: evalCase.id,
+        passed: false,
+        score: 0,
+        response: "",
+        latencyMs: 0,
+        error: error instanceof Error ? error.message : String(error)
+      };
+    }
+  }
+  grade(evalCase, response, defaultGrader) {
+    const grader = evalCase.graderKey ?? defaultGrader;
+    switch (grader) {
+      case "exact":
+        if (!evalCase.expectedOutput)
+          return { passed: true, score: 1 };
+        return {
+          passed: response.trim() === evalCase.expectedOutput.trim(),
+          score: response.trim() === evalCase.expectedOutput.trim() ? 1 : 0
+        };
+      case "contains":
+        if (!evalCase.expectedOutput)
+          return { passed: true, score: 1 };
+        return {
+          passed: response.includes(evalCase.expectedOutput),
+          score: response.includes(evalCase.expectedOutput) ? 1 : 0
+        };
+      case "regex": {
+        if (!evalCase.expectedPattern)
+          return { passed: true, score: 1 };
+        const regex = new RegExp(evalCase.expectedPattern);
+        const matches = regex.test(response);
+        return { passed: matches, score: matches ? 1 : 0 };
+      }
+      case "llm-judge":
+        return { passed: true, score: 0.5 };
+      default:
+        return { passed: true, score: 0.5 };
+    }
+  }
+}
+export {
+  EvalRunner
+};

package/dist/eval/runner.d.ts

@@ -0,0 +1,18 @@
+import type { EvalLLMAdapter, EvalRunResult, EvalSuite } from './types';
+interface EvalRunnerOptions {
+    maxConcurrency?: number;
+}
+/**
+ * Executes an eval suite against an LLM adapter and produces
+ * scored results that can be stored as BenchmarkResults.
+ */
+export declare class EvalRunner {
+    private readonly adapter;
+    private readonly options;
+    constructor(adapter: EvalLLMAdapter, options?: EvalRunnerOptions);
+    run(suite: EvalSuite, modelId: string, providerKey: string): Promise<EvalRunResult>;
+    private runCasesWithConcurrency;
+    private runSingleCase;
+    private grade;
+}
+export {};

package/dist/eval/runner.js

@@ -0,0 +1,102 @@
+// @bun
+// src/eval/runner.ts
+class EvalRunner {
+  adapter;
+  options;
+  constructor(adapter, options = {}) {
+    this.adapter = adapter;
+    this.options = options;
+  }
+  async run(suite, modelId, providerKey) {
+    const runId = `eval-${suite.key}-${modelId}-${Date.now()}`;
+    const startedAt = new Date;
+    const concurrency = this.options.maxConcurrency ?? 5;
+    const caseResults = await this.runCasesWithConcurrency(suite.cases, suite.defaultGrader, concurrency);
+    const passedCases = caseResults.filter((r) => r.passed).length;
+    const averageScore = caseResults.length > 0 ? caseResults.reduce((sum, r) => sum + r.score, 0) / caseResults.length : 0;
+    const averageLatencyMs = caseResults.length > 0 ? caseResults.reduce((sum, r) => sum + r.latencyMs, 0) / caseResults.length : 0;
+    return {
+      runId,
+      evalSuiteKey: suite.key,
+      modelId,
+      providerKey,
+      totalCases: suite.cases.length,
+      passedCases,
+      averageScore: Math.round(averageScore * 100) / 100,
+      averageLatencyMs: Math.round(averageLatencyMs),
+      caseResults,
+      startedAt,
+      completedAt: new Date
+    };
+  }
+  async runCasesWithConcurrency(cases, defaultGrader, concurrency) {
+    const results = [];
+    const queue = [...cases];
+    const workers = Array.from({ length: Math.min(concurrency, queue.length) }, async () => {
+      while (queue.length > 0) {
+        const evalCase = queue.shift();
+        if (!evalCase)
+          break;
+        const result = await this.runSingleCase(evalCase, defaultGrader);
+        results.push(result);
+      }
+    });
+    await Promise.all(workers);
+    return results;
+  }
+  async runSingleCase(evalCase, defaultGrader) {
+    try {
+      const { text, latencyMs } = await this.adapter.chat(evalCase.prompt);
+      const { passed, score } = this.grade(evalCase, text, defaultGrader);
+      return {
+        caseId: evalCase.id,
+        passed,
+        score,
+        response: text,
+        latencyMs
+      };
+    } catch (error) {
+      return {
+        caseId: evalCase.id,
+        passed: false,
+        score: 0,
+        response: "",
+        latencyMs: 0,
+        error: error instanceof Error ? error.message : String(error)
+      };
+    }
+  }
+  grade(evalCase, response, defaultGrader) {
+    const grader = evalCase.graderKey ?? defaultGrader;
+    switch (grader) {
+      case "exact":
+        if (!evalCase.expectedOutput)
+          return { passed: true, score: 1 };
+        return {
+          passed: response.trim() === evalCase.expectedOutput.trim(),
+          score: response.trim() === evalCase.expectedOutput.trim() ? 1 : 0
+        };
+      case "contains":
+        if (!evalCase.expectedOutput)
+          return { passed: true, score: 1 };
+        return {
+          passed: response.includes(evalCase.expectedOutput),
+          score: response.includes(evalCase.expectedOutput) ? 1 : 0
+        };
+      case "regex": {
+        if (!evalCase.expectedPattern)
+          return { passed: true, score: 1 };
+        const regex = new RegExp(evalCase.expectedPattern);
+        const matches = regex.test(response);
+        return { passed: matches, score: matches ? 1 : 0 };
+      }
+      case "llm-judge":
+        return { passed: true, score: 0.5 };
+      default:
+        return { passed: true, score: 0.5 };
+    }
+  }
+}
+export {
+  EvalRunner
+};

package/dist/eval/types.d.ts

@@ -0,0 +1,51 @@
+import type { BenchmarkDimension } from '../types';
+export interface EvalCase {
+    id: string;
+    prompt: string;
+    expectedOutput?: string;
+    /** Regex or substring that must appear in the response. */
+    expectedPattern?: string;
+    /** Custom grading function key (resolved at runtime). */
+    graderKey?: string;
+    metadata?: Record<string, unknown>;
+}
+export interface EvalSuite {
+    key: string;
+    displayName: string;
+    description: string;
+    dimension: BenchmarkDimension;
+    cases: EvalCase[];
+    /** Default grading strategy when individual cases don't specify one. */
+    defaultGrader: 'exact' | 'contains' | 'regex' | 'llm-judge';
+}
+export interface EvalCaseResult {
+    caseId: string;
+    passed: boolean;
+    score: number;
+    response: string;
+    latencyMs: number;
+    error?: string;
+}
+export interface EvalRunResult {
+    runId: string;
+    evalSuiteKey: string;
+    modelId: string;
+    providerKey: string;
+    totalCases: number;
+    passedCases: number;
+    averageScore: number;
+    averageLatencyMs: number;
+    caseResults: EvalCaseResult[];
+    startedAt: Date;
+    completedAt: Date;
+}
+/**
+ * Abstraction over the LLM provider for eval execution.
+ * Kept minimal to avoid coupling to a specific SDK.
+ */
+export interface EvalLLMAdapter {
+    chat(prompt: string): Promise<{
+        text: string;
+        latencyMs: number;
+    }>;
+}

package/dist/eval/types.js

@@ -0,0 +1 @@
+// @bun

package/dist/in-memory-store.d.ts

@@ -0,0 +1,17 @@
+import type { ProviderRankingStore } from './store';
+import type { BenchmarkResult, BenchmarkResultListResult, BenchmarkResultQuery, IngestionRun, ModelProfile, ModelRanking, RankingListResult, RankingQuery } from './types';
+export declare class InMemoryProviderRankingStore implements ProviderRankingStore {
+    private benchmarkResults;
+    private modelRankings;
+    private ingestionRuns;
+    upsertBenchmarkResult(result: BenchmarkResult): Promise<void>;
+    getBenchmarkResult(id: string): Promise<BenchmarkResult | null>;
+    listBenchmarkResults(query: BenchmarkResultQuery): Promise<BenchmarkResultListResult>;
+    upsertModelRanking(ranking: ModelRanking): Promise<void>;
+    getModelRanking(modelId: string): Promise<ModelRanking | null>;
+    listModelRankings(query: RankingQuery): Promise<RankingListResult>;
+    getModelProfile(modelId: string): Promise<ModelProfile | null>;
+    createIngestionRun(run: IngestionRun): Promise<void>;
+    updateIngestionRun(id: string, update: Partial<IngestionRun>): Promise<void>;
+    getIngestionRun(id: string): Promise<IngestionRun | null>;
+}