@contractspec/lib.provider-ranking 0.1.1

package/dist/types.d.ts

@@ -0,0 +1,100 @@
+/**
+ * Core domain types for the AI Provider Ranking system.
+ *
+ * Normalized scores use a 0-100 scale. Raw scores preserve the original
+ * format from each benchmark source for auditability.
+ */
+export type BenchmarkDimension = 'coding' | 'reasoning' | 'agentic' | 'cost' | 'latency' | 'context' | 'safety' | 'custom';
+export declare const BENCHMARK_DIMENSIONS: readonly BenchmarkDimension[];
+export type BenchmarkSource = 'chatbot-arena' | 'swe-bench' | 'human-eval' | 'mmlu' | 'gpqa' | 'arc' | 'truthfulqa' | 'tau-bench' | 'artificial-analysis' | 'custom';
+export interface BenchmarkResult {
+    id: string;
+    modelId: string;
+    providerKey: string;
+    source: BenchmarkSource;
+    dimension: BenchmarkDimension;
+    /** Normalized score on a 0-100 scale. */
+    score: number;
+    /** Original score from the source, preserved for auditability. */
+    rawScore: unknown;
+    metadata: Record<string, unknown>;
+    measuredAt: Date;
+    ingestedAt: Date;
+}
+export interface DimensionScore {
+    score: number;
+    /** 0-1 confidence based on number and recency of sources. */
+    confidence: number;
+    sources: string[];
+}
+export interface ModelRanking {
+    modelId: string;
+    providerKey: string;
+    compositeScore: number;
+    dimensionScores: Partial<Record<BenchmarkDimension, DimensionScore>>;
+    rank: number;
+    previousRank: number | null;
+    updatedAt: Date;
+}
+export interface ModelProfile {
+    modelId: string;
+    providerKey: string;
+    displayName: string;
+    contextWindow: number;
+    costPerMillion: {
+        input: number;
+        output: number;
+    } | null;
+    capabilities: string[];
+    ranking: ModelRanking | null;
+    benchmarkResults: BenchmarkResult[];
+    /** Transports supported by this model's provider. */
+    supportedTransports?: ProviderTransportSupport[];
+    /** Auth methods supported by this model's provider. */
+    supportedAuthMethods?: ProviderAuthSupport[];
+    /** Whether the provider supports BYOK. */
+    byokSupported?: boolean;
+}
+export interface BenchmarkResultQuery {
+    source?: BenchmarkSource;
+    modelId?: string;
+    dimension?: BenchmarkDimension;
+    providerKey?: string;
+    limit?: number;
+    offset?: number;
+}
+export interface BenchmarkResultListResult {
+    results: BenchmarkResult[];
+    total: number;
+    nextOffset?: number;
+}
+export interface RankingQuery {
+    dimension?: BenchmarkDimension;
+    providerKey?: string;
+    limit?: number;
+    offset?: number;
+    /** Filter by required transport support. */
+    requiredTransport?: ProviderTransportSupport;
+    /** Filter by required auth method. */
+    requiredAuthMethod?: ProviderAuthSupport;
+}
+export interface RankingListResult {
+    rankings: ModelRanking[];
+    total: number;
+    nextOffset?: number;
+}
+export interface IngestionRun {
+    id: string;
+    source: BenchmarkSource;
+    status: 'pending' | 'running' | 'completed' | 'failed';
+    resultsCount: number;
+    startedAt: Date;
+    completedAt: Date | null;
+    error: string | null;
+}
+export type ProviderTransportSupport = 'rest' | 'mcp' | 'webhook' | 'sdk';
+export type ProviderAuthSupport = 'api-key' | 'oauth2' | 'bearer' | 'header' | 'basic' | 'webhook-signing' | 'service-account';
+export interface DimensionWeightConfig {
+    dimension: BenchmarkDimension;
+    weight: number;
+}

package/dist/types.js

@@ -0,0 +1,15 @@
+// @bun
+// src/types.ts
+var BENCHMARK_DIMENSIONS = [
+  "coding",
+  "reasoning",
+  "agentic",
+  "cost",
+  "latency",
+  "context",
+  "safety",
+  "custom"
+];
+export {
+  BENCHMARK_DIMENSIONS
+};

package/package.json

@@ -0,0 +1,362 @@
+{
+  "name": "@contractspec/lib.provider-ranking",
+  "version": "0.1.1",
+  "description": "AI provider ranking: benchmark ingestion, scoring, and model comparison",
+  "keywords": [
+    "contractspec",
+    "ai",
+    "ranking",
+    "benchmark",
+    "llm"
+  ],
+  "type": "module",
+  "types": "./dist/index.d.ts",
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "scripts": {
+    "publish:pkg": "bun publish --tolerate-republish --ignore-scripts --verbose",
+    "publish:pkg:canary": "bun publish:pkg --tag canary",
+    "build": "bun run prebuild && bun run build:bundle && bun run build:types",
+    "build:bundle": "contractspec-bun-build transpile",
+    "build:types": "contractspec-bun-build types",
+    "dev": "contractspec-bun-build dev",
+    "clean": "rimraf dist .turbo",
+    "lint": "bun lint:fix",
+    "lint:fix": "eslint src --fix",
+    "lint:check": "eslint src",
+    "test": "bun test --pass-with-no-tests",
+    "prebuild": "contractspec-bun-build prebuild",
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {},
+  "devDependencies": {
+    "@contractspec/tool.typescript": "3.1.0",
+    "@contractspec/tool.bun": "3.1.0",
+    "typescript": "^5.9.3"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "bun": "./dist/index.js",
+      "node": "./dist/node/index.js",
+      "browser": "./dist/browser/index.js",
+      "default": "./dist/index.js"
+    },
+    "./eval": {
+      "types": "./dist/eval/index.d.ts",
+      "bun": "./dist/eval/index.js",
+      "node": "./dist/node/eval/index.js",
+      "browser": "./dist/browser/eval/index.js",
+      "default": "./dist/eval/index.js"
+    },
+    "./eval/index": {
+      "types": "./dist/eval/index.d.ts",
+      "bun": "./dist/eval/index.js",
+      "node": "./dist/node/eval/index.js",
+      "browser": "./dist/browser/eval/index.js",
+      "default": "./dist/eval/index.js"
+    },
+    "./eval/runner": {
+      "types": "./dist/eval/runner.d.ts",
+      "bun": "./dist/eval/runner.js",
+      "node": "./dist/node/eval/runner.js",
+      "browser": "./dist/browser/eval/runner.js",
+      "default": "./dist/eval/runner.js"
+    },
+    "./eval/types": {
+      "types": "./dist/eval/types.d.ts",
+      "bun": "./dist/eval/types.js",
+      "node": "./dist/node/eval/types.js",
+      "browser": "./dist/browser/eval/types.js",
+      "default": "./dist/eval/types.js"
+    },
+    "./in-memory-store": {
+      "types": "./dist/in-memory-store.d.ts",
+      "bun": "./dist/in-memory-store.js",
+      "node": "./dist/node/in-memory-store.js",
+      "browser": "./dist/browser/in-memory-store.js",
+      "default": "./dist/in-memory-store.js"
+    },
+    "./ingesters": {
+      "types": "./dist/ingesters/index.d.ts",
+      "bun": "./dist/ingesters/index.js",
+      "node": "./dist/node/ingesters/index.js",
+      "browser": "./dist/browser/ingesters/index.js",
+      "default": "./dist/ingesters/index.js"
+    },
+    "./ingesters/artificial-analysis": {
+      "types": "./dist/ingesters/artificial-analysis.d.ts",
+      "bun": "./dist/ingesters/artificial-analysis.js",
+      "node": "./dist/node/ingesters/artificial-analysis.js",
+      "browser": "./dist/browser/ingesters/artificial-analysis.js",
+      "default": "./dist/ingesters/artificial-analysis.js"
+    },
+    "./ingesters/chatbot-arena": {
+      "types": "./dist/ingesters/chatbot-arena.d.ts",
+      "bun": "./dist/ingesters/chatbot-arena.js",
+      "node": "./dist/node/ingesters/chatbot-arena.js",
+      "browser": "./dist/browser/ingesters/chatbot-arena.js",
+      "default": "./dist/ingesters/chatbot-arena.js"
+    },
+    "./ingesters/fetch-utils": {
+      "types": "./dist/ingesters/fetch-utils.d.ts",
+      "bun": "./dist/ingesters/fetch-utils.js",
+      "node": "./dist/node/ingesters/fetch-utils.js",
+      "browser": "./dist/browser/ingesters/fetch-utils.js",
+      "default": "./dist/ingesters/fetch-utils.js"
+    },
+    "./ingesters/index": {
+      "types": "./dist/ingesters/index.d.ts",
+      "bun": "./dist/ingesters/index.js",
+      "node": "./dist/node/ingesters/index.js",
+      "browser": "./dist/browser/ingesters/index.js",
+      "default": "./dist/ingesters/index.js"
+    },
+    "./ingesters/open-llm-leaderboard": {
+      "types": "./dist/ingesters/open-llm-leaderboard.d.ts",
+      "bun": "./dist/ingesters/open-llm-leaderboard.js",
+      "node": "./dist/node/ingesters/open-llm-leaderboard.js",
+      "browser": "./dist/browser/ingesters/open-llm-leaderboard.js",
+      "default": "./dist/ingesters/open-llm-leaderboard.js"
+    },
+    "./ingesters/registry": {
+      "types": "./dist/ingesters/registry.d.ts",
+      "bun": "./dist/ingesters/registry.js",
+      "node": "./dist/node/ingesters/registry.js",
+      "browser": "./dist/browser/ingesters/registry.js",
+      "default": "./dist/ingesters/registry.js"
+    },
+    "./ingesters/swe-bench": {
+      "types": "./dist/ingesters/swe-bench.d.ts",
+      "bun": "./dist/ingesters/swe-bench.js",
+      "node": "./dist/node/ingesters/swe-bench.js",
+      "browser": "./dist/browser/ingesters/swe-bench.js",
+      "default": "./dist/ingesters/swe-bench.js"
+    },
+    "./ingesters/types": {
+      "types": "./dist/ingesters/types.d.ts",
+      "bun": "./dist/ingesters/types.js",
+      "node": "./dist/node/ingesters/types.js",
+      "browser": "./dist/browser/ingesters/types.js",
+      "default": "./dist/ingesters/types.js"
+    },
+    "./scoring": {
+      "types": "./dist/scoring/index.d.ts",
+      "bun": "./dist/scoring/index.js",
+      "node": "./dist/node/scoring/index.js",
+      "browser": "./dist/browser/scoring/index.js",
+      "default": "./dist/scoring/index.js"
+    },
+    "./scoring/composite-scorer": {
+      "types": "./dist/scoring/composite-scorer.d.ts",
+      "bun": "./dist/scoring/composite-scorer.js",
+      "node": "./dist/node/scoring/composite-scorer.js",
+      "browser": "./dist/browser/scoring/composite-scorer.js",
+      "default": "./dist/scoring/composite-scorer.js"
+    },
+    "./scoring/dimension-weights": {
+      "types": "./dist/scoring/dimension-weights.d.ts",
+      "bun": "./dist/scoring/dimension-weights.js",
+      "node": "./dist/node/scoring/dimension-weights.js",
+      "browser": "./dist/browser/scoring/dimension-weights.js",
+      "default": "./dist/scoring/dimension-weights.js"
+    },
+    "./scoring/index": {
+      "types": "./dist/scoring/index.d.ts",
+      "bun": "./dist/scoring/index.js",
+      "node": "./dist/node/scoring/index.js",
+      "browser": "./dist/browser/scoring/index.js",
+      "default": "./dist/scoring/index.js"
+    },
+    "./scoring/normalizer": {
+      "types": "./dist/scoring/normalizer.d.ts",
+      "bun": "./dist/scoring/normalizer.js",
+      "node": "./dist/node/scoring/normalizer.js",
+      "browser": "./dist/browser/scoring/normalizer.js",
+      "default": "./dist/scoring/normalizer.js"
+    },
+    "./store": {
+      "types": "./dist/store.d.ts",
+      "bun": "./dist/store.js",
+      "node": "./dist/node/store.js",
+      "browser": "./dist/browser/store.js",
+      "default": "./dist/store.js"
+    },
+    "./types": {
+      "types": "./dist/types.d.ts",
+      "bun": "./dist/types.js",
+      "node": "./dist/node/types.js",
+      "browser": "./dist/browser/types.js",
+      "default": "./dist/types.js"
+    }
+  },
+  "publishConfig": {
+    "access": "public",
+    "exports": {
+      ".": {
+        "types": "./dist/index.d.ts",
+        "bun": "./dist/index.js",
+        "node": "./dist/node/index.js",
+        "browser": "./dist/browser/index.js",
+        "default": "./dist/index.js"
+      },
+      "./eval": {
+        "types": "./dist/eval/index.d.ts",
+        "bun": "./dist/eval/index.js",
+        "node": "./dist/node/eval/index.js",
+        "browser": "./dist/browser/eval/index.js",
+        "default": "./dist/eval/index.js"
+      },
+      "./eval/index": {
+        "types": "./dist/eval/index.d.ts",
+        "bun": "./dist/eval/index.js",
+        "node": "./dist/node/eval/index.js",
+        "browser": "./dist/browser/eval/index.js",
+        "default": "./dist/eval/index.js"
+      },
+      "./eval/runner": {
+        "types": "./dist/eval/runner.d.ts",
+        "bun": "./dist/eval/runner.js",
+        "node": "./dist/node/eval/runner.js",
+        "browser": "./dist/browser/eval/runner.js",
+        "default": "./dist/eval/runner.js"
+      },
+      "./eval/types": {
+        "types": "./dist/eval/types.d.ts",
+        "bun": "./dist/eval/types.js",
+        "node": "./dist/node/eval/types.js",
+        "browser": "./dist/browser/eval/types.js",
+        "default": "./dist/eval/types.js"
+      },
+      "./in-memory-store": {
+        "types": "./dist/in-memory-store.d.ts",
+        "bun": "./dist/in-memory-store.js",
+        "node": "./dist/node/in-memory-store.js",
+        "browser": "./dist/browser/in-memory-store.js",
+        "default": "./dist/in-memory-store.js"
+      },
+      "./ingesters": {
+        "types": "./dist/ingesters/index.d.ts",
+        "bun": "./dist/ingesters/index.js",
+        "node": "./dist/node/ingesters/index.js",
+        "browser": "./dist/browser/ingesters/index.js",
+        "default": "./dist/ingesters/index.js"
+      },
+      "./ingesters/artificial-analysis": {
+        "types": "./dist/ingesters/artificial-analysis.d.ts",
+        "bun": "./dist/ingesters/artificial-analysis.js",
+        "node": "./dist/node/ingesters/artificial-analysis.js",
+        "browser": "./dist/browser/ingesters/artificial-analysis.js",
+        "default": "./dist/ingesters/artificial-analysis.js"
+      },
+      "./ingesters/chatbot-arena": {
+        "types": "./dist/ingesters/chatbot-arena.d.ts",
+        "bun": "./dist/ingesters/chatbot-arena.js",
+        "node": "./dist/node/ingesters/chatbot-arena.js",
+        "browser": "./dist/browser/ingesters/chatbot-arena.js",
+        "default": "./dist/ingesters/chatbot-arena.js"
+      },
+      "./ingesters/fetch-utils": {
+        "types": "./dist/ingesters/fetch-utils.d.ts",
+        "bun": "./dist/ingesters/fetch-utils.js",
+        "node": "./dist/node/ingesters/fetch-utils.js",
+        "browser": "./dist/browser/ingesters/fetch-utils.js",
+        "default": "./dist/ingesters/fetch-utils.js"
+      },
+      "./ingesters/index": {
+        "types": "./dist/ingesters/index.d.ts",
+        "bun": "./dist/ingesters/index.js",
+        "node": "./dist/node/ingesters/index.js",
+        "browser": "./dist/browser/ingesters/index.js",
+        "default": "./dist/ingesters/index.js"
+      },
+      "./ingesters/open-llm-leaderboard": {
+        "types": "./dist/ingesters/open-llm-leaderboard.d.ts",
+        "bun": "./dist/ingesters/open-llm-leaderboard.js",
+        "node": "./dist/node/ingesters/open-llm-leaderboard.js",
+        "browser": "./dist/browser/ingesters/open-llm-leaderboard.js",
+        "default": "./dist/ingesters/open-llm-leaderboard.js"
+      },
+      "./ingesters/registry": {
+        "types": "./dist/ingesters/registry.d.ts",
+        "bun": "./dist/ingesters/registry.js",
+        "node": "./dist/node/ingesters/registry.js",
+        "browser": "./dist/browser/ingesters/registry.js",
+        "default": "./dist/ingesters/registry.js"
+      },
+      "./ingesters/swe-bench": {
+        "types": "./dist/ingesters/swe-bench.d.ts",
+        "bun": "./dist/ingesters/swe-bench.js",
+        "node": "./dist/node/ingesters/swe-bench.js",
+        "browser": "./dist/browser/ingesters/swe-bench.js",
+        "default": "./dist/ingesters/swe-bench.js"
+      },
+      "./ingesters/types": {
+        "types": "./dist/ingesters/types.d.ts",
+        "bun": "./dist/ingesters/types.js",
+        "node": "./dist/node/ingesters/types.js",
+        "browser": "./dist/browser/ingesters/types.js",
+        "default": "./dist/ingesters/types.js"
+      },
+      "./scoring": {
+        "types": "./dist/scoring/index.d.ts",
+        "bun": "./dist/scoring/index.js",
+        "node": "./dist/node/scoring/index.js",
+        "browser": "./dist/browser/scoring/index.js",
+        "default": "./dist/scoring/index.js"
+      },
+      "./scoring/composite-scorer": {
+        "types": "./dist/scoring/composite-scorer.d.ts",
+        "bun": "./dist/scoring/composite-scorer.js",
+        "node": "./dist/node/scoring/composite-scorer.js",
+        "browser": "./dist/browser/scoring/composite-scorer.js",
+        "default": "./dist/scoring/composite-scorer.js"
+      },
+      "./scoring/dimension-weights": {
+        "types": "./dist/scoring/dimension-weights.d.ts",
+        "bun": "./dist/scoring/dimension-weights.js",
+        "node": "./dist/node/scoring/dimension-weights.js",
+        "browser": "./dist/browser/scoring/dimension-weights.js",
+        "default": "./dist/scoring/dimension-weights.js"
+      },
+      "./scoring/index": {
+        "types": "./dist/scoring/index.d.ts",
+        "bun": "./dist/scoring/index.js",
+        "node": "./dist/node/scoring/index.js",
+        "browser": "./dist/browser/scoring/index.js",
+        "default": "./dist/scoring/index.js"
+      },
+      "./scoring/normalizer": {
+        "types": "./dist/scoring/normalizer.d.ts",
+        "bun": "./dist/scoring/normalizer.js",
+        "node": "./dist/node/scoring/normalizer.js",
+        "browser": "./dist/browser/scoring/normalizer.js",
+        "default": "./dist/scoring/normalizer.js"
+      },
+      "./store": {
+        "types": "./dist/store.d.ts",
+        "bun": "./dist/store.js",
+        "node": "./dist/node/store.js",
+        "browser": "./dist/browser/store.js",
+        "default": "./dist/store.js"
+      },
+      "./types": {
+        "types": "./dist/types.d.ts",
+        "bun": "./dist/types.js",
+        "node": "./dist/node/types.js",
+        "browser": "./dist/browser/types.js",
+        "default": "./dist/types.js"
+      }
+    },
+    "registry": "https://registry.npmjs.org/"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/lssm-tech/contractspec.git",
+    "directory": "packages/libs/provider-ranking"
+  },
+  "homepage": "https://contractspec.io"
+}