@contractspec/module.provider-ranking 0.1.2

package/dist/pipeline/ingestion-pipeline.d.ts

@@ -0,0 +1,38 @@
+import type { ProviderRankingStore } from '@contractspec/lib.provider-ranking/store';
+import type { BenchmarkDimension, BenchmarkSource, IngestionRun } from '@contractspec/lib.provider-ranking/types';
+import type { IngesterOptions } from '@contractspec/lib.provider-ranking/ingesters';
+import type { IngesterRegistry } from '@contractspec/lib.provider-ranking/ingesters';
+export interface IngestionPipelineOptions {
+    store: ProviderRankingStore;
+    ingesterRegistry: IngesterRegistry;
+    ingesterOptions?: IngesterOptions;
+}
+export interface IngestParams {
+    fromDate?: string;
+    toDate?: string;
+    dimensions?: BenchmarkDimension[];
+}
+export interface IngestionPipelineResult {
+    ingestionId: string;
+    source: BenchmarkSource;
+    resultsCount: number;
+    status: IngestionRun['status'];
+}
+/**
+ * Orchestrates the full ingestion flow:
+ * 1. Create ingestion run record
+ * 2. Fetch data via the appropriate ingester
+ * 3. Normalize scores
+ * 4. Store results
+ * 5. Update ingestion run status
+ */
+export declare class IngestionPipeline {
+    private readonly store;
+    private readonly registry;
+    private readonly ingesterOptions?;
+    constructor(options: IngestionPipelineOptions);
+    ingest(source: BenchmarkSource, params?: IngestParams): Promise<IngestionPipelineResult>;
+    ingestAll(params?: IngestParams): Promise<IngestionPipelineResult[]>;
+    private mergeOptions;
+    private runIngester;
+}

package/dist/pipeline/ingestion-pipeline.js

@@ -0,0 +1,87 @@
+// @bun
+// src/pipeline/ingestion-pipeline.ts
+import { normalizeBenchmarkResults } from "@contractspec/lib.provider-ranking/scoring";
+
+class IngestionPipeline {
+  store;
+  registry;
+  ingesterOptions;
+  constructor(options) {
+    this.store = options.store;
+    this.registry = options.ingesterRegistry;
+    this.ingesterOptions = options.ingesterOptions;
+  }
+  async ingest(source, params) {
+    const ingester = this.registry.get(source);
+    if (!ingester) {
+      throw new Error(`No ingester registered for source: ${source}`);
+    }
+    return this.runIngester(ingester, params);
+  }
+  async ingestAll(params) {
+    const results = [];
+    for (const ingester of this.registry.list()) {
+      const result = await this.runIngester(ingester, params);
+      results.push(result);
+    }
+    return results;
+  }
+  mergeOptions(params) {
+    const merged = { ...this.ingesterOptions };
+    if (params?.fromDate)
+      merged.fromDate = new Date(params.fromDate);
+    if (params?.toDate)
+      merged.toDate = new Date(params.toDate);
+    if (params?.dimensions?.length)
+      merged.dimensions = params.dimensions;
+    return merged;
+  }
+  async runIngester(ingester, params) {
+    const ingestionId = `ingest-${ingester.source}-${Date.now()}`;
+    const run = {
+      id: ingestionId,
+      source: ingester.source,
+      status: "running",
+      resultsCount: 0,
+      startedAt: new Date,
+      completedAt: null,
+      error: null
+    };
+    await this.store.createIngestionRun(run);
+    try {
+      const opts = this.mergeOptions(params);
+      const rawResults = await ingester.ingest(opts);
+      const normalized = normalizeBenchmarkResults(rawResults);
+      for (const result of normalized) {
+        await this.store.upsertBenchmarkResult(result);
+      }
+      await this.store.updateIngestionRun(ingestionId, {
+        status: "completed",
+        resultsCount: normalized.length,
+        completedAt: new Date
+      });
+      return {
+        ingestionId,
+        source: ingester.source,
+        resultsCount: normalized.length,
+        status: "completed"
+      };
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : String(error);
+      await this.store.updateIngestionRun(ingestionId, {
+        status: "failed",
+        completedAt: new Date,
+        error: errorMessage
+      });
+      return {
+        ingestionId,
+        source: ingester.source,
+        resultsCount: 0,
+        status: "failed"
+      };
+    }
+  }
+}
+export {
+  IngestionPipeline
+};

package/dist/pipeline/ranking-pipeline.d.ts

@@ -0,0 +1,31 @@
+import type { ProviderRankingStore } from '@contractspec/lib.provider-ranking/store';
+import type { BenchmarkDimension, DimensionWeightConfig, ProviderTransportSupport, ProviderAuthSupport } from '@contractspec/lib.provider-ranking/types';
+export interface RankingPipelineOptions {
+    store: ProviderRankingStore;
+}
+export interface RefreshParams {
+    weightOverrides?: DimensionWeightConfig[];
+    dimensions?: BenchmarkDimension[];
+    forceRecalculate?: boolean;
+    /** Filter rankings to models whose provider supports this transport. */
+    requiredTransport?: ProviderTransportSupport;
+    /** Filter rankings to models whose provider supports this auth method. */
+    requiredAuthMethod?: ProviderAuthSupport;
+}
+export interface RankingPipelineResult {
+    modelsRanked: number;
+    updatedAt: Date;
+}
+/**
+ * Orchestrates the ranking refresh flow:
+ * 1. Load all benchmark results from the store
+ * 2. Load existing rankings (for previousRank tracking)
+ * 3. Compute new composite rankings via the scoring engine
+ * 4. Persist updated rankings
+ */
+export declare class RankingPipeline {
+    private readonly store;
+    constructor(options: RankingPipelineOptions);
+    refresh(params?: RefreshParams): Promise<RankingPipelineResult>;
+    private loadAllBenchmarkResults;
+}

package/dist/pipeline/ranking-pipeline.js

@@ -0,0 +1,50 @@
+// @bun
+// src/pipeline/ranking-pipeline.ts
+import { computeModelRankings } from "@contractspec/lib.provider-ranking/scoring";
+
+class RankingPipeline {
+  store;
+  constructor(options) {
+    this.store = options.store;
+  }
+  async refresh(params) {
+    let allResults = await this.loadAllBenchmarkResults();
+    if (params?.dimensions?.length) {
+      const dimSet = new Set(params.dimensions);
+      allResults = allResults.filter((r) => dimSet.has(r.dimension));
+    }
+    const existingRankings = params?.forceRecalculate ? new Map : new Map((await this.store.listModelRankings({
+      limit: 1e4,
+      requiredTransport: params?.requiredTransport,
+      requiredAuthMethod: params?.requiredAuthMethod
+    })).rankings.map((r) => [r.modelId, r]));
+    const newRankings = computeModelRankings(allResults, params?.weightOverrides ? { weightOverrides: params.weightOverrides } : undefined, existingRankings);
+    for (const ranking of newRankings) {
+      await this.store.upsertModelRanking(ranking);
+    }
+    return {
+      modelsRanked: newRankings.length,
+      updatedAt: new Date
+    };
+  }
+  async loadAllBenchmarkResults() {
+    const pageSize = 500;
+    let offset = 0;
+    const allResults = [];
+    while (true) {
+      const page = await this.store.listBenchmarkResults({
+        limit: pageSize,
+        offset
+      });
+      allResults.push(...page.results);
+      if (allResults.length >= page.total || page.results.length < pageSize) {
+        break;
+      }
+      offset += pageSize;
+    }
+    return allResults;
+  }
+}
+export {
+  RankingPipeline
+};

package/dist/storage/index.d.ts

@@ -0,0 +1,30 @@
+import type { ProviderRankingStore } from '@contractspec/lib.provider-ranking/store';
+import type { BenchmarkResult, BenchmarkResultListResult, BenchmarkResultQuery, IngestionRun, ModelProfile, ModelRanking, RankingListResult, RankingQuery } from '@contractspec/lib.provider-ranking/types';
+import type { DatabaseProvider } from '@contractspec/lib.contracts-integrations';
+export interface PostgresProviderRankingStoreOptions {
+    database: DatabaseProvider;
+    schema?: string;
+    createTablesIfMissing?: boolean;
+}
+export declare class PostgresProviderRankingStore implements ProviderRankingStore {
+    private readonly database;
+    private readonly schema;
+    private readonly createTablesIfMissing;
+    private ensured;
+    constructor(options: PostgresProviderRankingStoreOptions);
+    upsertBenchmarkResult(result: BenchmarkResult): Promise<void>;
+    getBenchmarkResult(id: string): Promise<BenchmarkResult | null>;
+    listBenchmarkResults(query: BenchmarkResultQuery): Promise<BenchmarkResultListResult>;
+    upsertModelRanking(ranking: ModelRanking): Promise<void>;
+    getModelRanking(modelId: string): Promise<ModelRanking | null>;
+    listModelRankings(query: RankingQuery): Promise<RankingListResult>;
+    getModelProfile(modelId: string): Promise<ModelProfile | null>;
+    createIngestionRun(run: IngestionRun): Promise<void>;
+    updateIngestionRun(id: string, update: Partial<IngestionRun>): Promise<void>;
+    getIngestionRun(id: string): Promise<IngestionRun | null>;
+    private ensureTables;
+    private table;
+    private mapBenchmarkResult;
+    private mapModelRanking;
+    private mapIngestionRun;
+}

package/dist/storage/index.js

@@ -0,0 +1,300 @@
+// @bun
+// src/storage/index.ts
+class PostgresProviderRankingStore {
+  database;
+  schema;
+  createTablesIfMissing;
+  ensured = false;
+  constructor(options) {
+    this.database = options.database;
+    this.schema = options.schema ?? "lssm_ranking";
+    this.createTablesIfMissing = options.createTablesIfMissing ?? true;
+  }
+  async upsertBenchmarkResult(result) {
+    await this.ensureTables();
+    await this.database.execute(`INSERT INTO ${this.table("benchmark_result")}
+        (id, model_id, provider_key, source, dimension, score, raw_score, metadata, measured_at, ingested_at)
+       VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb, $8::jsonb, $9, $10)
+       ON CONFLICT (id)
+       DO UPDATE SET
+         score = EXCLUDED.score,
+         raw_score = EXCLUDED.raw_score,
+         metadata = EXCLUDED.metadata,
+         measured_at = EXCLUDED.measured_at,
+         ingested_at = EXCLUDED.ingested_at;`, [
+      result.id,
+      result.modelId,
+      result.providerKey,
+      result.source,
+      result.dimension,
+      result.score,
+      JSON.stringify(result.rawScore),
+      result.metadata ? JSON.stringify(result.metadata) : null,
+      result.measuredAt.toISOString(),
+      result.ingestedAt.toISOString()
+    ]);
+  }
+  async getBenchmarkResult(id) {
+    await this.ensureTables();
+    const rows = await this.database.query(`SELECT * FROM ${this.table("benchmark_result")} WHERE id = $1;`, [id]);
+    return rows.rows[0] ? this.mapBenchmarkResult(rows.rows[0]) : null;
+  }
+  async listBenchmarkResults(query) {
+    await this.ensureTables();
+    const limit = query.limit ?? 50;
+    const offset = query.offset ?? 0;
+    const countFilters = [];
+    const countParams = [];
+    if (query.source) {
+      countParams.push(query.source);
+      countFilters.push(`source = $${countParams.length}`);
+    }
+    if (query.modelId) {
+      countParams.push(query.modelId);
+      countFilters.push(`model_id = $${countParams.length}`);
+    }
+    if (query.dimension) {
+      countParams.push(query.dimension);
+      countFilters.push(`dimension = $${countParams.length}`);
+    }
+    if (query.providerKey) {
+      countParams.push(query.providerKey);
+      countFilters.push(`provider_key = $${countParams.length}`);
+    }
+    const where = countFilters.length ? `WHERE ${countFilters.join(" AND ")}` : "";
+    const countResult = await this.database.query(`SELECT COUNT(*)::int as total FROM ${this.table("benchmark_result")} ${where};`, countParams);
+    const total = Number(countResult.rows[0]?.total ?? 0);
+    const dataParams = [
+      limit,
+      offset,
+      ...countParams
+    ];
+    const dataFilters = countFilters.map((_f, i) => _f.replace(`$${i + 1}`, `$${i + 3}`));
+    const dataWhere = dataFilters.length ? `WHERE ${dataFilters.join(" AND ")}` : "";
+    const rows = await this.database.query(`SELECT * FROM ${this.table("benchmark_result")}
+       ${dataWhere}
+       ORDER BY ingested_at DESC
+       LIMIT $1 OFFSET $2;`, dataParams);
+    const results = rows.rows.map((row) => this.mapBenchmarkResult(row));
+    const nextOffset = offset + results.length < total ? offset + results.length : undefined;
+    return { results, total, nextOffset };
+  }
+  async upsertModelRanking(ranking) {
+    await this.ensureTables();
+    await this.database.execute(`INSERT INTO ${this.table("model_ranking")}
+        (model_id, provider_key, composite_score, dimension_scores, rank, previous_rank, updated_at)
+       VALUES ($1, $2, $3, $4::jsonb, $5, $6, $7)
+       ON CONFLICT (model_id)
+       DO UPDATE SET
+         provider_key = EXCLUDED.provider_key,
+         composite_score = EXCLUDED.composite_score,
+         dimension_scores = EXCLUDED.dimension_scores,
+         rank = EXCLUDED.rank,
+         previous_rank = EXCLUDED.previous_rank,
+         updated_at = EXCLUDED.updated_at;`, [
+      ranking.modelId,
+      ranking.providerKey,
+      ranking.compositeScore,
+      JSON.stringify(ranking.dimensionScores),
+      ranking.rank,
+      ranking.previousRank,
+      ranking.updatedAt.toISOString()
+    ]);
+  }
+  async getModelRanking(modelId) {
+    await this.ensureTables();
+    const rows = await this.database.query(`SELECT * FROM ${this.table("model_ranking")} WHERE model_id = $1;`, [modelId]);
+    return rows.rows[0] ? this.mapModelRanking(rows.rows[0]) : null;
+  }
+  async listModelRankings(query) {
+    await this.ensureTables();
+    const limit = query.limit ?? 50;
+    const offset = query.offset ?? 0;
+    const countFilters = [];
+    const countParams = [];
+    if (query.providerKey) {
+      countParams.push(query.providerKey);
+      countFilters.push(`provider_key = $${countParams.length}`);
+    }
+    const where = countFilters.length ? `WHERE ${countFilters.join(" AND ")}` : "";
+    const countResult = await this.database.query(`SELECT COUNT(*)::int as total FROM ${this.table("model_ranking")} ${where};`, countParams);
+    const total = Number(countResult.rows[0]?.total ?? 0);
+    const dataParams = [
+      limit,
+      offset,
+      ...countParams
+    ];
+    const dataFilters = countFilters.map((_f, i) => _f.replace(`$${i + 1}`, `$${i + 3}`));
+    const dataWhere = dataFilters.length ? `WHERE ${dataFilters.join(" AND ")}` : "";
+    const orderBy = query.dimension ? `(dimension_scores->>'${query.dimension}')::jsonb->>'score' DESC NULLS LAST` : "rank ASC";
+    const rows = await this.database.query(`SELECT * FROM ${this.table("model_ranking")}
+       ${dataWhere}
+       ORDER BY ${orderBy}
+       LIMIT $1 OFFSET $2;`, dataParams);
+    const rankings = rows.rows.map((row) => this.mapModelRanking(row));
+    const nextOffset = offset + rankings.length < total ? offset + rankings.length : undefined;
+    return { rankings, total, nextOffset };
+  }
+  async getModelProfile(modelId) {
+    await this.ensureTables();
+    const ranking = await this.getModelRanking(modelId);
+    const benchResults = await this.database.query(`SELECT * FROM ${this.table("benchmark_result")}
+       WHERE model_id = $1
+       ORDER BY ingested_at DESC;`, [modelId]);
+    if (!ranking && benchResults.rows.length === 0)
+      return null;
+    return {
+      modelId,
+      providerKey: ranking?.providerKey ?? String(benchResults.rows[0]?.provider_key ?? "unknown"),
+      displayName: modelId,
+      contextWindow: 0,
+      costPerMillion: null,
+      capabilities: [],
+      ranking: ranking ?? null,
+      benchmarkResults: benchResults.rows.map((row) => this.mapBenchmarkResult(row))
+    };
+  }
+  async createIngestionRun(run) {
+    await this.ensureTables();
+    await this.database.execute(`INSERT INTO ${this.table("ingestion_run")}
+        (id, source, status, results_count, started_at, completed_at, error)
+       VALUES ($1, $2, $3, $4, $5, $6, $7);`, [
+      run.id,
+      run.source,
+      run.status,
+      run.resultsCount,
+      run.startedAt.toISOString(),
+      run.completedAt?.toISOString() ?? null,
+      run.error
+    ]);
+  }
+  async updateIngestionRun(id, update) {
+    await this.ensureTables();
+    const sets = [];
+    const params = [id];
+    if (update.status !== undefined) {
+      params.push(update.status);
+      sets.push(`status = $${params.length}`);
+    }
+    if (update.resultsCount !== undefined) {
+      params.push(update.resultsCount);
+      sets.push(`results_count = $${params.length}`);
+    }
+    if (update.completedAt !== undefined) {
+      params.push(update.completedAt?.toISOString() ?? null);
+      sets.push(`completed_at = $${params.length}`);
+    }
+    if (update.error !== undefined) {
+      params.push(update.error);
+      sets.push(`error = $${params.length}`);
+    }
+    if (sets.length === 0)
+      return;
+    await this.database.execute(`UPDATE ${this.table("ingestion_run")} SET ${sets.join(", ")} WHERE id = $1;`, params);
+  }
+  async getIngestionRun(id) {
+    await this.ensureTables();
+    const rows = await this.database.query(`SELECT * FROM ${this.table("ingestion_run")} WHERE id = $1;`, [id]);
+    return rows.rows[0] ? this.mapIngestionRun(rows.rows[0]) : null;
+  }
+  async ensureTables() {
+    if (this.ensured || !this.createTablesIfMissing)
+      return;
+    await this.database.execute(`CREATE SCHEMA IF NOT EXISTS ${this.schema};`);
+    await this.database.execute(`CREATE TABLE IF NOT EXISTS ${this.table("benchmark_result")} (
+        id text PRIMARY KEY,
+        model_id text NOT NULL,
+        provider_key text NOT NULL,
+        source text NOT NULL,
+        dimension text NOT NULL,
+        score double precision NOT NULL,
+        raw_score jsonb,
+        metadata jsonb,
+        measured_at timestamptz NOT NULL,
+        ingested_at timestamptz NOT NULL
+      );`);
+    await this.database.execute(`CREATE INDEX IF NOT EXISTS benchmark_result_model_idx
+       ON ${this.table("benchmark_result")} (model_id);`);
+    await this.database.execute(`CREATE INDEX IF NOT EXISTS benchmark_result_source_idx
+       ON ${this.table("benchmark_result")} (source);`);
+    await this.database.execute(`CREATE INDEX IF NOT EXISTS benchmark_result_dimension_idx
+       ON ${this.table("benchmark_result")} (dimension);`);
+    await this.database.execute(`CREATE TABLE IF NOT EXISTS ${this.table("model_ranking")} (
+        model_id text PRIMARY KEY,
+        provider_key text NOT NULL,
+        composite_score double precision NOT NULL,
+        dimension_scores jsonb NOT NULL,
+        rank int NOT NULL,
+        previous_rank int,
+        updated_at timestamptz NOT NULL
+      );`);
+    await this.database.execute(`CREATE INDEX IF NOT EXISTS model_ranking_rank_idx
+       ON ${this.table("model_ranking")} (rank);`);
+    await this.database.execute(`CREATE TABLE IF NOT EXISTS ${this.table("ingestion_run")} (
+        id text PRIMARY KEY,
+        source text NOT NULL,
+        status text NOT NULL,
+        results_count int NOT NULL DEFAULT 0,
+        started_at timestamptz NOT NULL,
+        completed_at timestamptz,
+        error text
+      );`);
+    this.ensured = true;
+  }
+  table(name) {
+    return `${this.schema}.${name}`;
+  }
+  mapBenchmarkResult(row) {
+    return {
+      id: String(row.id),
+      modelId: String(row.model_id),
+      providerKey: String(row.provider_key),
+      source: String(row.source),
+      dimension: String(row.dimension),
+      score: Number(row.score),
+      rawScore: parseJson(row.raw_score),
+      metadata: parseJson(row.metadata) ?? {},
+      measuredAt: new Date(String(row.measured_at)),
+      ingestedAt: new Date(String(row.ingested_at))
+    };
+  }
+  mapModelRanking(row) {
+    return {
+      modelId: String(row.model_id),
+      providerKey: String(row.provider_key),
+      compositeScore: Number(row.composite_score),
+      dimensionScores: parseJson(row.dimension_scores) ?? {},
+      rank: Number(row.rank),
+      previousRank: row.previous_rank != null ? Number(row.previous_rank) : null,
+      updatedAt: new Date(String(row.updated_at))
+    };
+  }
+  mapIngestionRun(row) {
+    return {
+      id: String(row.id),
+      source: String(row.source),
+      status: String(row.status),
+      resultsCount: Number(row.results_count),
+      startedAt: new Date(String(row.started_at)),
+      completedAt: row.completed_at ? new Date(String(row.completed_at)) : null,
+      error: row.error ? String(row.error) : null
+    };
+  }
+}
+function parseJson(value) {
+  if (value == null)
+    return null;
+  if (typeof value === "object")
+    return value;
+  if (typeof value === "string") {
+    try {
+      return JSON.parse(value);
+    } catch {
+      return null;
+    }
+  }
+  return value;
+}
+export {
+  PostgresProviderRankingStore
+};