@contractspec/module.provider-ranking 0.1.2

package/dist/entities/index.js

@@ -0,0 +1,108 @@
+// @bun
+// src/entities/index.ts
+import { defineEntity, field, index } from "@contractspec/lib.schema";
+var BenchmarkResultEntity = defineEntity({
+  name: "BenchmarkResult",
+  description: "Individual benchmark score for a model from a specific source.",
+  schema: "lssm_ranking",
+  map: "benchmark_result",
+  fields: {
+    id: field.id({ description: "Benchmark result ID" }),
+    modelId: field.string({ description: "Model identifier" }),
+    providerKey: field.string({
+      description: "Provider key (e.g. openai, anthropic)"
+    }),
+    source: field.string({
+      description: "Benchmark source (e.g. swe-bench, chatbot-arena)"
+    }),
+    dimension: field.string({
+      description: "Ranking dimension (coding, reasoning, etc.)"
+    }),
+    score: field.float({ description: "Normalized score 0-100" }),
+    rawScore: field.json({ description: "Original score from source" }),
+    metadata: field.json({ isOptional: true }),
+    measuredAt: field.dateTime({
+      description: "When the benchmark was measured"
+    }),
+    ingestedAt: field.dateTime({ description: "When the result was ingested" })
+  },
+  indexes: [
+    index.unique(["id"]),
+    index.on(["modelId"]),
+    index.on(["providerKey"]),
+    index.on(["source"]),
+    index.on(["dimension"]),
+    index.on(["modelId", "source", "dimension"])
+  ]
+});
+var ModelRankingEntity = defineEntity({
+  name: "ModelRanking",
+  description: "Computed composite ranking for a model.",
+  schema: "lssm_ranking",
+  map: "model_ranking",
+  fields: {
+    modelId: field.id({ description: "Model identifier (primary key)" }),
+    providerKey: field.string({ description: "Provider key" }),
+    compositeScore: field.float({
+      description: "Weighted composite score 0-100"
+    }),
+    dimensionScores: field.json({
+      description: "Per-dimension score breakdown"
+    }),
+    rank: field.int({ description: "Current rank position" }),
+    previousRank: field.int({
+      isOptional: true,
+      description: "Previous rank position"
+    }),
+    updatedAt: field.updatedAt()
+  },
+  indexes: [
+    index.on(["providerKey"]),
+    index.on(["rank"]),
+    index.on(["compositeScore"])
+  ]
+});
+var IngestionRunEntity = defineEntity({
+  name: "IngestionRun",
+  description: "Tracks a benchmark data ingestion run.",
+  schema: "lssm_ranking",
+  map: "ingestion_run",
+  fields: {
+    id: field.id({ description: "Ingestion run ID" }),
+    source: field.string({ description: "Benchmark source" }),
+    status: field.string({
+      description: "Run status: pending, running, completed, failed"
+    }),
+    resultsCount: field.int({ description: "Number of results ingested" }),
+    startedAt: field.dateTime({ description: "When the run started" }),
+    completedAt: field.dateTime({
+      isOptional: true,
+      description: "When the run completed"
+    }),
+    error: field.string({
+      isOptional: true,
+      description: "Error message if failed"
+    })
+  },
+  indexes: [
+    index.on(["source"]),
+    index.on(["status"]),
+    index.on(["startedAt"])
+  ]
+});
+var providerRankingEntities = [
+  BenchmarkResultEntity,
+  ModelRankingEntity,
+  IngestionRunEntity
+];
+var providerRankingSchemaContribution = {
+  moduleId: "@contractspec/module.provider-ranking",
+  entities: providerRankingEntities
+};
+export {
+  providerRankingSchemaContribution,
+  providerRankingEntities,
+  ModelRankingEntity,
+  IngestionRunEntity,
+  BenchmarkResultEntity
+};

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+export * from './entities';
+export * from './storage';
+export * from './pipeline';

package/dist/index.js

@@ -0,0 +1,539 @@
+// @bun
+// src/entities/index.ts
+import { defineEntity, field, index } from "@contractspec/lib.schema";
+var BenchmarkResultEntity = defineEntity({
+  name: "BenchmarkResult",
+  description: "Individual benchmark score for a model from a specific source.",
+  schema: "lssm_ranking",
+  map: "benchmark_result",
+  fields: {
+    id: field.id({ description: "Benchmark result ID" }),
+    modelId: field.string({ description: "Model identifier" }),
+    providerKey: field.string({
+      description: "Provider key (e.g. openai, anthropic)"
+    }),
+    source: field.string({
+      description: "Benchmark source (e.g. swe-bench, chatbot-arena)"
+    }),
+    dimension: field.string({
+      description: "Ranking dimension (coding, reasoning, etc.)"
+    }),
+    score: field.float({ description: "Normalized score 0-100" }),
+    rawScore: field.json({ description: "Original score from source" }),
+    metadata: field.json({ isOptional: true }),
+    measuredAt: field.dateTime({
+      description: "When the benchmark was measured"
+    }),
+    ingestedAt: field.dateTime({ description: "When the result was ingested" })
+  },
+  indexes: [
+    index.unique(["id"]),
+    index.on(["modelId"]),
+    index.on(["providerKey"]),
+    index.on(["source"]),
+    index.on(["dimension"]),
+    index.on(["modelId", "source", "dimension"])
+  ]
+});
+var ModelRankingEntity = defineEntity({
+  name: "ModelRanking",
+  description: "Computed composite ranking for a model.",
+  schema: "lssm_ranking",
+  map: "model_ranking",
+  fields: {
+    modelId: field.id({ description: "Model identifier (primary key)" }),
+    providerKey: field.string({ description: "Provider key" }),
+    compositeScore: field.float({
+      description: "Weighted composite score 0-100"
+    }),
+    dimensionScores: field.json({
+      description: "Per-dimension score breakdown"
+    }),
+    rank: field.int({ description: "Current rank position" }),
+    previousRank: field.int({
+      isOptional: true,
+      description: "Previous rank position"
+    }),
+    updatedAt: field.updatedAt()
+  },
+  indexes: [
+    index.on(["providerKey"]),
+    index.on(["rank"]),
+    index.on(["compositeScore"])
+  ]
+});
+var IngestionRunEntity = defineEntity({
+  name: "IngestionRun",
+  description: "Tracks a benchmark data ingestion run.",
+  schema: "lssm_ranking",
+  map: "ingestion_run",
+  fields: {
+    id: field.id({ description: "Ingestion run ID" }),
+    source: field.string({ description: "Benchmark source" }),
+    status: field.string({
+      description: "Run status: pending, running, completed, failed"
+    }),
+    resultsCount: field.int({ description: "Number of results ingested" }),
+    startedAt: field.dateTime({ description: "When the run started" }),
+    completedAt: field.dateTime({
+      isOptional: true,
+      description: "When the run completed"
+    }),
+    error: field.string({
+      isOptional: true,
+      description: "Error message if failed"
+    })
+  },
+  indexes: [
+    index.on(["source"]),
+    index.on(["status"]),
+    index.on(["startedAt"])
+  ]
+});
+var providerRankingEntities = [
+  BenchmarkResultEntity,
+  ModelRankingEntity,
+  IngestionRunEntity
+];
+var providerRankingSchemaContribution = {
+  moduleId: "@contractspec/module.provider-ranking",
+  entities: providerRankingEntities
+};
+
+// src/storage/index.ts
+class PostgresProviderRankingStore {
+  database;
+  schema;
+  createTablesIfMissing;
+  ensured = false;
+  constructor(options) {
+    this.database = options.database;
+    this.schema = options.schema ?? "lssm_ranking";
+    this.createTablesIfMissing = options.createTablesIfMissing ?? true;
+  }
+  async upsertBenchmarkResult(result) {
+    await this.ensureTables();
+    await this.database.execute(`INSERT INTO ${this.table("benchmark_result")}
+        (id, model_id, provider_key, source, dimension, score, raw_score, metadata, measured_at, ingested_at)
+       VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb, $8::jsonb, $9, $10)
+       ON CONFLICT (id)
+       DO UPDATE SET
+         score = EXCLUDED.score,
+         raw_score = EXCLUDED.raw_score,
+         metadata = EXCLUDED.metadata,
+         measured_at = EXCLUDED.measured_at,
+         ingested_at = EXCLUDED.ingested_at;`, [
+      result.id,
+      result.modelId,
+      result.providerKey,
+      result.source,
+      result.dimension,
+      result.score,
+      JSON.stringify(result.rawScore),
+      result.metadata ? JSON.stringify(result.metadata) : null,
+      result.measuredAt.toISOString(),
+      result.ingestedAt.toISOString()
+    ]);
+  }
+  async getBenchmarkResult(id) {
+    await this.ensureTables();
+    const rows = await this.database.query(`SELECT * FROM ${this.table("benchmark_result")} WHERE id = $1;`, [id]);
+    return rows.rows[0] ? this.mapBenchmarkResult(rows.rows[0]) : null;
+  }
+  async listBenchmarkResults(query) {
+    await this.ensureTables();
+    const limit = query.limit ?? 50;
+    const offset = query.offset ?? 0;
+    const countFilters = [];
+    const countParams = [];
+    if (query.source) {
+      countParams.push(query.source);
+      countFilters.push(`source = $${countParams.length}`);
+    }
+    if (query.modelId) {
+      countParams.push(query.modelId);
+      countFilters.push(`model_id = $${countParams.length}`);
+    }
+    if (query.dimension) {
+      countParams.push(query.dimension);
+      countFilters.push(`dimension = $${countParams.length}`);
+    }
+    if (query.providerKey) {
+      countParams.push(query.providerKey);
+      countFilters.push(`provider_key = $${countParams.length}`);
+    }
+    const where = countFilters.length ? `WHERE ${countFilters.join(" AND ")}` : "";
+    const countResult = await this.database.query(`SELECT COUNT(*)::int as total FROM ${this.table("benchmark_result")} ${where};`, countParams);
+    const total = Number(countResult.rows[0]?.total ?? 0);
+    const dataParams = [
+      limit,
+      offset,
+      ...countParams
+    ];
+    const dataFilters = countFilters.map((_f, i) => _f.replace(`$${i + 1}`, `$${i + 3}`));
+    const dataWhere = dataFilters.length ? `WHERE ${dataFilters.join(" AND ")}` : "";
+    const rows = await this.database.query(`SELECT * FROM ${this.table("benchmark_result")}
+       ${dataWhere}
+       ORDER BY ingested_at DESC
+       LIMIT $1 OFFSET $2;`, dataParams);
+    const results = rows.rows.map((row) => this.mapBenchmarkResult(row));
+    const nextOffset = offset + results.length < total ? offset + results.length : undefined;
+    return { results, total, nextOffset };
+  }
+  async upsertModelRanking(ranking) {
+    await this.ensureTables();
+    await this.database.execute(`INSERT INTO ${this.table("model_ranking")}
+        (model_id, provider_key, composite_score, dimension_scores, rank, previous_rank, updated_at)
+       VALUES ($1, $2, $3, $4::jsonb, $5, $6, $7)
+       ON CONFLICT (model_id)
+       DO UPDATE SET
+         provider_key = EXCLUDED.provider_key,
+         composite_score = EXCLUDED.composite_score,
+         dimension_scores = EXCLUDED.dimension_scores,
+         rank = EXCLUDED.rank,
+         previous_rank = EXCLUDED.previous_rank,
+         updated_at = EXCLUDED.updated_at;`, [
+      ranking.modelId,
+      ranking.providerKey,
+      ranking.compositeScore,
+      JSON.stringify(ranking.dimensionScores),
+      ranking.rank,
+      ranking.previousRank,
+      ranking.updatedAt.toISOString()
+    ]);
+  }
+  async getModelRanking(modelId) {
+    await this.ensureTables();
+    const rows = await this.database.query(`SELECT * FROM ${this.table("model_ranking")} WHERE model_id = $1;`, [modelId]);
+    return rows.rows[0] ? this.mapModelRanking(rows.rows[0]) : null;
+  }
+  async listModelRankings(query) {
+    await this.ensureTables();
+    const limit = query.limit ?? 50;
+    const offset = query.offset ?? 0;
+    const countFilters = [];
+    const countParams = [];
+    if (query.providerKey) {
+      countParams.push(query.providerKey);
+      countFilters.push(`provider_key = $${countParams.length}`);
+    }
+    const where = countFilters.length ? `WHERE ${countFilters.join(" AND ")}` : "";
+    const countResult = await this.database.query(`SELECT COUNT(*)::int as total FROM ${this.table("model_ranking")} ${where};`, countParams);
+    const total = Number(countResult.rows[0]?.total ?? 0);
+    const dataParams = [
+      limit,
+      offset,
+      ...countParams
+    ];
+    const dataFilters = countFilters.map((_f, i) => _f.replace(`$${i + 1}`, `$${i + 3}`));
+    const dataWhere = dataFilters.length ? `WHERE ${dataFilters.join(" AND ")}` : "";
+    const orderBy = query.dimension ? `(dimension_scores->>'${query.dimension}')::jsonb->>'score' DESC NULLS LAST` : "rank ASC";
+    const rows = await this.database.query(`SELECT * FROM ${this.table("model_ranking")}
+       ${dataWhere}
+       ORDER BY ${orderBy}
+       LIMIT $1 OFFSET $2;`, dataParams);
+    const rankings = rows.rows.map((row) => this.mapModelRanking(row));
+    const nextOffset = offset + rankings.length < total ? offset + rankings.length : undefined;
+    return { rankings, total, nextOffset };
+  }
+  async getModelProfile(modelId) {
+    await this.ensureTables();
+    const ranking = await this.getModelRanking(modelId);
+    const benchResults = await this.database.query(`SELECT * FROM ${this.table("benchmark_result")}
+       WHERE model_id = $1
+       ORDER BY ingested_at DESC;`, [modelId]);
+    if (!ranking && benchResults.rows.length === 0)
+      return null;
+    return {
+      modelId,
+      providerKey: ranking?.providerKey ?? String(benchResults.rows[0]?.provider_key ?? "unknown"),
+      displayName: modelId,
+      contextWindow: 0,
+      costPerMillion: null,
+      capabilities: [],
+      ranking: ranking ?? null,
+      benchmarkResults: benchResults.rows.map((row) => this.mapBenchmarkResult(row))
+    };
+  }
+  async createIngestionRun(run) {
+    await this.ensureTables();
+    await this.database.execute(`INSERT INTO ${this.table("ingestion_run")}
+        (id, source, status, results_count, started_at, completed_at, error)
+       VALUES ($1, $2, $3, $4, $5, $6, $7);`, [
+      run.id,
+      run.source,
+      run.status,
+      run.resultsCount,
+      run.startedAt.toISOString(),
+      run.completedAt?.toISOString() ?? null,
+      run.error
+    ]);
+  }
+  async updateIngestionRun(id, update) {
+    await this.ensureTables();
+    const sets = [];
+    const params = [id];
+    if (update.status !== undefined) {
+      params.push(update.status);
+      sets.push(`status = $${params.length}`);
+    }
+    if (update.resultsCount !== undefined) {
+      params.push(update.resultsCount);
+      sets.push(`results_count = $${params.length}`);
+    }
+    if (update.completedAt !== undefined) {
+      params.push(update.completedAt?.toISOString() ?? null);
+      sets.push(`completed_at = $${params.length}`);
+    }
+    if (update.error !== undefined) {
+      params.push(update.error);
+      sets.push(`error = $${params.length}`);
+    }
+    if (sets.length === 0)
+      return;
+    await this.database.execute(`UPDATE ${this.table("ingestion_run")} SET ${sets.join(", ")} WHERE id = $1;`, params);
+  }
+  async getIngestionRun(id) {
+    await this.ensureTables();
+    const rows = await this.database.query(`SELECT * FROM ${this.table("ingestion_run")} WHERE id = $1;`, [id]);
+    return rows.rows[0] ? this.mapIngestionRun(rows.rows[0]) : null;
+  }
+  async ensureTables() {
+    if (this.ensured || !this.createTablesIfMissing)
+      return;
+    await this.database.execute(`CREATE SCHEMA IF NOT EXISTS ${this.schema};`);
+    await this.database.execute(`CREATE TABLE IF NOT EXISTS ${this.table("benchmark_result")} (
+        id text PRIMARY KEY,
+        model_id text NOT NULL,
+        provider_key text NOT NULL,
+        source text NOT NULL,
+        dimension text NOT NULL,
+        score double precision NOT NULL,
+        raw_score jsonb,
+        metadata jsonb,
+        measured_at timestamptz NOT NULL,
+        ingested_at timestamptz NOT NULL
+      );`);
+    await this.database.execute(`CREATE INDEX IF NOT EXISTS benchmark_result_model_idx
+       ON ${this.table("benchmark_result")} (model_id);`);
+    await this.database.execute(`CREATE INDEX IF NOT EXISTS benchmark_result_source_idx
+       ON ${this.table("benchmark_result")} (source);`);
+    await this.database.execute(`CREATE INDEX IF NOT EXISTS benchmark_result_dimension_idx
+       ON ${this.table("benchmark_result")} (dimension);`);
+    await this.database.execute(`CREATE TABLE IF NOT EXISTS ${this.table("model_ranking")} (
+        model_id text PRIMARY KEY,
+        provider_key text NOT NULL,
+        composite_score double precision NOT NULL,
+        dimension_scores jsonb NOT NULL,
+        rank int NOT NULL,
+        previous_rank int,
+        updated_at timestamptz NOT NULL
+      );`);
+    await this.database.execute(`CREATE INDEX IF NOT EXISTS model_ranking_rank_idx
+       ON ${this.table("model_ranking")} (rank);`);
+    await this.database.execute(`CREATE TABLE IF NOT EXISTS ${this.table("ingestion_run")} (
+        id text PRIMARY KEY,
+        source text NOT NULL,
+        status text NOT NULL,
+        results_count int NOT NULL DEFAULT 0,
+        started_at timestamptz NOT NULL,
+        completed_at timestamptz,
+        error text
+      );`);
+    this.ensured = true;
+  }
+  table(name) {
+    return `${this.schema}.${name}`;
+  }
+  mapBenchmarkResult(row) {
+    return {
+      id: String(row.id),
+      modelId: String(row.model_id),
+      providerKey: String(row.provider_key),
+      source: String(row.source),
+      dimension: String(row.dimension),
+      score: Number(row.score),
+      rawScore: parseJson(row.raw_score),
+      metadata: parseJson(row.metadata) ?? {},
+      measuredAt: new Date(String(row.measured_at)),
+      ingestedAt: new Date(String(row.ingested_at))
+    };
+  }
+  mapModelRanking(row) {
+    return {
+      modelId: String(row.model_id),
+      providerKey: String(row.provider_key),
+      compositeScore: Number(row.composite_score),
+      dimensionScores: parseJson(row.dimension_scores) ?? {},
+      rank: Number(row.rank),
+      previousRank: row.previous_rank != null ? Number(row.previous_rank) : null,
+      updatedAt: new Date(String(row.updated_at))
+    };
+  }
+  mapIngestionRun(row) {
+    return {
+      id: String(row.id),
+      source: String(row.source),
+      status: String(row.status),
+      resultsCount: Number(row.results_count),
+      startedAt: new Date(String(row.started_at)),
+      completedAt: row.completed_at ? new Date(String(row.completed_at)) : null,
+      error: row.error ? String(row.error) : null
+    };
+  }
+}
+function parseJson(value) {
+  if (value == null)
+    return null;
+  if (typeof value === "object")
+    return value;
+  if (typeof value === "string") {
+    try {
+      return JSON.parse(value);
+    } catch {
+      return null;
+    }
+  }
+  return value;
+}
+
+// src/pipeline/ingestion-pipeline.ts
+import { normalizeBenchmarkResults } from "@contractspec/lib.provider-ranking/scoring";
+
+class IngestionPipeline {
+  store;
+  registry;
+  ingesterOptions;
+  constructor(options) {
+    this.store = options.store;
+    this.registry = options.ingesterRegistry;
+    this.ingesterOptions = options.ingesterOptions;
+  }
+  async ingest(source, params) {
+    const ingester = this.registry.get(source);
+    if (!ingester) {
+      throw new Error(`No ingester registered for source: ${source}`);
+    }
+    return this.runIngester(ingester, params);
+  }
+  async ingestAll(params) {
+    const results = [];
+    for (const ingester of this.registry.list()) {
+      const result = await this.runIngester(ingester, params);
+      results.push(result);
+    }
+    return results;
+  }
+  mergeOptions(params) {
+    const merged = { ...this.ingesterOptions };
+    if (params?.fromDate)
+      merged.fromDate = new Date(params.fromDate);
+    if (params?.toDate)
+      merged.toDate = new Date(params.toDate);
+    if (params?.dimensions?.length)
+      merged.dimensions = params.dimensions;
+    return merged;
+  }
+  async runIngester(ingester, params) {
+    const ingestionId = `ingest-${ingester.source}-${Date.now()}`;
+    const run = {
+      id: ingestionId,
+      source: ingester.source,
+      status: "running",
+      resultsCount: 0,
+      startedAt: new Date,
+      completedAt: null,
+      error: null
+    };
+    await this.store.createIngestionRun(run);
+    try {
+      const opts = this.mergeOptions(params);
+      const rawResults = await ingester.ingest(opts);
+      const normalized = normalizeBenchmarkResults(rawResults);
+      for (const result of normalized) {
+        await this.store.upsertBenchmarkResult(result);
+      }
+      await this.store.updateIngestionRun(ingestionId, {
+        status: "completed",
+        resultsCount: normalized.length,
+        completedAt: new Date
+      });
+      return {
+        ingestionId,
+        source: ingester.source,
+        resultsCount: normalized.length,
+        status: "completed"
+      };
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : String(error);
+      await this.store.updateIngestionRun(ingestionId, {
+        status: "failed",
+        completedAt: new Date,
+        error: errorMessage
+      });
+      return {
+        ingestionId,
+        source: ingester.source,
+        resultsCount: 0,
+        status: "failed"
+      };
+    }
+  }
+}
+
+// src/pipeline/ranking-pipeline.ts
+import { computeModelRankings } from "@contractspec/lib.provider-ranking/scoring";
+
+class RankingPipeline {
+  store;
+  constructor(options) {
+    this.store = options.store;
+  }
+  async refresh(params) {
+    let allResults = await this.loadAllBenchmarkResults();
+    if (params?.dimensions?.length) {
+      const dimSet = new Set(params.dimensions);
+      allResults = allResults.filter((r) => dimSet.has(r.dimension));
+    }
+    const existingRankings = params?.forceRecalculate ? new Map : new Map((await this.store.listModelRankings({
+      limit: 1e4,
+      requiredTransport: params?.requiredTransport,
+      requiredAuthMethod: params?.requiredAuthMethod
+    })).rankings.map((r) => [r.modelId, r]));
+    const newRankings = computeModelRankings(allResults, params?.weightOverrides ? { weightOverrides: params.weightOverrides } : undefined, existingRankings);
+    for (const ranking of newRankings) {
+      await this.store.upsertModelRanking(ranking);
+    }
+    return {
+      modelsRanked: newRankings.length,
+      updatedAt: new Date
+    };
+  }
+  async loadAllBenchmarkResults() {
+    const pageSize = 500;
+    let offset = 0;
+    const allResults = [];
+    while (true) {
+      const page = await this.store.listBenchmarkResults({
+        limit: pageSize,
+        offset
+      });
+      allResults.push(...page.results);
+      if (allResults.length >= page.total || page.results.length < pageSize) {
+        break;
+      }
+      offset += pageSize;
+    }
+    return allResults;
+  }
+}
+export {
+  providerRankingSchemaContribution,
+  providerRankingEntities,
+  RankingPipeline,
+  PostgresProviderRankingStore,
+  ModelRankingEntity,
+  IngestionRunEntity,
+  IngestionPipeline,
+  BenchmarkResultEntity
+};