@tangle-network/agent-eval 0.120.2 → 0.120.3

package/dist/chunk-CVJP5TMD.js

@@ -0,0 +1,766 @@
+import {
+  fsCampaignStorage,
+  runCampaign
+} from "./chunk-SYV364BL.js";
+import {
+  __export
+} from "./chunk-PZ5AY32C.js";
+
+// src/benchmarks/index.ts
+var benchmarks_exports = {};
+__export(benchmarks_exports, {
+  BENCHMARK_SPLIT_SEED: () => BENCHMARK_SPLIT_SEED,
+  buildStandardRetrievalItems: () => buildStandardRetrievalItems,
+  calibrateBenchmarkMetric: () => calibrateBenchmarkMetric,
+  createRetrievalIdBenchmarkAdapter: () => createRetrievalIdBenchmarkAdapter,
+  deterministicSplit: () => deterministicSplit,
+  evaluateStandardRetrieval: () => evaluateStandardRetrieval,
+  normalizeRetrievedDocumentIds: () => normalizeRetrievedDocumentIds,
+  parseBeirCorpusJsonl: () => parseBeirCorpusJsonl,
+  parseBeirQueriesJsonl: () => parseBeirQueriesJsonl,
+  parseJsonlRows: () => parseJsonlRows,
+  parseQrels: () => parseQrels,
+  parseTsvRows: () => parseTsvRows,
+  renderBenchmarkReportMarkdown: () => renderBenchmarkReportMarkdown,
+  retrievalMetricsAtCutoff: () => retrievalMetricsAtCutoff,
+  routing: () => routing_exports,
+  runBenchmarkAdapter: () => runBenchmarkAdapter,
+  summarizeBenchmarkCampaign: () => summarizeBenchmarkCampaign
+});
+
+// src/benchmarks/calibration.ts
+async function calibrateBenchmarkMetric(options) {
+  const weak = await options.adapter.evaluate(options.item, options.weakArtifact);
+  const strong = await options.adapter.evaluate(options.item, options.strongArtifact);
+  const weakScore = clamp01(weak.score);
+  const strongScore = clamp01(strong.score);
+  const maxWeakScore = options.maxWeakScore ?? 0.3;
+  const minStrongScore = options.minStrongScore ?? 0.7;
+  const minGap = options.minGap ?? 0.4;
+  const gap = strongScore - weakScore;
+  const reasons = [
+    weakScore <= maxWeakScore ? void 0 : `weak score ${weakScore.toFixed(3)} exceeds max ${maxWeakScore.toFixed(3)}`,
+    strongScore >= minStrongScore ? void 0 : `strong score ${strongScore.toFixed(3)} below min ${minStrongScore.toFixed(3)}`,
+    gap >= minGap ? void 0 : `gap ${gap.toFixed(3)} below min ${minGap.toFixed(3)}`
+  ].filter((reason) => Boolean(reason));
+  return {
+    passed: reasons.length === 0,
+    weak,
+    strong,
+    weakScore,
+    strongScore,
+    gap,
+    reasons
+  };
+}
+function clamp01(value) {
+  if (!Number.isFinite(value)) return 0;
+  if (value < 0) return 0;
+  if (value > 1) return 1;
+  return value;
+}
+
+// src/benchmarks/routing/index.ts
+var routing_exports = {};
+__export(routing_exports, {
+  ROUTING_DATASET: () => ROUTING_DATASET,
+  RoutingAdapter: () => RoutingAdapter,
+  assignSplit: () => assignSplit,
+  evaluate: () => evaluate,
+  extractRouteTokens: () => extractRouteTokens,
+  loadDataset: () => loadDataset
+});
+
+// src/benchmarks/types.ts
+function fnv1a32(input) {
+  let h = 2166136261;
+  for (let i = 0; i < input.length; i++) {
+    h ^= input.charCodeAt(i) & 255;
+    h = h + ((h << 1) + (h << 4) + (h << 7) + (h << 8) + (h << 24)) >>> 0;
+  }
+  return h >>> 0;
+}
+var BENCHMARK_SPLIT_SEED = "agent-eval-v1";
+function deterministicSplit(itemId, seed = BENCHMARK_SPLIT_SEED) {
+  const h = fnv1a32(`${seed}::${itemId}`);
+  const pos = h / 4294967296;
+  if (pos < 0.6) return "search";
+  if (pos < 0.8) return "dev";
+  return "holdout";
+}
+
+// src/benchmarks/routing/dataset.ts
+var ROUTING_DATASET = [
+  {
+    id: "file_001",
+    category: "file",
+    prompt: "Save the meeting notes to /tmp/notes-2025-04.md as markdown.",
+    route: "fs.write",
+    synonyms: ["filesystem.write", "write_file"],
+    hardNegatives: ["fs.read", "chat.reply"]
+  },
+  {
+    id: "file_002",
+    category: "file",
+    prompt: "Read the contents of /etc/hosts and summarize the entries.",
+    route: "fs.read",
+    synonyms: ["filesystem.read", "read_file"],
+    hardNegatives: ["fs.write", "search.web"]
+  },
+  {
+    id: "file_003",
+    category: "file",
+    prompt: "List every Python file under src/ recursively.",
+    route: "fs.list",
+    synonyms: ["filesystem.list", "list_files"],
+    hardNegatives: ["fs.read", "search.code"]
+  },
+  {
+    id: "file_004",
+    category: "file",
+    prompt: "Delete the cached build at .turbo/cache.",
+    route: "fs.delete",
+    synonyms: ["filesystem.delete", "remove_file"],
+    hardNegatives: ["fs.write", "fs.list"]
+  },
+  {
+    id: "math_001",
+    category: "math",
+    prompt: "What is the integral of 3x^2 + 2x from 0 to 5?",
+    route: "math.integral",
+    synonyms: ["calculator.integral", "math.solve"],
+    hardNegatives: ["math.derivative", "chat.reply"]
+  },
+  {
+    id: "math_002",
+    category: "math",
+    prompt: "Compute the derivative of sin(x) * cos(x).",
+    route: "math.derivative",
+    synonyms: ["calculator.derivative", "math.solve"],
+    hardNegatives: ["math.integral", "math.algebra"]
+  },
+  {
+    id: "math_003",
+    category: "math",
+    prompt: "Solve 2x + 7 = 19 for x.",
+    route: "math.algebra",
+    synonyms: ["calculator.algebra", "math.solve"],
+    hardNegatives: ["math.derivative", "math.integral"]
+  },
+  {
+    id: "math_004",
+    category: "math",
+    prompt: "What is the prime factorization of 360?",
+    route: "math.numbertheory",
+    synonyms: ["calculator.factor", "math.solve"],
+    hardNegatives: ["math.algebra", "search.web"]
+  },
+  {
+    id: "search_001",
+    category: "search",
+    prompt: "Find recent papers on agent prompt optimization with held-out promotion gates.",
+    route: "search.web",
+    synonyms: ["web.search", "search.papers"],
+    hardNegatives: ["search.code", "chat.reply"]
+  },
+  {
+    id: "search_002",
+    category: "search",
+    prompt: "Search the codebase for every call site of `runProposeReview`.",
+    route: "search.code",
+    synonyms: ["code.search", "grep"],
+    hardNegatives: ["search.web", "fs.read"]
+  },
+  {
+    id: "search_003",
+    category: "search",
+    prompt: "What is the latest release of the Tangle network on GitHub?",
+    route: "search.web",
+    synonyms: ["web.search", "github.releases"],
+    hardNegatives: ["search.code", "chat.reply"]
+  },
+  {
+    id: "search_004",
+    category: "search",
+    prompt: "Find all TODO comments in the agent-eval src tree.",
+    route: "search.code",
+    synonyms: ["code.search", "grep"],
+    hardNegatives: ["search.web", "fs.list"]
+  },
+  {
+    id: "chat_001",
+    category: "chat",
+    prompt: "Hi there, how are you doing today?",
+    route: "chat.reply",
+    synonyms: ["conversation.reply"],
+    hardNegatives: ["search.web", "fs.read"]
+  },
+  {
+    id: "chat_002",
+    category: "chat",
+    prompt: "Please explain the difference between an LLM and a foundation model.",
+    route: "chat.reply",
+    synonyms: ["conversation.reply", "qa.answer"],
+    hardNegatives: ["search.web", "math.algebra"]
+  },
+  {
+    id: "chat_003",
+    category: "chat",
+    prompt: "Tell me a short joke about distributed systems.",
+    route: "chat.reply",
+    synonyms: ["conversation.reply"],
+    hardNegatives: ["search.web", "fs.read"]
+  },
+  {
+    id: "chat_004",
+    category: "chat",
+    prompt: "Acknowledge my last message with a thumbs up.",
+    route: "chat.reply",
+    synonyms: ["conversation.reply", "react"],
+    hardNegatives: ["fs.write", "search.web"]
+  }
+];
+
+// src/benchmarks/routing/index.ts
+var RoutingAdapter = class {
+  id = "first-party/routing";
+  family = "first-party";
+  taskKind = "routing";
+  description = "Synthetic fixed-route classification smoke benchmark";
+  defaultMetric = "route_exact_match";
+  async loadDataset(split) {
+    return ROUTING_DATASET.map((item) => ({ id: item.id, payload: item })).filter(
+      (it) => assignSplitImpl(it.id) === split
+    );
+  }
+  async evaluate(item, response) {
+    const tokens = extractRouteTokens(response);
+    const correct = new Set(
+      [item.payload.route, ...item.payload.synonyms].map((s) => s.toLowerCase())
+    );
+    const hardNeg = new Set(item.payload.hardNegatives.map((s) => s.toLowerCase()));
+    const firstMatch = tokens.find((t) => correct.has(t.toLowerCase())) ?? null;
+    const firstHardNeg = tokens.find((t) => hardNeg.has(t.toLowerCase())) ?? null;
+    const score = firstMatch ? 1 : 0;
+    return {
+      score,
+      raw: {
+        firstToken: tokens[0] ?? null,
+        matchedRoute: firstMatch,
+        hitHardNegative: Boolean(firstHardNeg),
+        hardNegativeRoute: firstHardNeg,
+        category: item.payload.category
+      }
+    };
+  }
+  assignSplit(itemId) {
+    return assignSplitImpl(itemId);
+  }
+};
+function assignSplitImpl(itemId) {
+  return deterministicSplit(`routing::${itemId}`);
+}
+function extractRouteTokens(response) {
+  const matches = response.match(/[a-z][a-z0-9_]*\.[a-z][a-z0-9_]*/gi);
+  return matches ?? [];
+}
+var adapter = new RoutingAdapter();
+var loadDataset = adapter.loadDataset.bind(adapter);
+var evaluate = adapter.evaluate.bind(adapter);
+var assignSplit = adapter.assignSplit.bind(adapter);
+
+// src/benchmarks/runner.ts
+import { join } from "path";
+async function runBenchmarkAdapter(options) {
+  const storage = options.storage ?? fsCampaignStorage();
+  const benchmarkId = benchmarkIdFor(options.adapter);
+  const scenarios = await loadBenchmarkScenarios(options.adapter, options.splits);
+  const judge = benchmarkAdapterJudge(options.adapter);
+  const dispatch = async (scenario, context) => {
+    return options.respond({ scenario, item: scenario.item, context });
+  };
+  const campaign = await runCampaign({
+    scenarios,
+    dispatch,
+    dispatchRef: `benchmark:${benchmarkId}`,
+    judges: [judge],
+    seed: options.seed,
+    reps: options.reps,
+    resumable: options.resumable,
+    costCeiling: options.costCeiling,
+    maxConcurrency: options.maxConcurrency,
+    dispatchTimeoutMs: options.dispatchTimeoutMs,
+    expectUsage: options.expectUsage ?? "off",
+    runDir: options.runDir,
+    repo: options.repo,
+    storage,
+    now: options.now
+  });
+  const report = summarizeBenchmarkCampaign({
+    adapter: options.adapter,
+    scenarios,
+    campaign
+  });
+  storage.ensureDir(campaign.runDir);
+  const reportJsonPath = join(campaign.runDir, "benchmark-report.json");
+  const reportMarkdownPath = join(campaign.runDir, "benchmark-report.md");
+  storage.write(reportJsonPath, `${JSON.stringify(report, null, 2)}
+`);
+  storage.write(reportMarkdownPath, renderBenchmarkReportMarkdown(report));
+  return { scenarios, campaign, report, reportJsonPath, reportMarkdownPath };
+}
+async function loadBenchmarkScenarios(adapter2, splits = ["search", "dev", "holdout"]) {
+  const benchmarkId = benchmarkIdFor(adapter2);
+  const family = adapter2.family ?? "custom";
+  const taskKind = adapter2.taskKind ?? "custom";
+  const out = [];
+  const seen = /* @__PURE__ */ new Set();
+  for (const split of splits) {
+    const items = await adapter2.loadDataset(split);
+    for (const item of items) {
+      const splitTag = item.split ?? adapter2.assignSplit(item.id);
+      if (splitTag !== split) continue;
+      const key = `${benchmarkId}:${item.id}`;
+      if (seen.has(key)) continue;
+      seen.add(key);
+      out.push({
+        id: key,
+        kind: "benchmark",
+        benchmarkId,
+        family: item.family ?? family,
+        taskKind: item.taskKind ?? taskKind,
+        splitTag,
+        tags: [.../* @__PURE__ */ new Set([splitTag, ...item.tags ?? []])],
+        item
+      });
+    }
+  }
+  return out;
+}
+function benchmarkAdapterJudge(adapter2) {
+  const benchmarkId = benchmarkIdFor(adapter2);
+  return {
+    name: `${benchmarkId}:score`,
+    dimensions: [
+      { key: "score", description: `Primary ${benchmarkId} benchmark score` },
+      { key: "passed", description: "Binary pass projection for aggregate reporting" }
+    ],
+    appliesTo: (scenario) => {
+      return scenario.kind === "benchmark" && scenario.id.startsWith(`${benchmarkId}:`);
+    },
+    async score({ artifact, scenario }) {
+      const evaluation = await adapter2.evaluate(scenario.item, artifact);
+      const dimensions = normalizeEvaluationDimensions(evaluation);
+      return {
+        dimensions,
+        composite: clamp012(evaluation.score),
+        notes: evaluation.notes ?? ""
+      };
+    }
+  };
+}
+function summarizeBenchmarkCampaign(input) {
+  const scenarioById = new Map(input.scenarios.map((scenario) => [scenario.id, scenario]));
+  const rows = input.campaign.cells.map((cell) => {
+    const scenario = scenarioById.get(cell.scenarioId);
+    const judge = firstJudgeScore(cell.judgeScores);
+    const score = judge?.composite ?? 0;
+    return {
+      cell,
+      scenario,
+      score,
+      passed: (judge?.dimensions.passed ?? (score > 0 ? 1 : 0)) >= 1,
+      dimensions: judge?.dimensions ?? {}
+    };
+  });
+  const successful = rows.filter((row) => !row.cell.error);
+  const scoreValues = successful.map((row) => row.score);
+  return {
+    benchmarkId: benchmarkIdFor(input.adapter),
+    family: input.adapter.family ?? "custom",
+    taskKind: input.adapter.taskKind ?? "custom",
+    ...input.adapter.source ? { source: input.adapter.source } : {},
+    runDir: input.campaign.runDir,
+    manifestHash: input.campaign.manifestHash,
+    seed: input.campaign.seed,
+    startedAt: input.campaign.startedAt,
+    endedAt: input.campaign.endedAt,
+    durationMs: input.campaign.durationMs,
+    totalItems: input.scenarios.length,
+    totalCells: input.campaign.cells.length,
+    cellsFailed: input.campaign.aggregates.cellsFailed,
+    cellsCached: input.campaign.aggregates.cellsCached,
+    totalCostUsd: input.campaign.aggregates.totalCostUsd,
+    splits: summarizeSlices(successful, (row) => row.scenario?.splitTag ?? "unknown", [
+      "search",
+      "dev",
+      "holdout"
+    ]),
+    tags: summarizeSlices(successful, (row) => row.scenario?.tags ?? []),
+    dimensions: summarizeDimensions(successful.map((row) => row.dimensions)),
+    score: distribution(scoreValues),
+    costUsd: distribution(successful.map((row) => row.cell.costUsd)),
+    latencyMs: distribution(successful.map((row) => row.cell.durationMs))
+  };
+}
+function renderBenchmarkReportMarkdown(report) {
+  const splitRows = Object.entries(report.splits).map(([split, summary]) => {
+    return `| ${split} | ${summary.n} | ${fmt(summary.meanScore)} | ${fmt(summary.passRate)} | ${fmt(summary.score.p90)} | ${fmt(summary.costUsd.mean)} | ${fmt(summary.latencyMs.p90)} |`;
+  }).join("\n");
+  const dimRows = Object.entries(report.dimensions).sort(([a], [b]) => a.localeCompare(b)).map(([key, dist]) => `| ${key} | ${dist.n} | ${fmt(dist.mean)} | ${fmt(dist.p90)} |`).join("\n");
+  return [
+    `# Benchmark Report: ${report.benchmarkId}`,
+    "",
+    `- family: ${report.family}`,
+    `- task kind: ${report.taskKind}`,
+    `- run dir: ${report.runDir}`,
+    `- manifest: ${report.manifestHash}`,
+    `- cells: ${report.totalCells} total, ${report.cellsFailed} failed, ${report.cellsCached} cached`,
+    `- cost: $${fmt(report.totalCostUsd)}`,
+    `- score: mean ${fmt(report.score.mean)}, median ${fmt(report.score.median)}, p90 ${fmt(report.score.p90)}, n=${report.score.n}`,
+    "",
+    "## Splits",
+    "",
+    "| split | n | mean score | pass rate | score p90 | mean cost | latency p90 ms |",
+    "| --- | ---: | ---: | ---: | ---: | ---: | ---: |",
+    splitRows || "| none | 0 | 0 | 0 | 0 | 0 | 0 |",
+    "",
+    "## Dimensions",
+    "",
+    "| dimension | n | mean | p90 |",
+    "| --- | ---: | ---: | ---: |",
+    dimRows || "| none | 0 | 0 | 0 |",
+    ""
+  ].join("\n");
+}
+function benchmarkIdFor(adapter2) {
+  return adapter2.id ?? `${adapter2.family ?? "custom"}/${adapter2.taskKind ?? "custom"}`;
+}
+function normalizeEvaluationDimensions(evaluation) {
+  const dimensions = { score: clamp012(evaluation.score) };
+  for (const [key, value] of Object.entries(evaluation.dimensions ?? {})) {
+    if (Number.isFinite(value)) dimensions[key] = value;
+  }
+  dimensions.passed = evaluation.passed ?? evaluation.score > 0 ? 1 : 0;
+  return dimensions;
+}
+function summarizeDimensions(rows) {
+  const values = /* @__PURE__ */ new Map();
+  for (const row of rows) {
+    for (const [key, value] of Object.entries(row)) {
+      if (!Number.isFinite(value)) continue;
+      const list = values.get(key) ?? [];
+      list.push(value);
+      values.set(key, list);
+    }
+  }
+  return Object.fromEntries([...values.entries()].map(([key, vals]) => [key, distribution(vals)]));
+}
+function summarizeSlices(rows, keyOf, knownKeys = []) {
+  const grouped = /* @__PURE__ */ new Map();
+  for (const key of knownKeys) grouped.set(key, []);
+  for (const row of rows) {
+    const keys = keyOf(row);
+    for (const key of Array.isArray(keys) ? keys : [keys]) {
+      const list = grouped.get(key) ?? [];
+      list.push(row);
+      grouped.set(key, list);
+    }
+  }
+  const out = {};
+  for (const [key, list] of grouped) {
+    const withShape = list;
+    out[key] = {
+      n: list.length,
+      meanScore: mean(withShape.map((row) => row.score)),
+      passRate: mean(withShape.map((row) => row.passed ? 1 : 0)),
+      score: distribution(withShape.map((row) => row.score)),
+      costUsd: distribution(withShape.map((row) => row.cell.costUsd)),
+      latencyMs: distribution(withShape.map((row) => row.cell.durationMs))
+    };
+  }
+  return out;
+}
+function firstJudgeScore(judgeScores) {
+  return Object.values(judgeScores)[0];
+}
+function distribution(values) {
+  const finite = [...values].filter(Number.isFinite).sort((a, b) => a - b);
+  if (finite.length === 0) return { n: 0, min: 0, mean: 0, median: 0, p90: 0, max: 0 };
+  return {
+    n: finite.length,
+    min: finite[0],
+    mean: mean(finite),
+    median: percentile(finite, 0.5),
+    p90: percentile(finite, 0.9),
+    max: finite[finite.length - 1]
+  };
+}
+function percentile(sortedValues, p) {
+  if (sortedValues.length === 0) return 0;
+  const index = Math.min(
+    sortedValues.length - 1,
+    Math.max(0, Math.ceil(p * sortedValues.length) - 1)
+  );
+  return sortedValues[index];
+}
+function mean(values) {
+  const finite = values.filter(Number.isFinite);
+  if (finite.length === 0) return 0;
+  return finite.reduce((sum, value) => sum + value, 0) / finite.length;
+}
+function clamp012(value) {
+  if (!Number.isFinite(value)) return 0;
+  if (value < 0) return 0;
+  if (value > 1) return 1;
+  return value;
+}
+function fmt(value) {
+  if (!Number.isFinite(value)) return "0";
+  return value.toFixed(value === 0 || Math.abs(value) >= 10 ? 0 : 3);
+}
+
+// src/benchmarks/standard-formats.ts
+function parseJsonlRows(text) {
+  return text.split(/\r?\n/).map((line) => line.trim()).filter(Boolean).map((line, index) => {
+    try {
+      return JSON.parse(line);
+    } catch (error) {
+      throw new Error(`invalid JSONL row ${index + 1}: ${error.message}`);
+    }
+  });
+}
+function parseTsvRows(text) {
+  return text.split(/\r?\n/).map((line) => line.trim()).filter(Boolean).filter((line) => !line.startsWith("#")).map((line) => line.split(/\t|\s+/));
+}
+function parseQrels(text) {
+  return parseTsvRows(text).flatMap((parts, index) => {
+    if (parts.length < 3) return [];
+    const [queryId, maybeZeroOrDocId, maybeDocIdOrScore, maybeScore] = parts;
+    if (!queryId || !maybeZeroOrDocId || !maybeDocIdOrScore) return [];
+    if (queryId.toLowerCase() === "query-id" || queryId.toLowerCase() === "qid") return [];
+    const documentId = maybeScore === void 0 ? maybeZeroOrDocId : maybeDocIdOrScore;
+    const scoreText = maybeScore === void 0 ? maybeDocIdOrScore : maybeScore;
+    const score = Number(scoreText);
+    if (!documentId || !Number.isFinite(score)) {
+      throw new Error(`invalid qrels row ${index + 1}: expected query id, doc id, score`);
+    }
+    return [{ queryId, documentId, score }];
+  });
+}
+function parseBeirCorpusJsonl(text) {
+  return parseJsonlRows(text).map((row, index) => {
+    const id = stringField(row, "_id") ?? stringField(row, "id");
+    const body = stringField(row, "text") ?? stringField(row, "contents");
+    if (!id || body === void 0) {
+      throw new Error(`invalid BEIR corpus row ${index + 1}: expected _id/id and text/contents`);
+    }
+    return {
+      id,
+      title: stringField(row, "title"),
+      text: body,
+      metadata: stripKnown(row, ["_id", "id", "title", "text", "contents"])
+    };
+  });
+}
+function parseBeirQueriesJsonl(text) {
+  return parseJsonlRows(text).map((row, index) => {
+    const id = stringField(row, "_id") ?? stringField(row, "id") ?? stringField(row, "query_id");
+    const query = stringField(row, "text") ?? stringField(row, "query");
+    if (!id || !query) {
+      throw new Error(`invalid BEIR query row ${index + 1}: expected _id/id and text/query`);
+    }
+    return {
+      id,
+      text: query,
+      metadata: stripKnown(row, ["_id", "id", "query_id", "text", "query"])
+    };
+  });
+}
+function buildStandardRetrievalItems(options) {
+  const qrelsByQuery = /* @__PURE__ */ new Map();
+  for (const qrel of options.qrels) {
+    if (qrel.score <= 0) continue;
+    const list = qrelsByQuery.get(qrel.queryId) ?? [];
+    list.push(qrel);
+    qrelsByQuery.set(qrel.queryId, list);
+  }
+  const corpus = options.includeCorpusInPayload && options.corpus ? Object.fromEntries(options.corpus.map((document) => [document.id, document])) : void 0;
+  return options.queries.flatMap((query) => {
+    const qrels = qrelsByQuery.get(query.id) ?? [];
+    if (qrels.length === 0) return [];
+    const split = options.splitOf?.(query.id) ?? deterministicSplit(`${options.benchmarkId}:${query.id}`);
+    return [
+      {
+        id: query.id,
+        split,
+        family: options.family,
+        taskKind: "retrieval",
+        tags: [.../* @__PURE__ */ new Set([...options.tags ?? [], split])],
+        ...options.source ? { source: options.source } : {},
+        ...query.metadata ? { metadata: query.metadata } : {},
+        payload: {
+          queryId: query.id,
+          query: query.text,
+          expectedDocumentIds: qrels.map((qrel) => qrel.documentId),
+          expectedScores: Object.fromEntries(qrels.map((qrel) => [qrel.documentId, qrel.score])),
+          ...corpus ? { corpus } : {},
+          ...query.metadata ? { metadata: query.metadata } : {}
+        }
+      }
+    ];
+  });
+}
+function createRetrievalIdBenchmarkAdapter(options) {
+  const items = buildStandardRetrievalItems(options);
+  return {
+    id: options.benchmarkId,
+    family: options.family,
+    taskKind: "retrieval",
+    source: options.source,
+    defaultMetric: options.primaryMetric ?? "ndcg@10",
+    async loadDataset(split) {
+      return items.filter((item) => item.split === split);
+    },
+    async evaluate(item, artifact) {
+      return evaluateStandardRetrieval(item.payload, artifact, options);
+    },
+    assignSplit(itemId) {
+      return options.splitOf?.(itemId) ?? deterministicSplit(`${options.benchmarkId}:${itemId}`);
+    }
+  };
+}
+function evaluateStandardRetrieval(payload, artifact, options = {}) {
+  const rankedDocumentIds = normalizeRetrievedDocumentIds(
+    artifact,
+    options.responseIdPattern ?? /[A-Za-z0-9_.:/-]+/g
+  );
+  const cutoffs = normalizeCutoffs(options.cutoffs ?? [1, 3, 5, 10]);
+  const dimensions = {
+    expected_count: payload.expectedDocumentIds.length,
+    returned_count: rankedDocumentIds.length
+  };
+  for (const cutoff of cutoffs) {
+    Object.assign(
+      dimensions,
+      retrievalMetricsAtCutoff({
+        rankedDocumentIds,
+        expectedScores: payload.expectedScores,
+        cutoff
+      })
+    );
+  }
+  const primaryMetric = options.primaryMetric ?? "ndcg@10";
+  const passMetric = options.passMetric ?? "hit@10";
+  const score = dimensions[primaryMetric] ?? dimensions[`ndcg@${cutoffs[cutoffs.length - 1]}`] ?? 0;
+  const passScore = dimensions[passMetric] ?? dimensions[`hit@${cutoffs[cutoffs.length - 1]}`] ?? score;
+  return {
+    score,
+    passed: passScore >= (options.passThreshold ?? 1),
+    dimensions,
+    raw: {
+      rankedDocumentIds,
+      expectedDocumentIds: payload.expectedDocumentIds,
+      expectedScores: payload.expectedScores
+    }
+  };
+}
+function normalizeRetrievedDocumentIds(artifact, responseIdPattern = /[A-Za-z0-9_.:/-]+/g) {
+  if (typeof artifact === "string") {
+    return uniqueOrdered((artifact.match(responseIdPattern) ?? []).map((value) => value.trim()));
+  }
+  if (Array.isArray(artifact)) {
+    const entries = artifact;
+    return uniqueOrdered(
+      entries.flatMap((entry) => {
+        if (typeof entry === "string") return [entry];
+        return [entry.documentId ?? entry.docId ?? entry.id ?? ""];
+      })
+    );
+  }
+  const objectArtifact = artifact;
+  return uniqueOrdered(
+    [
+      ...objectArtifact.documentIds ?? [],
+      ...objectArtifact.ids ?? [],
+      ...(objectArtifact.results ?? []).map(
+        (entry) => entry.documentId ?? entry.docId ?? entry.id ?? ""
+      )
+    ].map((value) => value.trim())
+  );
+}
+function retrievalMetricsAtCutoff(input) {
+  const top = input.rankedDocumentIds.slice(0, input.cutoff);
+  const relevantIds = Object.keys(input.expectedScores).filter(
+    (id) => input.expectedScores[id] > 0
+  );
+  const relevant = new Set(relevantIds);
+  const hits = top.filter((id) => relevant.has(id)).length;
+  const firstRelevantRank = top.findIndex((id) => relevant.has(id));
+  const prefix = `@${input.cutoff}`;
+  return {
+    [`hit${prefix}`]: hits > 0 ? 1 : 0,
+    [`recall${prefix}`]: relevant.size === 0 ? 1 : hits / relevant.size,
+    [`precision${prefix}`]: hits / input.cutoff,
+    [`mrr${prefix}`]: firstRelevantRank === -1 ? 0 : 1 / (firstRelevantRank + 1),
+    [`ndcg${prefix}`]: ndcgAt(input.rankedDocumentIds, input.expectedScores, input.cutoff)
+  };
+}
+function stringField(row, key) {
+  const value = row[key];
+  return typeof value === "string" ? value : void 0;
+}
+function stripKnown(row, keys) {
+  const known = new Set(keys);
+  return Object.fromEntries(Object.entries(row).filter(([key]) => !known.has(key)));
+}
+function normalizeCutoffs(cutoffs) {
+  return [
+    ...new Set(cutoffs.map((cutoff) => Math.trunc(cutoff)).filter((cutoff) => cutoff > 0))
+  ].sort((a, b) => a - b);
+}
+function uniqueOrdered(values) {
+  const seen = /* @__PURE__ */ new Set();
+  const out = [];
+  for (const value of values) {
+    const trimmed = value.trim();
+    if (!trimmed || seen.has(trimmed)) continue;
+    seen.add(trimmed);
+    out.push(trimmed);
+  }
+  return out;
+}
+function ndcgAt(rankedDocumentIds, expectedScores, cutoff) {
+  const dcg = rankedDocumentIds.slice(0, cutoff).reduce(
+    (sum, documentId, index) => sum + discountedGain(expectedScores[documentId] ?? 0, index),
+    0
+  );
+  const ideal = Object.values(expectedScores).filter((score) => score > 0).sort((a, b) => b - a).slice(0, cutoff).reduce((sum, score, index) => sum + discountedGain(score, index), 0);
+  return ideal === 0 ? 1 : dcg / ideal;
+}
+function discountedGain(relevance, zeroBasedRank) {
+  if (relevance <= 0) return 0;
+  return (2 ** relevance - 1) / Math.log2(zeroBasedRank + 2);
+}
+
+export {
+  calibrateBenchmarkMetric,
+  BENCHMARK_SPLIT_SEED,
+  deterministicSplit,
+  routing_exports,
+  runBenchmarkAdapter,
+  summarizeBenchmarkCampaign,
+  renderBenchmarkReportMarkdown,
+  parseJsonlRows,
+  parseTsvRows,
+  parseQrels,
+  parseBeirCorpusJsonl,
+  parseBeirQueriesJsonl,
+  buildStandardRetrievalItems,
+  createRetrievalIdBenchmarkAdapter,
+  evaluateStandardRetrieval,
+  normalizeRetrievedDocumentIds,
+  retrievalMetricsAtCutoff,
+  benchmarks_exports
+};
+//# sourceMappingURL=chunk-CVJP5TMD.js.map