@tangle-network/agent-eval 0.129.0 → 0.130.0

package/dist/chunk-IYCLP2N2.js

@@ -1,766 +0,0 @@
-import {
-  fsCampaignStorage,
-  runCampaign
-} from "./chunk-C6LXANRU.js";
-import {
-  __export
-} from "./chunk-PZ5AY32C.js";
-
-// src/benchmarks/index.ts
-var benchmarks_exports = {};
-__export(benchmarks_exports, {
-  BENCHMARK_SPLIT_SEED: () => BENCHMARK_SPLIT_SEED,
-  buildStandardRetrievalItems: () => buildStandardRetrievalItems,
-  calibrateBenchmarkMetric: () => calibrateBenchmarkMetric,
-  createRetrievalIdBenchmarkAdapter: () => createRetrievalIdBenchmarkAdapter,
-  deterministicSplit: () => deterministicSplit,
-  evaluateStandardRetrieval: () => evaluateStandardRetrieval,
-  normalizeRetrievedDocumentIds: () => normalizeRetrievedDocumentIds,
-  parseBeirCorpusJsonl: () => parseBeirCorpusJsonl,
-  parseBeirQueriesJsonl: () => parseBeirQueriesJsonl,
-  parseJsonlRows: () => parseJsonlRows,
-  parseQrels: () => parseQrels,
-  parseTsvRows: () => parseTsvRows,
-  renderBenchmarkReportMarkdown: () => renderBenchmarkReportMarkdown,
-  retrievalMetricsAtCutoff: () => retrievalMetricsAtCutoff,
-  routing: () => routing_exports,
-  runBenchmarkAdapter: () => runBenchmarkAdapter,
-  summarizeBenchmarkCampaign: () => summarizeBenchmarkCampaign
-});
-
-// src/benchmarks/calibration.ts
-async function calibrateBenchmarkMetric(options) {
-  const weak = await options.adapter.evaluate(options.item, options.weakArtifact);
-  const strong = await options.adapter.evaluate(options.item, options.strongArtifact);
-  const weakScore = clamp01(weak.score);
-  const strongScore = clamp01(strong.score);
-  const maxWeakScore = options.maxWeakScore ?? 0.3;
-  const minStrongScore = options.minStrongScore ?? 0.7;
-  const minGap = options.minGap ?? 0.4;
-  const gap = strongScore - weakScore;
-  const reasons = [
-    weakScore <= maxWeakScore ? void 0 : `weak score ${weakScore.toFixed(3)} exceeds max ${maxWeakScore.toFixed(3)}`,
-    strongScore >= minStrongScore ? void 0 : `strong score ${strongScore.toFixed(3)} below min ${minStrongScore.toFixed(3)}`,
-    gap >= minGap ? void 0 : `gap ${gap.toFixed(3)} below min ${minGap.toFixed(3)}`
-  ].filter((reason) => Boolean(reason));
-  return {
-    passed: reasons.length === 0,
-    weak,
-    strong,
-    weakScore,
-    strongScore,
-    gap,
-    reasons
-  };
-}
-function clamp01(value) {
-  if (!Number.isFinite(value)) return 0;
-  if (value < 0) return 0;
-  if (value > 1) return 1;
-  return value;
-}
-
-// src/benchmarks/routing/index.ts
-var routing_exports = {};
-__export(routing_exports, {
-  ROUTING_DATASET: () => ROUTING_DATASET,
-  RoutingAdapter: () => RoutingAdapter,
-  assignSplit: () => assignSplit,
-  evaluate: () => evaluate,
-  extractRouteTokens: () => extractRouteTokens,
-  loadDataset: () => loadDataset
-});
-
-// src/benchmarks/types.ts
-function fnv1a32(input) {
-  let h = 2166136261;
-  for (let i = 0; i < input.length; i++) {
-    h ^= input.charCodeAt(i) & 255;
-    h = h + ((h << 1) + (h << 4) + (h << 7) + (h << 8) + (h << 24)) >>> 0;
-  }
-  return h >>> 0;
-}
-var BENCHMARK_SPLIT_SEED = "agent-eval-v1";
-function deterministicSplit(itemId, seed = BENCHMARK_SPLIT_SEED) {
-  const h = fnv1a32(`${seed}::${itemId}`);
-  const pos = h / 4294967296;
-  if (pos < 0.6) return "search";
-  if (pos < 0.8) return "dev";
-  return "holdout";
-}
-
-// src/benchmarks/routing/dataset.ts
-var ROUTING_DATASET = [
-  {
-    id: "file_001",
-    category: "file",
-    prompt: "Save the meeting notes to /tmp/notes-2025-04.md as markdown.",
-    route: "fs.write",
-    synonyms: ["filesystem.write", "write_file"],
-    hardNegatives: ["fs.read", "chat.reply"]
-  },
-  {
-    id: "file_002",
-    category: "file",
-    prompt: "Read the contents of /etc/hosts and summarize the entries.",
-    route: "fs.read",
-    synonyms: ["filesystem.read", "read_file"],
-    hardNegatives: ["fs.write", "search.web"]
-  },
-  {
-    id: "file_003",
-    category: "file",
-    prompt: "List every Python file under src/ recursively.",
-    route: "fs.list",
-    synonyms: ["filesystem.list", "list_files"],
-    hardNegatives: ["fs.read", "search.code"]
-  },
-  {
-    id: "file_004",
-    category: "file",
-    prompt: "Delete the cached build at .turbo/cache.",
-    route: "fs.delete",
-    synonyms: ["filesystem.delete", "remove_file"],
-    hardNegatives: ["fs.write", "fs.list"]
-  },
-  {
-    id: "math_001",
-    category: "math",
-    prompt: "What is the integral of 3x^2 + 2x from 0 to 5?",
-    route: "math.integral",
-    synonyms: ["calculator.integral", "math.solve"],
-    hardNegatives: ["math.derivative", "chat.reply"]
-  },
-  {
-    id: "math_002",
-    category: "math",
-    prompt: "Compute the derivative of sin(x) * cos(x).",
-    route: "math.derivative",
-    synonyms: ["calculator.derivative", "math.solve"],
-    hardNegatives: ["math.integral", "math.algebra"]
-  },
-  {
-    id: "math_003",
-    category: "math",
-    prompt: "Solve 2x + 7 = 19 for x.",
-    route: "math.algebra",
-    synonyms: ["calculator.algebra", "math.solve"],
-    hardNegatives: ["math.derivative", "math.integral"]
-  },
-  {
-    id: "math_004",
-    category: "math",
-    prompt: "What is the prime factorization of 360?",
-    route: "math.numbertheory",
-    synonyms: ["calculator.factor", "math.solve"],
-    hardNegatives: ["math.algebra", "search.web"]
-  },
-  {
-    id: "search_001",
-    category: "search",
-    prompt: "Find recent papers on agent prompt optimization with held-out promotion gates.",
-    route: "search.web",
-    synonyms: ["web.search", "search.papers"],
-    hardNegatives: ["search.code", "chat.reply"]
-  },
-  {
-    id: "search_002",
-    category: "search",
-    prompt: "Search the codebase for every call site of `runProposeReview`.",
-    route: "search.code",
-    synonyms: ["code.search", "grep"],
-    hardNegatives: ["search.web", "fs.read"]
-  },
-  {
-    id: "search_003",
-    category: "search",
-    prompt: "What is the latest release of the Tangle network on GitHub?",
-    route: "search.web",
-    synonyms: ["web.search", "github.releases"],
-    hardNegatives: ["search.code", "chat.reply"]
-  },
-  {
-    id: "search_004",
-    category: "search",
-    prompt: "Find all TODO comments in the agent-eval src tree.",
-    route: "search.code",
-    synonyms: ["code.search", "grep"],
-    hardNegatives: ["search.web", "fs.list"]
-  },
-  {
-    id: "chat_001",
-    category: "chat",
-    prompt: "Hi there, how are you doing today?",
-    route: "chat.reply",
-    synonyms: ["conversation.reply"],
-    hardNegatives: ["search.web", "fs.read"]
-  },
-  {
-    id: "chat_002",
-    category: "chat",
-    prompt: "Please explain the difference between an LLM and a foundation model.",
-    route: "chat.reply",
-    synonyms: ["conversation.reply", "qa.answer"],
-    hardNegatives: ["search.web", "math.algebra"]
-  },
-  {
-    id: "chat_003",
-    category: "chat",
-    prompt: "Tell me a short joke about distributed systems.",
-    route: "chat.reply",
-    synonyms: ["conversation.reply"],
-    hardNegatives: ["search.web", "fs.read"]
-  },
-  {
-    id: "chat_004",
-    category: "chat",
-    prompt: "Acknowledge my last message with a thumbs up.",
-    route: "chat.reply",
-    synonyms: ["conversation.reply", "react"],
-    hardNegatives: ["fs.write", "search.web"]
-  }
-];
-
-// src/benchmarks/routing/index.ts
-var RoutingAdapter = class {
-  id = "first-party/routing";
-  family = "first-party";
-  taskKind = "routing";
-  description = "Synthetic fixed-route classification smoke benchmark";
-  defaultMetric = "route_exact_match";
-  async loadDataset(split) {
-    return ROUTING_DATASET.map((item) => ({ id: item.id, payload: item })).filter(
-      (it) => assignSplitImpl(it.id) === split
-    );
-  }
-  async evaluate(item, response) {
-    const tokens = extractRouteTokens(response);
-    const correct = new Set(
-      [item.payload.route, ...item.payload.synonyms].map((s) => s.toLowerCase())
-    );
-    const hardNeg = new Set(item.payload.hardNegatives.map((s) => s.toLowerCase()));
-    const firstMatch = tokens.find((t) => correct.has(t.toLowerCase())) ?? null;
-    const firstHardNeg = tokens.find((t) => hardNeg.has(t.toLowerCase())) ?? null;
-    const score = firstMatch ? 1 : 0;
-    return {
-      score,
-      raw: {
-        firstToken: tokens[0] ?? null,
-        matchedRoute: firstMatch,
-        hitHardNegative: Boolean(firstHardNeg),
-        hardNegativeRoute: firstHardNeg,
-        category: item.payload.category
-      }
-    };
-  }
-  assignSplit(itemId) {
-    return assignSplitImpl(itemId);
-  }
-};
-function assignSplitImpl(itemId) {
-  return deterministicSplit(`routing::${itemId}`);
-}
-function extractRouteTokens(response) {
-  const matches = response.match(/[a-z][a-z0-9_]*\.[a-z][a-z0-9_]*/gi);
-  return matches ?? [];
-}
-var adapter = new RoutingAdapter();
-var loadDataset = adapter.loadDataset.bind(adapter);
-var evaluate = adapter.evaluate.bind(adapter);
-var assignSplit = adapter.assignSplit.bind(adapter);
-
-// src/benchmarks/runner.ts
-import { join } from "path";
-async function runBenchmarkAdapter(options) {
-  const storage = options.storage ?? fsCampaignStorage();
-  const benchmarkId = benchmarkIdFor(options.adapter);
-  const scenarios = await loadBenchmarkScenarios(options.adapter, options.splits);
-  const judge = benchmarkAdapterJudge(options.adapter);
-  const dispatch = async (scenario, context) => {
-    return options.respond({ scenario, item: scenario.item, context });
-  };
-  const campaign = await runCampaign({
-    scenarios,
-    dispatch,
-    dispatchRef: `benchmark:${benchmarkId}`,
-    judges: [judge],
-    seed: options.seed,
-    reps: options.reps,
-    resumable: options.resumable,
-    costCeiling: options.costCeiling,
-    maxConcurrency: options.maxConcurrency,
-    dispatchTimeoutMs: options.dispatchTimeoutMs,
-    expectUsage: options.expectUsage ?? "off",
-    runDir: options.runDir,
-    repo: options.repo,
-    storage,
-    now: options.now
-  });
-  const report = summarizeBenchmarkCampaign({
-    adapter: options.adapter,
-    scenarios,
-    campaign
-  });
-  storage.ensureDir(campaign.runDir);
-  const reportJsonPath = join(campaign.runDir, "benchmark-report.json");
-  const reportMarkdownPath = join(campaign.runDir, "benchmark-report.md");
-  storage.write(reportJsonPath, `${JSON.stringify(report, null, 2)}
-`);
-  storage.write(reportMarkdownPath, renderBenchmarkReportMarkdown(report));
-  return { scenarios, campaign, report, reportJsonPath, reportMarkdownPath };
-}
-async function loadBenchmarkScenarios(adapter2, splits = ["search", "dev", "holdout"]) {
-  const benchmarkId = benchmarkIdFor(adapter2);
-  const family = adapter2.family ?? "custom";
-  const taskKind = adapter2.taskKind ?? "custom";
-  const out = [];
-  const seen = /* @__PURE__ */ new Set();
-  for (const split of splits) {
-    const items = await adapter2.loadDataset(split);
-    for (const item of items) {
-      const splitTag = item.split ?? adapter2.assignSplit(item.id);
-      if (splitTag !== split) continue;
-      const key = `${benchmarkId}:${item.id}`;
-      if (seen.has(key)) continue;
-      seen.add(key);
-      out.push({
-        id: key,
-        kind: "benchmark",
-        benchmarkId,
-        family: item.family ?? family,
-        taskKind: item.taskKind ?? taskKind,
-        splitTag,
-        tags: [.../* @__PURE__ */ new Set([splitTag, ...item.tags ?? []])],
-        item
-      });
-    }
-  }
-  return out;
-}
-function benchmarkAdapterJudge(adapter2) {
-  const benchmarkId = benchmarkIdFor(adapter2);
-  return {
-    name: `${benchmarkId}:score`,
-    dimensions: [
-      { key: "score", description: `Primary ${benchmarkId} benchmark score` },
-      { key: "passed", description: "Binary pass projection for aggregate reporting" }
-    ],
-    appliesTo: (scenario) => {
-      return scenario.kind === "benchmark" && scenario.id.startsWith(`${benchmarkId}:`);
-    },
-    async score({ artifact, scenario }) {
-      const evaluation = await adapter2.evaluate(scenario.item, artifact);
-      const dimensions = normalizeEvaluationDimensions(evaluation);
-      return {
-        dimensions,
-        composite: clamp012(evaluation.score),
-        notes: evaluation.notes ?? ""
-      };
-    }
-  };
-}
-function summarizeBenchmarkCampaign(input) {
-  const scenarioById = new Map(input.scenarios.map((scenario) => [scenario.id, scenario]));
-  const rows = input.campaign.cells.map((cell) => {
-    const scenario = scenarioById.get(cell.scenarioId);
-    const judge = firstJudgeScore(cell.judgeScores);
-    const score = judge?.composite ?? 0;
-    return {
-      cell,
-      scenario,
-      score,
-      passed: (judge?.dimensions.passed ?? (score > 0 ? 1 : 0)) >= 1,
-      dimensions: judge?.dimensions ?? {}
-    };
-  });
-  const successful = rows.filter((row) => !row.cell.error);
-  const scoreValues = successful.map((row) => row.score);
-  return {
-    benchmarkId: benchmarkIdFor(input.adapter),
-    family: input.adapter.family ?? "custom",
-    taskKind: input.adapter.taskKind ?? "custom",
-    ...input.adapter.source ? { source: input.adapter.source } : {},
-    runDir: input.campaign.runDir,
-    manifestHash: input.campaign.manifestHash,
-    seed: input.campaign.seed,
-    startedAt: input.campaign.startedAt,
-    endedAt: input.campaign.endedAt,
-    durationMs: input.campaign.durationMs,
-    totalItems: input.scenarios.length,
-    totalCells: input.campaign.cells.length,
-    cellsFailed: input.campaign.aggregates.cellsFailed,
-    cellsCached: input.campaign.aggregates.cellsCached,
-    totalCostUsd: input.campaign.aggregates.cost.totalCostUsd,
-    splits: summarizeSlices(successful, (row) => row.scenario?.splitTag ?? "unknown", [
-      "search",
-      "dev",
-      "holdout"
-    ]),
-    tags: summarizeSlices(successful, (row) => row.scenario?.tags ?? []),
-    dimensions: summarizeDimensions(successful.map((row) => row.dimensions)),
-    score: distribution(scoreValues),
-    costUsd: distribution(successful.map((row) => row.cell.costUsd)),
-    latencyMs: distribution(successful.map((row) => row.cell.durationMs))
-  };
-}
-function renderBenchmarkReportMarkdown(report) {
-  const splitRows = Object.entries(report.splits).map(([split, summary]) => {
-    return `| ${split} | ${summary.n} | ${fmt(summary.meanScore)} | ${fmt(summary.passRate)} | ${fmt(summary.score.p90)} | ${fmt(summary.costUsd.mean)} | ${fmt(summary.latencyMs.p90)} |`;
-  }).join("\n");
-  const dimRows = Object.entries(report.dimensions).sort(([a], [b]) => a.localeCompare(b)).map(([key, dist]) => `| ${key} | ${dist.n} | ${fmt(dist.mean)} | ${fmt(dist.p90)} |`).join("\n");
-  return [
-    `# Benchmark Report: ${report.benchmarkId}`,
-    "",
-    `- family: ${report.family}`,
-    `- task kind: ${report.taskKind}`,
-    `- run dir: ${report.runDir}`,
-    `- manifest: ${report.manifestHash}`,
-    `- cells: ${report.totalCells} total, ${report.cellsFailed} failed, ${report.cellsCached} cached`,
-    `- cost: $${fmt(report.totalCostUsd)}`,
-    `- score: mean ${fmt(report.score.mean)}, median ${fmt(report.score.median)}, p90 ${fmt(report.score.p90)}, n=${report.score.n}`,
-    "",
-    "## Splits",
-    "",
-    "| split | n | mean score | pass rate | score p90 | mean cost | latency p90 ms |",
-    "| --- | ---: | ---: | ---: | ---: | ---: | ---: |",
-    splitRows || "| none | 0 | 0 | 0 | 0 | 0 | 0 |",
-    "",
-    "## Dimensions",
-    "",
-    "| dimension | n | mean | p90 |",
-    "| --- | ---: | ---: | ---: |",
-    dimRows || "| none | 0 | 0 | 0 |",
-    ""
-  ].join("\n");
-}
-function benchmarkIdFor(adapter2) {
-  return adapter2.id ?? `${adapter2.family ?? "custom"}/${adapter2.taskKind ?? "custom"}`;
-}
-function normalizeEvaluationDimensions(evaluation) {
-  const dimensions = { score: clamp012(evaluation.score) };
-  for (const [key, value] of Object.entries(evaluation.dimensions ?? {})) {
-    if (Number.isFinite(value)) dimensions[key] = value;
-  }
-  dimensions.passed = evaluation.passed ?? evaluation.score > 0 ? 1 : 0;
-  return dimensions;
-}
-function summarizeDimensions(rows) {
-  const values = /* @__PURE__ */ new Map();
-  for (const row of rows) {
-    for (const [key, value] of Object.entries(row)) {
-      if (!Number.isFinite(value)) continue;
-      const list = values.get(key) ?? [];
-      list.push(value);
-      values.set(key, list);
-    }
-  }
-  return Object.fromEntries([...values.entries()].map(([key, vals]) => [key, distribution(vals)]));
-}
-function summarizeSlices(rows, keyOf, knownKeys = []) {
-  const grouped = /* @__PURE__ */ new Map();
-  for (const key of knownKeys) grouped.set(key, []);
-  for (const row of rows) {
-    const keys = keyOf(row);
-    for (const key of Array.isArray(keys) ? keys : [keys]) {
-      const list = grouped.get(key) ?? [];
-      list.push(row);
-      grouped.set(key, list);
-    }
-  }
-  const out = {};
-  for (const [key, list] of grouped) {
-    const withShape = list;
-    out[key] = {
-      n: list.length,
-      meanScore: mean(withShape.map((row) => row.score)),
-      passRate: mean(withShape.map((row) => row.passed ? 1 : 0)),
-      score: distribution(withShape.map((row) => row.score)),
-      costUsd: distribution(withShape.map((row) => row.cell.costUsd)),
-      latencyMs: distribution(withShape.map((row) => row.cell.durationMs))
-    };
-  }
-  return out;
-}
-function firstJudgeScore(judgeScores) {
-  return Object.values(judgeScores)[0];
-}
-function distribution(values) {
-  const finite = [...values].filter(Number.isFinite).sort((a, b) => a - b);
-  if (finite.length === 0) return { n: 0, min: 0, mean: 0, median: 0, p90: 0, max: 0 };
-  return {
-    n: finite.length,
-    min: finite[0],
-    mean: mean(finite),
-    median: percentile(finite, 0.5),
-    p90: percentile(finite, 0.9),
-    max: finite[finite.length - 1]
-  };
-}
-function percentile(sortedValues, p) {
-  if (sortedValues.length === 0) return 0;
-  const index = Math.min(
-    sortedValues.length - 1,
-    Math.max(0, Math.ceil(p * sortedValues.length) - 1)
-  );
-  return sortedValues[index];
-}
-function mean(values) {
-  const finite = values.filter(Number.isFinite);
-  if (finite.length === 0) return 0;
-  return finite.reduce((sum, value) => sum + value, 0) / finite.length;
-}
-function clamp012(value) {
-  if (!Number.isFinite(value)) return 0;
-  if (value < 0) return 0;
-  if (value > 1) return 1;
-  return value;
-}
-function fmt(value) {
-  if (!Number.isFinite(value)) return "0";
-  return value.toFixed(value === 0 || Math.abs(value) >= 10 ? 0 : 3);
-}
-
-// src/benchmarks/standard-formats.ts
-function parseJsonlRows(text) {
-  return text.split(/\r?\n/).map((line) => line.trim()).filter(Boolean).map((line, index) => {
-    try {
-      return JSON.parse(line);
-    } catch (error) {
-      throw new Error(`invalid JSONL row ${index + 1}: ${error.message}`);
-    }
-  });
-}
-function parseTsvRows(text) {
-  return text.split(/\r?\n/).map((line) => line.trim()).filter(Boolean).filter((line) => !line.startsWith("#")).map((line) => line.split(/\t|\s+/));
-}
-function parseQrels(text) {
-  return parseTsvRows(text).flatMap((parts, index) => {
-    if (parts.length < 3) return [];
-    const [queryId, maybeZeroOrDocId, maybeDocIdOrScore, maybeScore] = parts;
-    if (!queryId || !maybeZeroOrDocId || !maybeDocIdOrScore) return [];
-    if (queryId.toLowerCase() === "query-id" || queryId.toLowerCase() === "qid") return [];
-    const documentId = maybeScore === void 0 ? maybeZeroOrDocId : maybeDocIdOrScore;
-    const scoreText = maybeScore === void 0 ? maybeDocIdOrScore : maybeScore;
-    const score = Number(scoreText);
-    if (!documentId || !Number.isFinite(score)) {
-      throw new Error(`invalid qrels row ${index + 1}: expected query id, doc id, score`);
-    }
-    return [{ queryId, documentId, score }];
-  });
-}
-function parseBeirCorpusJsonl(text) {
-  return parseJsonlRows(text).map((row, index) => {
-    const id = stringField(row, "_id") ?? stringField(row, "id");
-    const body = stringField(row, "text") ?? stringField(row, "contents");
-    if (!id || body === void 0) {
-      throw new Error(`invalid BEIR corpus row ${index + 1}: expected _id/id and text/contents`);
-    }
-    return {
-      id,
-      title: stringField(row, "title"),
-      text: body,
-      metadata: stripKnown(row, ["_id", "id", "title", "text", "contents"])
-    };
-  });
-}
-function parseBeirQueriesJsonl(text) {
-  return parseJsonlRows(text).map((row, index) => {
-    const id = stringField(row, "_id") ?? stringField(row, "id") ?? stringField(row, "query_id");
-    const query = stringField(row, "text") ?? stringField(row, "query");
-    if (!id || !query) {
-      throw new Error(`invalid BEIR query row ${index + 1}: expected _id/id and text/query`);
-    }
-    return {
-      id,
-      text: query,
-      metadata: stripKnown(row, ["_id", "id", "query_id", "text", "query"])
-    };
-  });
-}
-function buildStandardRetrievalItems(options) {
-  const qrelsByQuery = /* @__PURE__ */ new Map();
-  for (const qrel of options.qrels) {
-    if (qrel.score <= 0) continue;
-    const list = qrelsByQuery.get(qrel.queryId) ?? [];
-    list.push(qrel);
-    qrelsByQuery.set(qrel.queryId, list);
-  }
-  const corpus = options.includeCorpusInPayload && options.corpus ? Object.fromEntries(options.corpus.map((document) => [document.id, document])) : void 0;
-  return options.queries.flatMap((query) => {
-    const qrels = qrelsByQuery.get(query.id) ?? [];
-    if (qrels.length === 0) return [];
-    const split = options.splitOf?.(query.id) ?? deterministicSplit(`${options.benchmarkId}:${query.id}`);
-    return [
-      {
-        id: query.id,
-        split,
-        family: options.family,
-        taskKind: "retrieval",
-        tags: [.../* @__PURE__ */ new Set([...options.tags ?? [], split])],
-        ...options.source ? { source: options.source } : {},
-        ...query.metadata ? { metadata: query.metadata } : {},
-        payload: {
-          queryId: query.id,
-          query: query.text,
-          expectedDocumentIds: qrels.map((qrel) => qrel.documentId),
-          expectedScores: Object.fromEntries(qrels.map((qrel) => [qrel.documentId, qrel.score])),
-          ...corpus ? { corpus } : {},
-          ...query.metadata ? { metadata: query.metadata } : {}
-        }
-      }
-    ];
-  });
-}
-function createRetrievalIdBenchmarkAdapter(options) {
-  const items = buildStandardRetrievalItems(options);
-  return {
-    id: options.benchmarkId,
-    family: options.family,
-    taskKind: "retrieval",
-    source: options.source,
-    defaultMetric: options.primaryMetric ?? "ndcg@10",
-    async loadDataset(split) {
-      return items.filter((item) => item.split === split);
-    },
-    async evaluate(item, artifact) {
-      return evaluateStandardRetrieval(item.payload, artifact, options);
-    },
-    assignSplit(itemId) {
-      return options.splitOf?.(itemId) ?? deterministicSplit(`${options.benchmarkId}:${itemId}`);
-    }
-  };
-}
-function evaluateStandardRetrieval(payload, artifact, options = {}) {
-  const rankedDocumentIds = normalizeRetrievedDocumentIds(
-    artifact,
-    options.responseIdPattern ?? /[A-Za-z0-9_.:/-]+/g
-  );
-  const cutoffs = normalizeCutoffs(options.cutoffs ?? [1, 3, 5, 10]);
-  const dimensions = {
-    expected_count: payload.expectedDocumentIds.length,
-    returned_count: rankedDocumentIds.length
-  };
-  for (const cutoff of cutoffs) {
-    Object.assign(
-      dimensions,
-      retrievalMetricsAtCutoff({
-        rankedDocumentIds,
-        expectedScores: payload.expectedScores,
-        cutoff
-      })
-    );
-  }
-  const primaryMetric = options.primaryMetric ?? "ndcg@10";
-  const passMetric = options.passMetric ?? "hit@10";
-  const score = dimensions[primaryMetric] ?? dimensions[`ndcg@${cutoffs[cutoffs.length - 1]}`] ?? 0;
-  const passScore = dimensions[passMetric] ?? dimensions[`hit@${cutoffs[cutoffs.length - 1]}`] ?? score;
-  return {
-    score,
-    passed: passScore >= (options.passThreshold ?? 1),
-    dimensions,
-    raw: {
-      rankedDocumentIds,
-      expectedDocumentIds: payload.expectedDocumentIds,
-      expectedScores: payload.expectedScores
-    }
-  };
-}
-function normalizeRetrievedDocumentIds(artifact, responseIdPattern = /[A-Za-z0-9_.:/-]+/g) {
-  if (typeof artifact === "string") {
-    return uniqueOrdered((artifact.match(responseIdPattern) ?? []).map((value) => value.trim()));
-  }
-  if (Array.isArray(artifact)) {
-    const entries = artifact;
-    return uniqueOrdered(
-      entries.flatMap((entry) => {
-        if (typeof entry === "string") return [entry];
-        return [entry.documentId ?? entry.docId ?? entry.id ?? ""];
-      })
-    );
-  }
-  const objectArtifact = artifact;
-  return uniqueOrdered(
-    [
-      ...objectArtifact.documentIds ?? [],
-      ...objectArtifact.ids ?? [],
-      ...(objectArtifact.results ?? []).map(
-        (entry) => entry.documentId ?? entry.docId ?? entry.id ?? ""
-      )
-    ].map((value) => value.trim())
-  );
-}
-function retrievalMetricsAtCutoff(input) {
-  const top = input.rankedDocumentIds.slice(0, input.cutoff);
-  const relevantIds = Object.keys(input.expectedScores).filter(
-    (id) => input.expectedScores[id] > 0
-  );
-  const relevant = new Set(relevantIds);
-  const hits = top.filter((id) => relevant.has(id)).length;
-  const firstRelevantRank = top.findIndex((id) => relevant.has(id));
-  const prefix = `@${input.cutoff}`;
-  return {
-    [`hit${prefix}`]: hits > 0 ? 1 : 0,
-    [`recall${prefix}`]: relevant.size === 0 ? 1 : hits / relevant.size,
-    [`precision${prefix}`]: hits / input.cutoff,
-    [`mrr${prefix}`]: firstRelevantRank === -1 ? 0 : 1 / (firstRelevantRank + 1),
-    [`ndcg${prefix}`]: ndcgAt(input.rankedDocumentIds, input.expectedScores, input.cutoff)
-  };
-}
-function stringField(row, key) {
-  const value = row[key];
-  return typeof value === "string" ? value : void 0;
-}
-function stripKnown(row, keys) {
-  const known = new Set(keys);
-  return Object.fromEntries(Object.entries(row).filter(([key]) => !known.has(key)));
-}
-function normalizeCutoffs(cutoffs) {
-  return [
-    ...new Set(cutoffs.map((cutoff) => Math.trunc(cutoff)).filter((cutoff) => cutoff > 0))
-  ].sort((a, b) => a - b);
-}
-function uniqueOrdered(values) {
-  const seen = /* @__PURE__ */ new Set();
-  const out = [];
-  for (const value of values) {
-    const trimmed = value.trim();
-    if (!trimmed || seen.has(trimmed)) continue;
-    seen.add(trimmed);
-    out.push(trimmed);
-  }
-  return out;
-}
-function ndcgAt(rankedDocumentIds, expectedScores, cutoff) {
-  const dcg = rankedDocumentIds.slice(0, cutoff).reduce(
-    (sum, documentId, index) => sum + discountedGain(expectedScores[documentId] ?? 0, index),
-    0
-  );
-  const ideal = Object.values(expectedScores).filter((score) => score > 0).sort((a, b) => b - a).slice(0, cutoff).reduce((sum, score, index) => sum + discountedGain(score, index), 0);
-  return ideal === 0 ? 1 : dcg / ideal;
-}
-function discountedGain(relevance, zeroBasedRank) {
-  if (relevance <= 0) return 0;
-  return (2 ** relevance - 1) / Math.log2(zeroBasedRank + 2);
-}
-
-export {
-  calibrateBenchmarkMetric,
-  BENCHMARK_SPLIT_SEED,
-  deterministicSplit,
-  routing_exports,
-  runBenchmarkAdapter,
-  summarizeBenchmarkCampaign,
-  renderBenchmarkReportMarkdown,
-  parseJsonlRows,
-  parseTsvRows,
-  parseQrels,
-  parseBeirCorpusJsonl,
-  parseBeirQueriesJsonl,
-  buildStandardRetrievalItems,
-  createRetrievalIdBenchmarkAdapter,
-  evaluateStandardRetrieval,
-  normalizeRetrievedDocumentIds,
-  retrievalMetricsAtCutoff,
-  benchmarks_exports
-};
-//# sourceMappingURL=chunk-IYCLP2N2.js.map