@tangle-network/agent-eval 0.144.12 → 0.144.13

package/dist/benchmarks/index.js

@@ -1,2 +1,734 @@
-import { _ as deterministicSplit, a as normalizeRetrievedDocumentIds, c as parseJsonlRows, d as retrievalMetricsAtCutoff, f as renderBenchmarkReportMarkdown, g as BENCHMARK_SPLIT_SEED, h as routing_exports, i as evaluateStandardRetrieval, l as parseQrels, m as summarizeBenchmarkCampaign, n as buildStandardRetrievalItems, o as parseBeirCorpusJsonl, p as runBenchmarkAdapter, r as createRetrievalIdBenchmarkAdapter, s as parseBeirQueriesJsonl, u as parseTsvRows, v as calibrateBenchmarkMetric } from "../benchmarks-DZFX_WHN.js";
+import { t as __exportAll } from "../rolldown-runtime-8H4AJuhK.js";
+import { K as runCampaign } from "../llm-judge-dZ8P6nGI.js";
+import { x as fsCampaignStorage } from "../external-optimizer-subprocess-DrJ9hR8u.js";
+import "../campaign-BYjBAypg.js";
+import { join } from "node:path";
+//#region src/benchmarks/calibration.ts
+async function calibrateBenchmarkMetric(options) {
+	const weak = await options.adapter.evaluate(options.item, options.weakArtifact);
+	const strong = await options.adapter.evaluate(options.item, options.strongArtifact);
+	const weakScore = clamp01$1(weak.score);
+	const strongScore = clamp01$1(strong.score);
+	const maxWeakScore = options.maxWeakScore ?? .3;
+	const minStrongScore = options.minStrongScore ?? .7;
+	const minGap = options.minGap ?? .4;
+	const gap = strongScore - weakScore;
+	const reasons = [
+		weakScore <= maxWeakScore ? void 0 : `weak score ${weakScore.toFixed(3)} exceeds max ${maxWeakScore.toFixed(3)}`,
+		strongScore >= minStrongScore ? void 0 : `strong score ${strongScore.toFixed(3)} below min ${minStrongScore.toFixed(3)}`,
+		gap >= minGap ? void 0 : `gap ${gap.toFixed(3)} below min ${minGap.toFixed(3)}`
+	].filter((reason) => Boolean(reason));
+	return {
+		passed: reasons.length === 0,
+		weak,
+		strong,
+		weakScore,
+		strongScore,
+		gap,
+		reasons
+	};
+}
+function clamp01$1(value) {
+	if (!Number.isFinite(value)) return 0;
+	if (value < 0) return 0;
+	if (value > 1) return 1;
+	return value;
+}
+//#endregion
+//#region src/benchmarks/types.ts
+/**
+* 32-bit FNV-1a hash. Stable, allocation-free, deterministic across
+* runtimes. We use it to assign items to splits rather than depending
+* on a polyfilled crypto.subtle path.
+*/
+function fnv1a32(input) {
+	let h = 2166136261;
+	for (let i = 0; i < input.length; i++) {
+		h ^= input.charCodeAt(i) & 255;
+		h = h + ((h << 1) + (h << 4) + (h << 7) + (h << 8) + (h << 24)) >>> 0;
+	}
+	return h >>> 0;
+}
+/** Split-assignment seed shared across all benchmarks. Bumping this
+*  value reshuffles every split — do NOT do that lightly. */
+const BENCHMARK_SPLIT_SEED = "agent-eval-v1";
+/**
+* Assign an item id to one of `'search' | 'dev' | 'holdout'` using a
+* stable 32-bit hash of `${seed}::${id}`. Default proportions:
+*
+*   search: 60%   (optimization-readable)
+*   dev:    20%   (held-out for tuning, leak-on-purpose during dev)
+*   holdout:20%   (paper-grade held-out, gated reads)
+*/
+function deterministicSplit(itemId, seed = BENCHMARK_SPLIT_SEED) {
+	const pos = fnv1a32(`${seed}::${itemId}`) / 4294967296;
+	if (pos < .6) return "search";
+	if (pos < .8) return "dev";
+	return "holdout";
+}
+//#endregion
+//#region src/benchmarks/routing/dataset.ts
+const ROUTING_DATASET = [
+	{
+		id: "file_001",
+		category: "file",
+		prompt: "Save the meeting notes to /tmp/notes-2025-04.md as markdown.",
+		route: "fs.write",
+		synonyms: ["filesystem.write", "write_file"],
+		hardNegatives: ["fs.read", "chat.reply"]
+	},
+	{
+		id: "file_002",
+		category: "file",
+		prompt: "Read the contents of /etc/hosts and summarize the entries.",
+		route: "fs.read",
+		synonyms: ["filesystem.read", "read_file"],
+		hardNegatives: ["fs.write", "search.web"]
+	},
+	{
+		id: "file_003",
+		category: "file",
+		prompt: "List every Python file under src/ recursively.",
+		route: "fs.list",
+		synonyms: ["filesystem.list", "list_files"],
+		hardNegatives: ["fs.read", "search.code"]
+	},
+	{
+		id: "file_004",
+		category: "file",
+		prompt: "Delete the cached build at .turbo/cache.",
+		route: "fs.delete",
+		synonyms: ["filesystem.delete", "remove_file"],
+		hardNegatives: ["fs.write", "fs.list"]
+	},
+	{
+		id: "math_001",
+		category: "math",
+		prompt: "What is the integral of 3x^2 + 2x from 0 to 5?",
+		route: "math.integral",
+		synonyms: ["calculator.integral", "math.solve"],
+		hardNegatives: ["math.derivative", "chat.reply"]
+	},
+	{
+		id: "math_002",
+		category: "math",
+		prompt: "Compute the derivative of sin(x) * cos(x).",
+		route: "math.derivative",
+		synonyms: ["calculator.derivative", "math.solve"],
+		hardNegatives: ["math.integral", "math.algebra"]
+	},
+	{
+		id: "math_003",
+		category: "math",
+		prompt: "Solve 2x + 7 = 19 for x.",
+		route: "math.algebra",
+		synonyms: ["calculator.algebra", "math.solve"],
+		hardNegatives: ["math.derivative", "math.integral"]
+	},
+	{
+		id: "math_004",
+		category: "math",
+		prompt: "What is the prime factorization of 360?",
+		route: "math.numbertheory",
+		synonyms: ["calculator.factor", "math.solve"],
+		hardNegatives: ["math.algebra", "search.web"]
+	},
+	{
+		id: "search_001",
+		category: "search",
+		prompt: "Find recent papers on agent prompt optimization with held-out promotion gates.",
+		route: "search.web",
+		synonyms: ["web.search", "search.papers"],
+		hardNegatives: ["search.code", "chat.reply"]
+	},
+	{
+		id: "search_002",
+		category: "search",
+		prompt: "Search the codebase for every call site of `runProposeReview`.",
+		route: "search.code",
+		synonyms: ["code.search", "grep"],
+		hardNegatives: ["search.web", "fs.read"]
+	},
+	{
+		id: "search_003",
+		category: "search",
+		prompt: "What is the latest release of the Tangle network on GitHub?",
+		route: "search.web",
+		synonyms: ["web.search", "github.releases"],
+		hardNegatives: ["search.code", "chat.reply"]
+	},
+	{
+		id: "search_004",
+		category: "search",
+		prompt: "Find all TODO comments in the agent-eval src tree.",
+		route: "search.code",
+		synonyms: ["code.search", "grep"],
+		hardNegatives: ["search.web", "fs.list"]
+	},
+	{
+		id: "chat_001",
+		category: "chat",
+		prompt: "Hi there, how are you doing today?",
+		route: "chat.reply",
+		synonyms: ["conversation.reply"],
+		hardNegatives: ["search.web", "fs.read"]
+	},
+	{
+		id: "chat_002",
+		category: "chat",
+		prompt: "Please explain the difference between an LLM and a foundation model.",
+		route: "chat.reply",
+		synonyms: ["conversation.reply", "qa.answer"],
+		hardNegatives: ["search.web", "math.algebra"]
+	},
+	{
+		id: "chat_003",
+		category: "chat",
+		prompt: "Tell me a short joke about distributed systems.",
+		route: "chat.reply",
+		synonyms: ["conversation.reply"],
+		hardNegatives: ["search.web", "fs.read"]
+	},
+	{
+		id: "chat_004",
+		category: "chat",
+		prompt: "Acknowledge my last message with a thumbs up.",
+		route: "chat.reply",
+		synonyms: ["conversation.reply", "react"],
+		hardNegatives: ["fs.write", "search.web"]
+	}
+];
+//#endregion
+//#region src/benchmarks/routing/index.ts
+var routing_exports = /* @__PURE__ */ __exportAll({
+	ROUTING_DATASET: () => ROUTING_DATASET,
+	RoutingAdapter: () => RoutingAdapter,
+	assignSplit: () => assignSplit,
+	evaluate: () => evaluate,
+	extractRouteTokens: () => extractRouteTokens,
+	loadDataset: () => loadDataset
+});
+var RoutingAdapter = class {
+	id = "first-party/routing";
+	family = "first-party";
+	taskKind = "routing";
+	description = "Synthetic fixed-route classification smoke benchmark";
+	defaultMetric = "route_exact_match";
+	async loadDataset(split) {
+		return ROUTING_DATASET.map((item) => ({
+			id: item.id,
+			payload: item
+		})).filter((it) => assignSplitImpl(it.id) === split);
+	}
+	async evaluate(item, response) {
+		const tokens = extractRouteTokens(response);
+		const correct = new Set([item.payload.route, ...item.payload.synonyms].map((s) => s.toLowerCase()));
+		const hardNeg = new Set(item.payload.hardNegatives.map((s) => s.toLowerCase()));
+		const firstMatch = tokens.find((t) => correct.has(t.toLowerCase())) ?? null;
+		const firstHardNeg = tokens.find((t) => hardNeg.has(t.toLowerCase())) ?? null;
+		return {
+			score: firstMatch ? 1 : 0,
+			raw: {
+				firstToken: tokens[0] ?? null,
+				matchedRoute: firstMatch,
+				hitHardNegative: Boolean(firstHardNeg),
+				hardNegativeRoute: firstHardNeg,
+				category: item.payload.category
+			}
+		};
+	}
+	assignSplit(itemId) {
+		return assignSplitImpl(itemId);
+	}
+};
+function assignSplitImpl(itemId) {
+	return deterministicSplit(`routing::${itemId}`);
+}
+/**
+* Pull route-shaped tokens out of a model response. Routes look like
+* `category.action` (`fs.write`, `chat.reply`). Bare alphanumerics
+* are not routes, but `category.action` patterns are robust to most
+* model wrappers (JSON output, prose explanations, code fences).
+*/
+function extractRouteTokens(response) {
+	return response.match(/[a-z][a-z0-9_]*\.[a-z][a-z0-9_]*/gi) ?? [];
+}
+const adapter = new RoutingAdapter();
+const loadDataset = adapter.loadDataset.bind(adapter);
+const evaluate = adapter.evaluate.bind(adapter);
+const assignSplit = adapter.assignSplit.bind(adapter);
+//#endregion
+//#region src/benchmarks/runner.ts
+async function runBenchmarkAdapter(options) {
+	const storage = options.storage ?? fsCampaignStorage();
+	const benchmarkId = benchmarkIdFor(options.adapter);
+	const scenarios = await loadBenchmarkScenarios(options.adapter, options.splits);
+	const judge = benchmarkAdapterJudge(options.adapter);
+	const dispatch = async (scenario, context) => {
+		return options.respond({
+			scenario,
+			item: scenario.item,
+			context
+		});
+	};
+	const campaign = await runCampaign({
+		scenarios,
+		dispatch,
+		dispatchRef: `benchmark:${benchmarkId}`,
+		judges: [judge],
+		seed: options.seed,
+		reps: options.reps,
+		resumable: options.resumable,
+		costCeiling: options.costCeiling,
+		maxConcurrency: options.maxConcurrency,
+		dispatchTimeoutMs: options.dispatchTimeoutMs,
+		expectUsage: options.expectUsage ?? "off",
+		runDir: options.runDir,
+		repo: options.repo,
+		storage,
+		now: options.now
+	});
+	const report = summarizeBenchmarkCampaign({
+		adapter: options.adapter,
+		scenarios,
+		campaign
+	});
+	storage.ensureDir(campaign.runDir);
+	const reportJsonPath = join(campaign.runDir, "benchmark-report.json");
+	const reportMarkdownPath = join(campaign.runDir, "benchmark-report.md");
+	storage.write(reportJsonPath, `${JSON.stringify(report, null, 2)}\n`);
+	storage.write(reportMarkdownPath, renderBenchmarkReportMarkdown(report));
+	return {
+		scenarios,
+		campaign,
+		report,
+		reportJsonPath,
+		reportMarkdownPath
+	};
+}
+async function loadBenchmarkScenarios(adapter, splits = [
+	"search",
+	"dev",
+	"holdout"
+]) {
+	const benchmarkId = benchmarkIdFor(adapter);
+	const family = adapter.family ?? "custom";
+	const taskKind = adapter.taskKind ?? "custom";
+	const out = [];
+	const seen = /* @__PURE__ */ new Set();
+	for (const split of splits) {
+		const items = await adapter.loadDataset(split);
+		for (const item of items) {
+			const splitTag = item.split ?? adapter.assignSplit(item.id);
+			if (splitTag !== split) continue;
+			const key = `${benchmarkId}:${item.id}`;
+			if (seen.has(key)) continue;
+			seen.add(key);
+			out.push({
+				id: key,
+				kind: "benchmark",
+				benchmarkId,
+				family: item.family ?? family,
+				taskKind: item.taskKind ?? taskKind,
+				splitTag,
+				tags: [.../* @__PURE__ */ new Set([splitTag, ...item.tags ?? []])],
+				item
+			});
+		}
+	}
+	return out;
+}
+function benchmarkAdapterJudge(adapter) {
+	const benchmarkId = benchmarkIdFor(adapter);
+	return {
+		name: `${benchmarkId}:score`,
+		dimensions: [{
+			key: "score",
+			description: `Primary ${benchmarkId} benchmark score`
+		}, {
+			key: "passed",
+			description: "Binary pass projection for aggregate reporting"
+		}],
+		appliesTo: (scenario) => {
+			return scenario.kind === "benchmark" && scenario.id.startsWith(`${benchmarkId}:`);
+		},
+		async score({ artifact, scenario }) {
+			const evaluation = await adapter.evaluate(scenario.item, artifact);
+			return {
+				dimensions: normalizeEvaluationDimensions(evaluation),
+				composite: clamp01(evaluation.score),
+				notes: evaluation.notes ?? ""
+			};
+		}
+	};
+}
+function summarizeBenchmarkCampaign(input) {
+	const scenarioById = new Map(input.scenarios.map((scenario) => [scenario.id, scenario]));
+	const successful = input.campaign.cells.map((cell) => {
+		const scenario = scenarioById.get(cell.scenarioId);
+		const judge = firstJudgeScore(cell.judgeScores);
+		const score = judge?.composite ?? 0;
+		return {
+			cell,
+			scenario,
+			score,
+			passed: (judge?.dimensions.passed ?? (score > 0 ? 1 : 0)) >= 1,
+			dimensions: judge?.dimensions ?? {}
+		};
+	}).filter((row) => !row.cell.error);
+	const scoreValues = successful.map((row) => row.score);
+	return {
+		benchmarkId: benchmarkIdFor(input.adapter),
+		family: input.adapter.family ?? "custom",
+		taskKind: input.adapter.taskKind ?? "custom",
+		...input.adapter.source ? { source: input.adapter.source } : {},
+		runDir: input.campaign.runDir,
+		manifestHash: input.campaign.manifestHash,
+		seed: input.campaign.seed,
+		startedAt: input.campaign.startedAt,
+		endedAt: input.campaign.endedAt,
+		durationMs: input.campaign.durationMs,
+		totalItems: input.scenarios.length,
+		totalCells: input.campaign.cells.length,
+		cellsFailed: input.campaign.aggregates.cellsFailed,
+		cellsCached: input.campaign.aggregates.cellsCached,
+		totalCostUsd: input.campaign.aggregates.cost.totalCostUsd,
+		splits: summarizeSlices(successful, (row) => row.scenario?.splitTag ?? "unknown", [
+			"search",
+			"dev",
+			"holdout"
+		]),
+		tags: summarizeSlices(successful, (row) => row.scenario?.tags ?? []),
+		dimensions: summarizeDimensions(successful.map((row) => row.dimensions)),
+		score: distribution(scoreValues),
+		costUsd: distribution(successful.map((row) => row.cell.costUsd)),
+		latencyMs: distribution(successful.map((row) => row.cell.durationMs))
+	};
+}
+function renderBenchmarkReportMarkdown(report) {
+	const splitRows = Object.entries(report.splits).map(([split, summary]) => {
+		return `| ${split} | ${summary.n} | ${fmt(summary.meanScore)} | ${fmt(summary.passRate)} | ${fmt(summary.score.p90)} | ${fmt(summary.costUsd.mean)} | ${fmt(summary.latencyMs.p90)} |`;
+	}).join("\n");
+	const dimRows = Object.entries(report.dimensions).sort(([a], [b]) => a.localeCompare(b)).map(([key, dist]) => `| ${key} | ${dist.n} | ${fmt(dist.mean)} | ${fmt(dist.p90)} |`).join("\n");
+	return [
+		`# Benchmark Report: ${report.benchmarkId}`,
+		"",
+		`- family: ${report.family}`,
+		`- task kind: ${report.taskKind}`,
+		`- run dir: ${report.runDir}`,
+		`- manifest: ${report.manifestHash}`,
+		`- cells: ${report.totalCells} total, ${report.cellsFailed} failed, ${report.cellsCached} cached`,
+		`- cost: $${fmt(report.totalCostUsd)}`,
+		`- score: mean ${fmt(report.score.mean)}, median ${fmt(report.score.median)}, p90 ${fmt(report.score.p90)}, n=${report.score.n}`,
+		"",
+		"## Splits",
+		"",
+		"| split | n | mean score | pass rate | score p90 | mean cost | latency p90 ms |",
+		"| --- | ---: | ---: | ---: | ---: | ---: | ---: |",
+		splitRows || "| none | 0 | 0 | 0 | 0 | 0 | 0 |",
+		"",
+		"## Dimensions",
+		"",
+		"| dimension | n | mean | p90 |",
+		"| --- | ---: | ---: | ---: |",
+		dimRows || "| none | 0 | 0 | 0 |",
+		""
+	].join("\n");
+}
+function benchmarkIdFor(adapter) {
+	return adapter.id ?? `${adapter.family ?? "custom"}/${adapter.taskKind ?? "custom"}`;
+}
+function normalizeEvaluationDimensions(evaluation) {
+	const dimensions = { score: clamp01(evaluation.score) };
+	for (const [key, value] of Object.entries(evaluation.dimensions ?? {})) if (Number.isFinite(value)) dimensions[key] = value;
+	dimensions.passed = evaluation.passed ?? evaluation.score > 0 ? 1 : 0;
+	return dimensions;
+}
+function summarizeDimensions(rows) {
+	const values = /* @__PURE__ */ new Map();
+	for (const row of rows) for (const [key, value] of Object.entries(row)) {
+		if (!Number.isFinite(value)) continue;
+		const list = values.get(key) ?? [];
+		list.push(value);
+		values.set(key, list);
+	}
+	return Object.fromEntries([...values.entries()].map(([key, vals]) => [key, distribution(vals)]));
+}
+function summarizeSlices(rows, keyOf, knownKeys = []) {
+	const grouped = /* @__PURE__ */ new Map();
+	for (const key of knownKeys) grouped.set(key, []);
+	for (const row of rows) {
+		const keys = keyOf(row);
+		for (const key of Array.isArray(keys) ? keys : [keys]) {
+			const list = grouped.get(key) ?? [];
+			list.push(row);
+			grouped.set(key, list);
+		}
+	}
+	const out = {};
+	for (const [key, list] of grouped) {
+		const withShape = list;
+		out[key] = {
+			n: list.length,
+			meanScore: mean(withShape.map((row) => row.score)),
+			passRate: mean(withShape.map((row) => row.passed ? 1 : 0)),
+			score: distribution(withShape.map((row) => row.score)),
+			costUsd: distribution(withShape.map((row) => row.cell.costUsd)),
+			latencyMs: distribution(withShape.map((row) => row.cell.durationMs))
+		};
+	}
+	return out;
+}
+function firstJudgeScore(judgeScores) {
+	return Object.values(judgeScores)[0];
+}
+function distribution(values) {
+	const finite = [...values].filter(Number.isFinite).sort((a, b) => a - b);
+	if (finite.length === 0) return {
+		n: 0,
+		min: 0,
+		mean: 0,
+		median: 0,
+		p90: 0,
+		max: 0
+	};
+	return {
+		n: finite.length,
+		min: finite[0],
+		mean: mean(finite),
+		median: percentile(finite, .5),
+		p90: percentile(finite, .9),
+		max: finite[finite.length - 1]
+	};
+}
+function percentile(sortedValues, p) {
+	if (sortedValues.length === 0) return 0;
+	return sortedValues[Math.min(sortedValues.length - 1, Math.max(0, Math.ceil(p * sortedValues.length) - 1))];
+}
+function mean(values) {
+	const finite = values.filter(Number.isFinite);
+	if (finite.length === 0) return 0;
+	return finite.reduce((sum, value) => sum + value, 0) / finite.length;
+}
+function clamp01(value) {
+	if (!Number.isFinite(value)) return 0;
+	if (value < 0) return 0;
+	if (value > 1) return 1;
+	return value;
+}
+function fmt(value) {
+	if (!Number.isFinite(value)) return "0";
+	return value.toFixed(value === 0 || Math.abs(value) >= 10 ? 0 : 3);
+}
+//#endregion
+//#region src/benchmarks/standard-formats.ts
+function parseJsonlRows(text) {
+	return text.split(/\r?\n/).map((line) => line.trim()).filter(Boolean).map((line, index) => {
+		try {
+			return JSON.parse(line);
+		} catch (error) {
+			throw new Error(`invalid JSONL row ${index + 1}: ${error.message}`);
+		}
+	});
+}
+function parseTsvRows(text) {
+	return text.split(/\r?\n/).map((line) => line.trim()).filter(Boolean).filter((line) => !line.startsWith("#")).map((line) => line.split(/\t|\s+/));
+}
+function parseQrels(text) {
+	return parseTsvRows(text).flatMap((parts, index) => {
+		if (parts.length < 3) return [];
+		const [queryId, maybeZeroOrDocId, maybeDocIdOrScore, maybeScore] = parts;
+		if (!queryId || !maybeZeroOrDocId || !maybeDocIdOrScore) return [];
+		if (queryId.toLowerCase() === "query-id" || queryId.toLowerCase() === "qid") return [];
+		const documentId = maybeScore === void 0 ? maybeZeroOrDocId : maybeDocIdOrScore;
+		const score = Number(maybeScore === void 0 ? maybeDocIdOrScore : maybeScore);
+		if (!documentId || !Number.isFinite(score)) throw new Error(`invalid qrels row ${index + 1}: expected query id, doc id, score`);
+		return [{
+			queryId,
+			documentId,
+			score
+		}];
+	});
+}
+function parseBeirCorpusJsonl(text) {
+	return parseJsonlRows(text).map((row, index) => {
+		const id = stringField(row, "_id") ?? stringField(row, "id");
+		const body = stringField(row, "text") ?? stringField(row, "contents");
+		if (!id || body === void 0) throw new Error(`invalid BEIR corpus row ${index + 1}: expected _id/id and text/contents`);
+		return {
+			id,
+			title: stringField(row, "title"),
+			text: body,
+			metadata: stripKnown(row, [
+				"_id",
+				"id",
+				"title",
+				"text",
+				"contents"
+			])
+		};
+	});
+}
+function parseBeirQueriesJsonl(text) {
+	return parseJsonlRows(text).map((row, index) => {
+		const id = stringField(row, "_id") ?? stringField(row, "id") ?? stringField(row, "query_id");
+		const query = stringField(row, "text") ?? stringField(row, "query");
+		if (!id || !query) throw new Error(`invalid BEIR query row ${index + 1}: expected _id/id and text/query`);
+		return {
+			id,
+			text: query,
+			metadata: stripKnown(row, [
+				"_id",
+				"id",
+				"query_id",
+				"text",
+				"query"
+			])
+		};
+	});
+}
+function buildStandardRetrievalItems(options) {
+	const qrelsByQuery = /* @__PURE__ */ new Map();
+	for (const qrel of options.qrels) {
+		if (qrel.score <= 0) continue;
+		const list = qrelsByQuery.get(qrel.queryId) ?? [];
+		list.push(qrel);
+		qrelsByQuery.set(qrel.queryId, list);
+	}
+	const corpus = options.includeCorpusInPayload && options.corpus ? Object.fromEntries(options.corpus.map((document) => [document.id, document])) : void 0;
+	return options.queries.flatMap((query) => {
+		const qrels = qrelsByQuery.get(query.id) ?? [];
+		if (qrels.length === 0) return [];
+		const split = options.splitOf?.(query.id) ?? deterministicSplit(`${options.benchmarkId}:${query.id}`);
+		return [{
+			id: query.id,
+			split,
+			family: options.family,
+			taskKind: "retrieval",
+			tags: [.../* @__PURE__ */ new Set([...options.tags ?? [], split])],
+			...options.source ? { source: options.source } : {},
+			...query.metadata ? { metadata: query.metadata } : {},
+			payload: {
+				queryId: query.id,
+				query: query.text,
+				expectedDocumentIds: qrels.map((qrel) => qrel.documentId),
+				expectedScores: Object.fromEntries(qrels.map((qrel) => [qrel.documentId, qrel.score])),
+				...corpus ? { corpus } : {},
+				...query.metadata ? { metadata: query.metadata } : {}
+			}
+		}];
+	});
+}
+function createRetrievalIdBenchmarkAdapter(options) {
+	const items = buildStandardRetrievalItems(options);
+	return {
+		id: options.benchmarkId,
+		family: options.family,
+		taskKind: "retrieval",
+		source: options.source,
+		defaultMetric: options.primaryMetric ?? "ndcg@10",
+		async loadDataset(split) {
+			return items.filter((item) => item.split === split);
+		},
+		async evaluate(item, artifact) {
+			return evaluateStandardRetrieval(item.payload, artifact, options);
+		},
+		assignSplit(itemId) {
+			return options.splitOf?.(itemId) ?? deterministicSplit(`${options.benchmarkId}:${itemId}`);
+		}
+	};
+}
+function evaluateStandardRetrieval(payload, artifact, options = {}) {
+	const rankedDocumentIds = normalizeRetrievedDocumentIds(artifact, options.responseIdPattern ?? /[A-Za-z0-9_.:/-]+/g);
+	const cutoffs = normalizeCutoffs(options.cutoffs ?? [
+		1,
+		3,
+		5,
+		10
+	]);
+	const dimensions = {
+		expected_count: payload.expectedDocumentIds.length,
+		returned_count: rankedDocumentIds.length
+	};
+	for (const cutoff of cutoffs) Object.assign(dimensions, retrievalMetricsAtCutoff({
+		rankedDocumentIds,
+		expectedScores: payload.expectedScores,
+		cutoff
+	}));
+	const primaryMetric = options.primaryMetric ?? "ndcg@10";
+	const passMetric = options.passMetric ?? "hit@10";
+	const score = dimensions[primaryMetric] ?? dimensions[`ndcg@${cutoffs[cutoffs.length - 1]}`] ?? 0;
+	return {
+		score,
+		passed: (dimensions[passMetric] ?? dimensions[`hit@${cutoffs[cutoffs.length - 1]}`] ?? score) >= (options.passThreshold ?? 1),
+		dimensions,
+		raw: {
+			rankedDocumentIds,
+			expectedDocumentIds: payload.expectedDocumentIds,
+			expectedScores: payload.expectedScores
+		}
+	};
+}
+function normalizeRetrievedDocumentIds(artifact, responseIdPattern = /[A-Za-z0-9_.:/-]+/g) {
+	if (typeof artifact === "string") return uniqueOrdered((artifact.match(responseIdPattern) ?? []).map((value) => value.trim()));
+	if (Array.isArray(artifact)) return uniqueOrdered(artifact.flatMap((entry) => {
+		if (typeof entry === "string") return [entry];
+		return [entry.documentId ?? entry.docId ?? entry.id ?? ""];
+	}));
+	const objectArtifact = artifact;
+	return uniqueOrdered([
+		...objectArtifact.documentIds ?? [],
+		...objectArtifact.ids ?? [],
+		...(objectArtifact.results ?? []).map((entry) => entry.documentId ?? entry.docId ?? entry.id ?? "")
+	].map((value) => value.trim()));
+}
+function retrievalMetricsAtCutoff(input) {
+	const top = input.rankedDocumentIds.slice(0, input.cutoff);
+	const relevantIds = Object.keys(input.expectedScores).filter((id) => input.expectedScores[id] > 0);
+	const relevant = new Set(relevantIds);
+	const hits = top.filter((id) => relevant.has(id)).length;
+	const firstRelevantRank = top.findIndex((id) => relevant.has(id));
+	const prefix = `@${input.cutoff}`;
+	return {
+		[`hit${prefix}`]: hits > 0 ? 1 : 0,
+		[`recall${prefix}`]: relevant.size === 0 ? 1 : hits / relevant.size,
+		[`precision${prefix}`]: hits / input.cutoff,
+		[`mrr${prefix}`]: firstRelevantRank === -1 ? 0 : 1 / (firstRelevantRank + 1),
+		[`ndcg${prefix}`]: ndcgAt(input.rankedDocumentIds, input.expectedScores, input.cutoff)
+	};
+}
+function stringField(row, key) {
+	const value = row[key];
+	return typeof value === "string" ? value : void 0;
+}
+function stripKnown(row, keys) {
+	const known = new Set(keys);
+	return Object.fromEntries(Object.entries(row).filter(([key]) => !known.has(key)));
+}
+function normalizeCutoffs(cutoffs) {
+	return [...new Set(cutoffs.map((cutoff) => Math.trunc(cutoff)).filter((cutoff) => cutoff > 0))].sort((a, b) => a - b);
+}
+function uniqueOrdered(values) {
+	const seen = /* @__PURE__ */ new Set();
+	const out = [];
+	for (const value of values) {
+		const trimmed = value.trim();
+		if (!trimmed || seen.has(trimmed)) continue;
+		seen.add(trimmed);
+		out.push(trimmed);
+	}
+	return out;
+}
+function ndcgAt(rankedDocumentIds, expectedScores, cutoff) {
+	const dcg = rankedDocumentIds.slice(0, cutoff).reduce((sum, documentId, index) => sum + discountedGain(expectedScores[documentId] ?? 0, index), 0);
+	const ideal = Object.values(expectedScores).filter((score) => score > 0).sort((a, b) => b - a).slice(0, cutoff).reduce((sum, score, index) => sum + discountedGain(score, index), 0);
+	return ideal === 0 ? 1 : dcg / ideal;
+}
+function discountedGain(relevance, zeroBasedRank) {
+	if (relevance <= 0) return 0;
+	return (2 ** relevance - 1) / Math.log2(zeroBasedRank + 2);
+}
+//#endregion
 export { BENCHMARK_SPLIT_SEED, buildStandardRetrievalItems, calibrateBenchmarkMetric, createRetrievalIdBenchmarkAdapter, deterministicSplit, evaluateStandardRetrieval, normalizeRetrievedDocumentIds, parseBeirCorpusJsonl, parseBeirQueriesJsonl, parseJsonlRows, parseQrels, parseTsvRows, renderBenchmarkReportMarkdown, retrievalMetricsAtCutoff, routing_exports as routing, runBenchmarkAdapter, summarizeBenchmarkCampaign };
+
+//# sourceMappingURL=index.js.map