@tangle-network/agent-eval 0.144.11 → 0.144.13

package/dist/benchmarks-Dzs8CKb1.js

@@ -1,755 +0,0 @@
-import { t as __exportAll } from "./rolldown-runtime-8H4AJuhK.js";
-import { i as fsCampaignStorage } from "./single-run-lock-DFWHEB09.js";
-import { U as runCampaign } from "./skillopt-optimization-method-CQdVeM8k.js";
-import "./campaign-C2TTzQII.js";
-import { join } from "node:path";
-//#region src/benchmarks/calibration.ts
-async function calibrateBenchmarkMetric(options) {
-	const weak = await options.adapter.evaluate(options.item, options.weakArtifact);
-	const strong = await options.adapter.evaluate(options.item, options.strongArtifact);
-	const weakScore = clamp01$1(weak.score);
-	const strongScore = clamp01$1(strong.score);
-	const maxWeakScore = options.maxWeakScore ?? .3;
-	const minStrongScore = options.minStrongScore ?? .7;
-	const minGap = options.minGap ?? .4;
-	const gap = strongScore - weakScore;
-	const reasons = [
-		weakScore <= maxWeakScore ? void 0 : `weak score ${weakScore.toFixed(3)} exceeds max ${maxWeakScore.toFixed(3)}`,
-		strongScore >= minStrongScore ? void 0 : `strong score ${strongScore.toFixed(3)} below min ${minStrongScore.toFixed(3)}`,
-		gap >= minGap ? void 0 : `gap ${gap.toFixed(3)} below min ${minGap.toFixed(3)}`
-	].filter((reason) => Boolean(reason));
-	return {
-		passed: reasons.length === 0,
-		weak,
-		strong,
-		weakScore,
-		strongScore,
-		gap,
-		reasons
-	};
-}
-function clamp01$1(value) {
-	if (!Number.isFinite(value)) return 0;
-	if (value < 0) return 0;
-	if (value > 1) return 1;
-	return value;
-}
-//#endregion
-//#region src/benchmarks/types.ts
-/**
-* 32-bit FNV-1a hash. Stable, allocation-free, deterministic across
-* runtimes. We use it to assign items to splits rather than depending
-* on a polyfilled crypto.subtle path.
-*/
-function fnv1a32(input) {
-	let h = 2166136261;
-	for (let i = 0; i < input.length; i++) {
-		h ^= input.charCodeAt(i) & 255;
-		h = h + ((h << 1) + (h << 4) + (h << 7) + (h << 8) + (h << 24)) >>> 0;
-	}
-	return h >>> 0;
-}
-/** Split-assignment seed shared across all benchmarks. Bumping this
-*  value reshuffles every split — do NOT do that lightly. */
-const BENCHMARK_SPLIT_SEED = "agent-eval-v1";
-/**
-* Assign an item id to one of `'search' | 'dev' | 'holdout'` using a
-* stable 32-bit hash of `${seed}::${id}`. Default proportions:
-*
-*   search: 60%   (optimization-readable)
-*   dev:    20%   (held-out for tuning, leak-on-purpose during dev)
-*   holdout:20%   (paper-grade held-out, gated reads)
-*/
-function deterministicSplit(itemId, seed = BENCHMARK_SPLIT_SEED) {
-	const pos = fnv1a32(`${seed}::${itemId}`) / 4294967296;
-	if (pos < .6) return "search";
-	if (pos < .8) return "dev";
-	return "holdout";
-}
-//#endregion
-//#region src/benchmarks/routing/dataset.ts
-const ROUTING_DATASET = [
-	{
-		id: "file_001",
-		category: "file",
-		prompt: "Save the meeting notes to /tmp/notes-2025-04.md as markdown.",
-		route: "fs.write",
-		synonyms: ["filesystem.write", "write_file"],
-		hardNegatives: ["fs.read", "chat.reply"]
-	},
-	{
-		id: "file_002",
-		category: "file",
-		prompt: "Read the contents of /etc/hosts and summarize the entries.",
-		route: "fs.read",
-		synonyms: ["filesystem.read", "read_file"],
-		hardNegatives: ["fs.write", "search.web"]
-	},
-	{
-		id: "file_003",
-		category: "file",
-		prompt: "List every Python file under src/ recursively.",
-		route: "fs.list",
-		synonyms: ["filesystem.list", "list_files"],
-		hardNegatives: ["fs.read", "search.code"]
-	},
-	{
-		id: "file_004",
-		category: "file",
-		prompt: "Delete the cached build at .turbo/cache.",
-		route: "fs.delete",
-		synonyms: ["filesystem.delete", "remove_file"],
-		hardNegatives: ["fs.write", "fs.list"]
-	},
-	{
-		id: "math_001",
-		category: "math",
-		prompt: "What is the integral of 3x^2 + 2x from 0 to 5?",
-		route: "math.integral",
-		synonyms: ["calculator.integral", "math.solve"],
-		hardNegatives: ["math.derivative", "chat.reply"]
-	},
-	{
-		id: "math_002",
-		category: "math",
-		prompt: "Compute the derivative of sin(x) * cos(x).",
-		route: "math.derivative",
-		synonyms: ["calculator.derivative", "math.solve"],
-		hardNegatives: ["math.integral", "math.algebra"]
-	},
-	{
-		id: "math_003",
-		category: "math",
-		prompt: "Solve 2x + 7 = 19 for x.",
-		route: "math.algebra",
-		synonyms: ["calculator.algebra", "math.solve"],
-		hardNegatives: ["math.derivative", "math.integral"]
-	},
-	{
-		id: "math_004",
-		category: "math",
-		prompt: "What is the prime factorization of 360?",
-		route: "math.numbertheory",
-		synonyms: ["calculator.factor", "math.solve"],
-		hardNegatives: ["math.algebra", "search.web"]
-	},
-	{
-		id: "search_001",
-		category: "search",
-		prompt: "Find recent papers on agent prompt optimization with held-out promotion gates.",
-		route: "search.web",
-		synonyms: ["web.search", "search.papers"],
-		hardNegatives: ["search.code", "chat.reply"]
-	},
-	{
-		id: "search_002",
-		category: "search",
-		prompt: "Search the codebase for every call site of `runProposeReview`.",
-		route: "search.code",
-		synonyms: ["code.search", "grep"],
-		hardNegatives: ["search.web", "fs.read"]
-	},
-	{
-		id: "search_003",
-		category: "search",
-		prompt: "What is the latest release of the Tangle network on GitHub?",
-		route: "search.web",
-		synonyms: ["web.search", "github.releases"],
-		hardNegatives: ["search.code", "chat.reply"]
-	},
-	{
-		id: "search_004",
-		category: "search",
-		prompt: "Find all TODO comments in the agent-eval src tree.",
-		route: "search.code",
-		synonyms: ["code.search", "grep"],
-		hardNegatives: ["search.web", "fs.list"]
-	},
-	{
-		id: "chat_001",
-		category: "chat",
-		prompt: "Hi there, how are you doing today?",
-		route: "chat.reply",
-		synonyms: ["conversation.reply"],
-		hardNegatives: ["search.web", "fs.read"]
-	},
-	{
-		id: "chat_002",
-		category: "chat",
-		prompt: "Please explain the difference between an LLM and a foundation model.",
-		route: "chat.reply",
-		synonyms: ["conversation.reply", "qa.answer"],
-		hardNegatives: ["search.web", "math.algebra"]
-	},
-	{
-		id: "chat_003",
-		category: "chat",
-		prompt: "Tell me a short joke about distributed systems.",
-		route: "chat.reply",
-		synonyms: ["conversation.reply"],
-		hardNegatives: ["search.web", "fs.read"]
-	},
-	{
-		id: "chat_004",
-		category: "chat",
-		prompt: "Acknowledge my last message with a thumbs up.",
-		route: "chat.reply",
-		synonyms: ["conversation.reply", "react"],
-		hardNegatives: ["fs.write", "search.web"]
-	}
-];
-//#endregion
-//#region src/benchmarks/routing/index.ts
-var routing_exports = /* @__PURE__ */ __exportAll({
-	ROUTING_DATASET: () => ROUTING_DATASET,
-	RoutingAdapter: () => RoutingAdapter,
-	assignSplit: () => assignSplit,
-	evaluate: () => evaluate,
-	extractRouteTokens: () => extractRouteTokens,
-	loadDataset: () => loadDataset
-});
-var RoutingAdapter = class {
-	id = "first-party/routing";
-	family = "first-party";
-	taskKind = "routing";
-	description = "Synthetic fixed-route classification smoke benchmark";
-	defaultMetric = "route_exact_match";
-	async loadDataset(split) {
-		return ROUTING_DATASET.map((item) => ({
-			id: item.id,
-			payload: item
-		})).filter((it) => assignSplitImpl(it.id) === split);
-	}
-	async evaluate(item, response) {
-		const tokens = extractRouteTokens(response);
-		const correct = new Set([item.payload.route, ...item.payload.synonyms].map((s) => s.toLowerCase()));
-		const hardNeg = new Set(item.payload.hardNegatives.map((s) => s.toLowerCase()));
-		const firstMatch = tokens.find((t) => correct.has(t.toLowerCase())) ?? null;
-		const firstHardNeg = tokens.find((t) => hardNeg.has(t.toLowerCase())) ?? null;
-		return {
-			score: firstMatch ? 1 : 0,
-			raw: {
-				firstToken: tokens[0] ?? null,
-				matchedRoute: firstMatch,
-				hitHardNegative: Boolean(firstHardNeg),
-				hardNegativeRoute: firstHardNeg,
-				category: item.payload.category
-			}
-		};
-	}
-	assignSplit(itemId) {
-		return assignSplitImpl(itemId);
-	}
-};
-function assignSplitImpl(itemId) {
-	return deterministicSplit(`routing::${itemId}`);
-}
-/**
-* Pull route-shaped tokens out of a model response. Routes look like
-* `category.action` (`fs.write`, `chat.reply`). Bare alphanumerics
-* are not routes, but `category.action` patterns are robust to most
-* model wrappers (JSON output, prose explanations, code fences).
-*/
-function extractRouteTokens(response) {
-	return response.match(/[a-z][a-z0-9_]*\.[a-z][a-z0-9_]*/gi) ?? [];
-}
-const adapter = new RoutingAdapter();
-const loadDataset = adapter.loadDataset.bind(adapter);
-const evaluate = adapter.evaluate.bind(adapter);
-const assignSplit = adapter.assignSplit.bind(adapter);
-//#endregion
-//#region src/benchmarks/runner.ts
-async function runBenchmarkAdapter(options) {
-	const storage = options.storage ?? fsCampaignStorage();
-	const benchmarkId = benchmarkIdFor(options.adapter);
-	const scenarios = await loadBenchmarkScenarios(options.adapter, options.splits);
-	const judge = benchmarkAdapterJudge(options.adapter);
-	const dispatch = async (scenario, context) => {
-		return options.respond({
-			scenario,
-			item: scenario.item,
-			context
-		});
-	};
-	const campaign = await runCampaign({
-		scenarios,
-		dispatch,
-		dispatchRef: `benchmark:${benchmarkId}`,
-		judges: [judge],
-		seed: options.seed,
-		reps: options.reps,
-		resumable: options.resumable,
-		costCeiling: options.costCeiling,
-		maxConcurrency: options.maxConcurrency,
-		dispatchTimeoutMs: options.dispatchTimeoutMs,
-		expectUsage: options.expectUsage ?? "off",
-		runDir: options.runDir,
-		repo: options.repo,
-		storage,
-		now: options.now
-	});
-	const report = summarizeBenchmarkCampaign({
-		adapter: options.adapter,
-		scenarios,
-		campaign
-	});
-	storage.ensureDir(campaign.runDir);
-	const reportJsonPath = join(campaign.runDir, "benchmark-report.json");
-	const reportMarkdownPath = join(campaign.runDir, "benchmark-report.md");
-	storage.write(reportJsonPath, `${JSON.stringify(report, null, 2)}\n`);
-	storage.write(reportMarkdownPath, renderBenchmarkReportMarkdown(report));
-	return {
-		scenarios,
-		campaign,
-		report,
-		reportJsonPath,
-		reportMarkdownPath
-	};
-}
-async function loadBenchmarkScenarios(adapter, splits = [
-	"search",
-	"dev",
-	"holdout"
-]) {
-	const benchmarkId = benchmarkIdFor(adapter);
-	const family = adapter.family ?? "custom";
-	const taskKind = adapter.taskKind ?? "custom";
-	const out = [];
-	const seen = /* @__PURE__ */ new Set();
-	for (const split of splits) {
-		const items = await adapter.loadDataset(split);
-		for (const item of items) {
-			const splitTag = item.split ?? adapter.assignSplit(item.id);
-			if (splitTag !== split) continue;
-			const key = `${benchmarkId}:${item.id}`;
-			if (seen.has(key)) continue;
-			seen.add(key);
-			out.push({
-				id: key,
-				kind: "benchmark",
-				benchmarkId,
-				family: item.family ?? family,
-				taskKind: item.taskKind ?? taskKind,
-				splitTag,
-				tags: [.../* @__PURE__ */ new Set([splitTag, ...item.tags ?? []])],
-				item
-			});
-		}
-	}
-	return out;
-}
-function benchmarkAdapterJudge(adapter) {
-	const benchmarkId = benchmarkIdFor(adapter);
-	return {
-		name: `${benchmarkId}:score`,
-		dimensions: [{
-			key: "score",
-			description: `Primary ${benchmarkId} benchmark score`
-		}, {
-			key: "passed",
-			description: "Binary pass projection for aggregate reporting"
-		}],
-		appliesTo: (scenario) => {
-			return scenario.kind === "benchmark" && scenario.id.startsWith(`${benchmarkId}:`);
-		},
-		async score({ artifact, scenario }) {
-			const evaluation = await adapter.evaluate(scenario.item, artifact);
-			return {
-				dimensions: normalizeEvaluationDimensions(evaluation),
-				composite: clamp01(evaluation.score),
-				notes: evaluation.notes ?? ""
-			};
-		}
-	};
-}
-function summarizeBenchmarkCampaign(input) {
-	const scenarioById = new Map(input.scenarios.map((scenario) => [scenario.id, scenario]));
-	const successful = input.campaign.cells.map((cell) => {
-		const scenario = scenarioById.get(cell.scenarioId);
-		const judge = firstJudgeScore(cell.judgeScores);
-		const score = judge?.composite ?? 0;
-		return {
-			cell,
-			scenario,
-			score,
-			passed: (judge?.dimensions.passed ?? (score > 0 ? 1 : 0)) >= 1,
-			dimensions: judge?.dimensions ?? {}
-		};
-	}).filter((row) => !row.cell.error);
-	const scoreValues = successful.map((row) => row.score);
-	return {
-		benchmarkId: benchmarkIdFor(input.adapter),
-		family: input.adapter.family ?? "custom",
-		taskKind: input.adapter.taskKind ?? "custom",
-		...input.adapter.source ? { source: input.adapter.source } : {},
-		runDir: input.campaign.runDir,
-		manifestHash: input.campaign.manifestHash,
-		seed: input.campaign.seed,
-		startedAt: input.campaign.startedAt,
-		endedAt: input.campaign.endedAt,
-		durationMs: input.campaign.durationMs,
-		totalItems: input.scenarios.length,
-		totalCells: input.campaign.cells.length,
-		cellsFailed: input.campaign.aggregates.cellsFailed,
-		cellsCached: input.campaign.aggregates.cellsCached,
-		totalCostUsd: input.campaign.aggregates.cost.totalCostUsd,
-		splits: summarizeSlices(successful, (row) => row.scenario?.splitTag ?? "unknown", [
-			"search",
-			"dev",
-			"holdout"
-		]),
-		tags: summarizeSlices(successful, (row) => row.scenario?.tags ?? []),
-		dimensions: summarizeDimensions(successful.map((row) => row.dimensions)),
-		score: distribution(scoreValues),
-		costUsd: distribution(successful.map((row) => row.cell.costUsd)),
-		latencyMs: distribution(successful.map((row) => row.cell.durationMs))
-	};
-}
-function renderBenchmarkReportMarkdown(report) {
-	const splitRows = Object.entries(report.splits).map(([split, summary]) => {
-		return `| ${split} | ${summary.n} | ${fmt(summary.meanScore)} | ${fmt(summary.passRate)} | ${fmt(summary.score.p90)} | ${fmt(summary.costUsd.mean)} | ${fmt(summary.latencyMs.p90)} |`;
-	}).join("\n");
-	const dimRows = Object.entries(report.dimensions).sort(([a], [b]) => a.localeCompare(b)).map(([key, dist]) => `| ${key} | ${dist.n} | ${fmt(dist.mean)} | ${fmt(dist.p90)} |`).join("\n");
-	return [
-		`# Benchmark Report: ${report.benchmarkId}`,
-		"",
-		`- family: ${report.family}`,
-		`- task kind: ${report.taskKind}`,
-		`- run dir: ${report.runDir}`,
-		`- manifest: ${report.manifestHash}`,
-		`- cells: ${report.totalCells} total, ${report.cellsFailed} failed, ${report.cellsCached} cached`,
-		`- cost: $${fmt(report.totalCostUsd)}`,
-		`- score: mean ${fmt(report.score.mean)}, median ${fmt(report.score.median)}, p90 ${fmt(report.score.p90)}, n=${report.score.n}`,
-		"",
-		"## Splits",
-		"",
-		"| split | n | mean score | pass rate | score p90 | mean cost | latency p90 ms |",
-		"| --- | ---: | ---: | ---: | ---: | ---: | ---: |",
-		splitRows || "| none | 0 | 0 | 0 | 0 | 0 | 0 |",
-		"",
-		"## Dimensions",
-		"",
-		"| dimension | n | mean | p90 |",
-		"| --- | ---: | ---: | ---: |",
-		dimRows || "| none | 0 | 0 | 0 |",
-		""
-	].join("\n");
-}
-function benchmarkIdFor(adapter) {
-	return adapter.id ?? `${adapter.family ?? "custom"}/${adapter.taskKind ?? "custom"}`;
-}
-function normalizeEvaluationDimensions(evaluation) {
-	const dimensions = { score: clamp01(evaluation.score) };
-	for (const [key, value] of Object.entries(evaluation.dimensions ?? {})) if (Number.isFinite(value)) dimensions[key] = value;
-	dimensions.passed = evaluation.passed ?? evaluation.score > 0 ? 1 : 0;
-	return dimensions;
-}
-function summarizeDimensions(rows) {
-	const values = /* @__PURE__ */ new Map();
-	for (const row of rows) for (const [key, value] of Object.entries(row)) {
-		if (!Number.isFinite(value)) continue;
-		const list = values.get(key) ?? [];
-		list.push(value);
-		values.set(key, list);
-	}
-	return Object.fromEntries([...values.entries()].map(([key, vals]) => [key, distribution(vals)]));
-}
-function summarizeSlices(rows, keyOf, knownKeys = []) {
-	const grouped = /* @__PURE__ */ new Map();
-	for (const key of knownKeys) grouped.set(key, []);
-	for (const row of rows) {
-		const keys = keyOf(row);
-		for (const key of Array.isArray(keys) ? keys : [keys]) {
-			const list = grouped.get(key) ?? [];
-			list.push(row);
-			grouped.set(key, list);
-		}
-	}
-	const out = {};
-	for (const [key, list] of grouped) {
-		const withShape = list;
-		out[key] = {
-			n: list.length,
-			meanScore: mean(withShape.map((row) => row.score)),
-			passRate: mean(withShape.map((row) => row.passed ? 1 : 0)),
-			score: distribution(withShape.map((row) => row.score)),
-			costUsd: distribution(withShape.map((row) => row.cell.costUsd)),
-			latencyMs: distribution(withShape.map((row) => row.cell.durationMs))
-		};
-	}
-	return out;
-}
-function firstJudgeScore(judgeScores) {
-	return Object.values(judgeScores)[0];
-}
-function distribution(values) {
-	const finite = [...values].filter(Number.isFinite).sort((a, b) => a - b);
-	if (finite.length === 0) return {
-		n: 0,
-		min: 0,
-		mean: 0,
-		median: 0,
-		p90: 0,
-		max: 0
-	};
-	return {
-		n: finite.length,
-		min: finite[0],
-		mean: mean(finite),
-		median: percentile(finite, .5),
-		p90: percentile(finite, .9),
-		max: finite[finite.length - 1]
-	};
-}
-function percentile(sortedValues, p) {
-	if (sortedValues.length === 0) return 0;
-	return sortedValues[Math.min(sortedValues.length - 1, Math.max(0, Math.ceil(p * sortedValues.length) - 1))];
-}
-function mean(values) {
-	const finite = values.filter(Number.isFinite);
-	if (finite.length === 0) return 0;
-	return finite.reduce((sum, value) => sum + value, 0) / finite.length;
-}
-function clamp01(value) {
-	if (!Number.isFinite(value)) return 0;
-	if (value < 0) return 0;
-	if (value > 1) return 1;
-	return value;
-}
-function fmt(value) {
-	if (!Number.isFinite(value)) return "0";
-	return value.toFixed(value === 0 || Math.abs(value) >= 10 ? 0 : 3);
-}
-//#endregion
-//#region src/benchmarks/standard-formats.ts
-function parseJsonlRows(text) {
-	return text.split(/\r?\n/).map((line) => line.trim()).filter(Boolean).map((line, index) => {
-		try {
-			return JSON.parse(line);
-		} catch (error) {
-			throw new Error(`invalid JSONL row ${index + 1}: ${error.message}`);
-		}
-	});
-}
-function parseTsvRows(text) {
-	return text.split(/\r?\n/).map((line) => line.trim()).filter(Boolean).filter((line) => !line.startsWith("#")).map((line) => line.split(/\t|\s+/));
-}
-function parseQrels(text) {
-	return parseTsvRows(text).flatMap((parts, index) => {
-		if (parts.length < 3) return [];
-		const [queryId, maybeZeroOrDocId, maybeDocIdOrScore, maybeScore] = parts;
-		if (!queryId || !maybeZeroOrDocId || !maybeDocIdOrScore) return [];
-		if (queryId.toLowerCase() === "query-id" || queryId.toLowerCase() === "qid") return [];
-		const documentId = maybeScore === void 0 ? maybeZeroOrDocId : maybeDocIdOrScore;
-		const score = Number(maybeScore === void 0 ? maybeDocIdOrScore : maybeScore);
-		if (!documentId || !Number.isFinite(score)) throw new Error(`invalid qrels row ${index + 1}: expected query id, doc id, score`);
-		return [{
-			queryId,
-			documentId,
-			score
-		}];
-	});
-}
-function parseBeirCorpusJsonl(text) {
-	return parseJsonlRows(text).map((row, index) => {
-		const id = stringField(row, "_id") ?? stringField(row, "id");
-		const body = stringField(row, "text") ?? stringField(row, "contents");
-		if (!id || body === void 0) throw new Error(`invalid BEIR corpus row ${index + 1}: expected _id/id and text/contents`);
-		return {
-			id,
-			title: stringField(row, "title"),
-			text: body,
-			metadata: stripKnown(row, [
-				"_id",
-				"id",
-				"title",
-				"text",
-				"contents"
-			])
-		};
-	});
-}
-function parseBeirQueriesJsonl(text) {
-	return parseJsonlRows(text).map((row, index) => {
-		const id = stringField(row, "_id") ?? stringField(row, "id") ?? stringField(row, "query_id");
-		const query = stringField(row, "text") ?? stringField(row, "query");
-		if (!id || !query) throw new Error(`invalid BEIR query row ${index + 1}: expected _id/id and text/query`);
-		return {
-			id,
-			text: query,
-			metadata: stripKnown(row, [
-				"_id",
-				"id",
-				"query_id",
-				"text",
-				"query"
-			])
-		};
-	});
-}
-function buildStandardRetrievalItems(options) {
-	const qrelsByQuery = /* @__PURE__ */ new Map();
-	for (const qrel of options.qrels) {
-		if (qrel.score <= 0) continue;
-		const list = qrelsByQuery.get(qrel.queryId) ?? [];
-		list.push(qrel);
-		qrelsByQuery.set(qrel.queryId, list);
-	}
-	const corpus = options.includeCorpusInPayload && options.corpus ? Object.fromEntries(options.corpus.map((document) => [document.id, document])) : void 0;
-	return options.queries.flatMap((query) => {
-		const qrels = qrelsByQuery.get(query.id) ?? [];
-		if (qrels.length === 0) return [];
-		const split = options.splitOf?.(query.id) ?? deterministicSplit(`${options.benchmarkId}:${query.id}`);
-		return [{
-			id: query.id,
-			split,
-			family: options.family,
-			taskKind: "retrieval",
-			tags: [.../* @__PURE__ */ new Set([...options.tags ?? [], split])],
-			...options.source ? { source: options.source } : {},
-			...query.metadata ? { metadata: query.metadata } : {},
-			payload: {
-				queryId: query.id,
-				query: query.text,
-				expectedDocumentIds: qrels.map((qrel) => qrel.documentId),
-				expectedScores: Object.fromEntries(qrels.map((qrel) => [qrel.documentId, qrel.score])),
-				...corpus ? { corpus } : {},
-				...query.metadata ? { metadata: query.metadata } : {}
-			}
-		}];
-	});
-}
-function createRetrievalIdBenchmarkAdapter(options) {
-	const items = buildStandardRetrievalItems(options);
-	return {
-		id: options.benchmarkId,
-		family: options.family,
-		taskKind: "retrieval",
-		source: options.source,
-		defaultMetric: options.primaryMetric ?? "ndcg@10",
-		async loadDataset(split) {
-			return items.filter((item) => item.split === split);
-		},
-		async evaluate(item, artifact) {
-			return evaluateStandardRetrieval(item.payload, artifact, options);
-		},
-		assignSplit(itemId) {
-			return options.splitOf?.(itemId) ?? deterministicSplit(`${options.benchmarkId}:${itemId}`);
-		}
-	};
-}
-function evaluateStandardRetrieval(payload, artifact, options = {}) {
-	const rankedDocumentIds = normalizeRetrievedDocumentIds(artifact, options.responseIdPattern ?? /[A-Za-z0-9_.:/-]+/g);
-	const cutoffs = normalizeCutoffs(options.cutoffs ?? [
-		1,
-		3,
-		5,
-		10
-	]);
-	const dimensions = {
-		expected_count: payload.expectedDocumentIds.length,
-		returned_count: rankedDocumentIds.length
-	};
-	for (const cutoff of cutoffs) Object.assign(dimensions, retrievalMetricsAtCutoff({
-		rankedDocumentIds,
-		expectedScores: payload.expectedScores,
-		cutoff
-	}));
-	const primaryMetric = options.primaryMetric ?? "ndcg@10";
-	const passMetric = options.passMetric ?? "hit@10";
-	const score = dimensions[primaryMetric] ?? dimensions[`ndcg@${cutoffs[cutoffs.length - 1]}`] ?? 0;
-	return {
-		score,
-		passed: (dimensions[passMetric] ?? dimensions[`hit@${cutoffs[cutoffs.length - 1]}`] ?? score) >= (options.passThreshold ?? 1),
-		dimensions,
-		raw: {
-			rankedDocumentIds,
-			expectedDocumentIds: payload.expectedDocumentIds,
-			expectedScores: payload.expectedScores
-		}
-	};
-}
-function normalizeRetrievedDocumentIds(artifact, responseIdPattern = /[A-Za-z0-9_.:/-]+/g) {
-	if (typeof artifact === "string") return uniqueOrdered((artifact.match(responseIdPattern) ?? []).map((value) => value.trim()));
-	if (Array.isArray(artifact)) return uniqueOrdered(artifact.flatMap((entry) => {
-		if (typeof entry === "string") return [entry];
-		return [entry.documentId ?? entry.docId ?? entry.id ?? ""];
-	}));
-	const objectArtifact = artifact;
-	return uniqueOrdered([
-		...objectArtifact.documentIds ?? [],
-		...objectArtifact.ids ?? [],
-		...(objectArtifact.results ?? []).map((entry) => entry.documentId ?? entry.docId ?? entry.id ?? "")
-	].map((value) => value.trim()));
-}
-function retrievalMetricsAtCutoff(input) {
-	const top = input.rankedDocumentIds.slice(0, input.cutoff);
-	const relevantIds = Object.keys(input.expectedScores).filter((id) => input.expectedScores[id] > 0);
-	const relevant = new Set(relevantIds);
-	const hits = top.filter((id) => relevant.has(id)).length;
-	const firstRelevantRank = top.findIndex((id) => relevant.has(id));
-	const prefix = `@${input.cutoff}`;
-	return {
-		[`hit${prefix}`]: hits > 0 ? 1 : 0,
-		[`recall${prefix}`]: relevant.size === 0 ? 1 : hits / relevant.size,
-		[`precision${prefix}`]: hits / input.cutoff,
-		[`mrr${prefix}`]: firstRelevantRank === -1 ? 0 : 1 / (firstRelevantRank + 1),
-		[`ndcg${prefix}`]: ndcgAt(input.rankedDocumentIds, input.expectedScores, input.cutoff)
-	};
-}
-function stringField(row, key) {
-	const value = row[key];
-	return typeof value === "string" ? value : void 0;
-}
-function stripKnown(row, keys) {
-	const known = new Set(keys);
-	return Object.fromEntries(Object.entries(row).filter(([key]) => !known.has(key)));
-}
-function normalizeCutoffs(cutoffs) {
-	return [...new Set(cutoffs.map((cutoff) => Math.trunc(cutoff)).filter((cutoff) => cutoff > 0))].sort((a, b) => a - b);
-}
-function uniqueOrdered(values) {
-	const seen = /* @__PURE__ */ new Set();
-	const out = [];
-	for (const value of values) {
-		const trimmed = value.trim();
-		if (!trimmed || seen.has(trimmed)) continue;
-		seen.add(trimmed);
-		out.push(trimmed);
-	}
-	return out;
-}
-function ndcgAt(rankedDocumentIds, expectedScores, cutoff) {
-	const dcg = rankedDocumentIds.slice(0, cutoff).reduce((sum, documentId, index) => sum + discountedGain(expectedScores[documentId] ?? 0, index), 0);
-	const ideal = Object.values(expectedScores).filter((score) => score > 0).sort((a, b) => b - a).slice(0, cutoff).reduce((sum, score, index) => sum + discountedGain(score, index), 0);
-	return ideal === 0 ? 1 : dcg / ideal;
-}
-function discountedGain(relevance, zeroBasedRank) {
-	if (relevance <= 0) return 0;
-	return (2 ** relevance - 1) / Math.log2(zeroBasedRank + 2);
-}
-//#endregion
-//#region src/benchmarks/index.ts
-var benchmarks_exports = /* @__PURE__ */ __exportAll({
-	BENCHMARK_SPLIT_SEED: () => BENCHMARK_SPLIT_SEED,
-	buildStandardRetrievalItems: () => buildStandardRetrievalItems,
-	calibrateBenchmarkMetric: () => calibrateBenchmarkMetric,
-	createRetrievalIdBenchmarkAdapter: () => createRetrievalIdBenchmarkAdapter,
-	deterministicSplit: () => deterministicSplit,
-	evaluateStandardRetrieval: () => evaluateStandardRetrieval,
-	normalizeRetrievedDocumentIds: () => normalizeRetrievedDocumentIds,
-	parseBeirCorpusJsonl: () => parseBeirCorpusJsonl,
-	parseBeirQueriesJsonl: () => parseBeirQueriesJsonl,
-	parseJsonlRows: () => parseJsonlRows,
-	parseQrels: () => parseQrels,
-	parseTsvRows: () => parseTsvRows,
-	renderBenchmarkReportMarkdown: () => renderBenchmarkReportMarkdown,
-	retrievalMetricsAtCutoff: () => retrievalMetricsAtCutoff,
-	routing: () => routing_exports,
-	runBenchmarkAdapter: () => runBenchmarkAdapter,
-	summarizeBenchmarkCampaign: () => summarizeBenchmarkCampaign
-});
-//#endregion
-export { deterministicSplit as _, normalizeRetrievedDocumentIds as a, parseJsonlRows as c, retrievalMetricsAtCutoff as d, renderBenchmarkReportMarkdown as f, BENCHMARK_SPLIT_SEED as g, routing_exports as h, evaluateStandardRetrieval as i, parseQrels as l, summarizeBenchmarkCampaign as m, buildStandardRetrievalItems as n, parseBeirCorpusJsonl as o, runBenchmarkAdapter as p, createRetrievalIdBenchmarkAdapter as r, parseBeirQueriesJsonl as s, benchmarks_exports as t, parseTsvRows as u, calibrateBenchmarkMetric as v };
-
-//# sourceMappingURL=benchmarks-Dzs8CKb1.js.map