@tangle-network/agent-eval 0.216.0 → 0.217.1

package/dist/benchmark-BywPeAUo.js

@@ -1,562 +0,0 @@
-import { i as compareCodeUnits } from "./canonical-BsZVgBhW.js";
-import { t as assertValidAnalystUsageReceipt } from "./usage-receipt-Cx3xNeyB.js";
-import { performance } from "node:perf_hooks";
-import { linearSumAssignment } from "linear-sum-assignment";
-//#region src/analyst/benchmark-scoring.ts
-function scoreAnalystFindings(testCase, findings) {
-	assertValidAnalystScoringCase(testCase);
-	const matchedFindingByIssue = matchFindingsToIssues(testCase.expectedIssues, findings);
-	const matchedIssueIds = testCase.expectedIssues.filter((_, index) => matchedFindingByIssue.has(index)).map((issue) => issue.id);
-	const missedIssueIds = testCase.expectedIssues.filter((_, index) => !matchedFindingByIssue.has(index)).map((issue) => issue.id);
-	const supportedFindingIndexes = new Set(matchedFindingByIssue.values());
-	const unsupportedFindingIndexes = findings.map((_, index) => index).filter((index) => !supportedFindingIndexes.has(index));
-	const expectedIssueCount = testCase.expectedIssues.length;
-	const issueRecall = expectedIssueCount === 0 ? 1 : matchedIssueIds.length / expectedIssueCount;
-	const findingPrecision = findings.length === 0 ? expectedIssueCount === 0 ? 1 : 0 : supportedFindingIndexes.size / findings.length;
-	const f1 = harmonicMeanScore(findingPrecision, issueRecall);
-	const allEvidence = findings.flatMap((finding) => finding.evidence_refs);
-	const criticalIssues = testCase.expectedIssues.filter((issue) => (issue.criticalEvidence?.length ?? 0) > 0);
-	const criticalHits = testCase.expectedIssues.filter((issue) => {
-		if ((issue.criticalEvidence?.length ?? 0) === 0) return false;
-		return matchesEvidence(allEvidence, issue.criticalEvidence ?? [], "any");
-	}).length;
-	const findingsWithEvidence = findings.filter((finding) => finding.evidence_refs.length > 0).length;
-	const unlabeledEvidence = testCase.labeledEvidence ? allEvidence.filter((ref) => !testCase.labeledEvidence.some((expected) => evidenceMatches(ref, expected))) : [];
-	return {
-		expectedIssueCount,
-		matchedIssueIds,
-		missedIssueIds,
-		supportedFindingIndexes: [...supportedFindingIndexes].sort((a, b) => a - b),
-		unsupportedFindingIndexes,
-		unlabeledEvidence,
-		issueRecall,
-		findingPrecision,
-		f1,
-		criticalStepAccuracy: criticalIssues.length === 0 ? null : criticalHits / criticalIssues.length,
-		citationCoverage: findings.length === 0 ? null : findingsWithEvidence / findings.length,
-		citationExcerptCoverage: allEvidence.length === 0 ? null : allEvidence.filter((evidence) => Boolean(evidence.excerpt?.trim())).length / allEvidence.length,
-		citationLabelAgreement: testCase.labeledEvidence === void 0 ? null : allEvidence.length === 0 ? findings.length === 0 ? null : 0 : (allEvidence.length - unlabeledEvidence.length) / allEvidence.length,
-		predictionOnLabelEmptyCase: expectedIssueCount === 0 && findings.length > 0
-	};
-}
-function assertValidAnalystScoringCase(testCase) {
-	if (!testCase.id.trim()) throw new TypeError("analyst benchmark case id must not be empty");
-	const ids = /* @__PURE__ */ new Set();
-	for (const issue of testCase.expectedIssues) {
-		if (!issue.id.trim()) throw new TypeError(`${testCase.id}: expected issue id must not be empty`);
-		if (ids.has(issue.id)) throw new TypeError(`${testCase.id}: duplicate expected issue id '${issue.id}'`);
-		ids.add(issue.id);
-		if (!issue.findingIds?.length && !issue.areas?.length && !issue.subjects?.length && !issue.evidence?.length) throw new TypeError(`${testCase.id}/${issue.id}: expected issue must identify a finding by id, area, subject, or evidence`);
-	}
-	for (const ref of testCase.labeledEvidence ?? []) if (!ref.uri.trim()) throw new TypeError(`${testCase.id}: labeled evidence URI must not be empty`);
-}
-function harmonicMeanScore(a, b) {
-	return a + b === 0 ? 0 : 2 * a * b / (a + b);
-}
-function matchFindingsToIssues(issues, findings) {
-	if (issues.length === 0) return /* @__PURE__ */ new Map();
-	const cardinalityWeight = issues.length + 1;
-	const scores = issues.map((issue) => [...findings.map((finding) => {
-		if (!findingMatchesIssue(finding, issue)) return -1;
-		const criticalHit = (issue.criticalEvidence?.length ?? 0) > 0 && matchesEvidence(finding.evidence_refs, issue.criticalEvidence ?? [], "any");
-		return cardinalityWeight + Number(criticalHit);
-	}), ...Array.from({ length: issues.length }, () => 0)]);
-	const assignment = linearSumAssignment(scores, { maximaze: true }).rowAssignments;
-	const matches = /* @__PURE__ */ new Map();
-	for (const [issueIndex, column] of assignment.entries()) {
-		if (column < 0 || column >= findings.length) continue;
-		if (!findingMatchesIssue(findings[column], issues[issueIndex])) continue;
-		matches.set(issueIndex, column);
-	}
-	return matches;
-}
-function findingMatchesIssue(finding, issue) {
-	if (issue.findingIds && !issue.findingIds.includes(finding.finding_id)) return false;
-	if (issue.areas && !issue.areas.includes(finding.area)) return false;
-	if (issue.subjects && (!finding.subject || !issue.subjects.includes(finding.subject))) return false;
-	if (issue.evidence && !matchesEvidence(finding.evidence_refs, issue.evidence, issue.evidenceMode ?? "any")) return false;
-	return true;
-}
-function matchesEvidence(actual, expected, mode) {
-	if (expected.length === 0) return true;
-	const match = (target) => actual.some((ref) => evidenceMatches(ref, target));
-	return mode === "all" ? expected.every(match) : expected.some(match);
-}
-function evidenceMatches(actual, expected) {
-	return actual.uri === expected.uri && (expected.kind === void 0 || actual.kind === expected.kind);
-}
-//#endregion
-//#region src/analyst/benchmark-summary.ts
-function summarizeAnalystBenchmarkRunner(runnerId, observations) {
-	const issueBearing = observations.filter((observation) => observation.labelState === "positive");
-	const completed = observations.filter((observation) => !observation.error);
-	const expectedIssues = issueBearing.reduce((sum, observation) => sum + observation.score.expectedIssueCount, 0);
-	const matchedIssues = issueBearing.reduce((sum, observation) => sum + observation.score.matchedIssueIds.length, 0);
-	const issueFindings = issueBearing.reduce((sum, observation) => sum + (observation.error ? 0 : observation.findings.length), 0);
-	const supportedFindings = issueBearing.reduce((sum, observation) => sum + observation.score.supportedFindingIndexes.length, 0);
-	const issueRecall = expectedIssues === 0 ? null : matchedIssues / expectedIssues;
-	const findingPrecision = expectedIssues === 0 ? null : issueFindings === 0 ? 0 : supportedFindings / issueFindings;
-	const macroIssueRecall = issueBearing.length === 0 ? null : mean(issueBearing.map((observation) => observation.score.issueRecall));
-	const macroFindingPrecision = issueBearing.length === 0 ? null : mean(issueBearing.map((observation) => observation.score.findingPrecision));
-	const macroF1 = issueBearing.length === 0 ? null : mean(issueBearing.map((observation) => observation.score.f1));
-	const critical = observations.map((observation) => observation.score.criticalStepAccuracy).filter((value) => value !== null);
-	const findingsWithEvidence = completed.reduce((sum, observation) => sum + observation.findings.filter((finding) => finding.evidence_refs.length > 0).length, 0);
-	const allFindings = completed.reduce((sum, observation) => sum + observation.findings.length, 0);
-	const allCitations = completed.flatMap((observation) => observation.findings.flatMap((finding) => finding.evidence_refs));
-	const citationObservations = completed.filter((observation) => observation.score.citationLabelAgreement !== null);
-	const citationCount = citationObservations.reduce((sum, observation) => sum + observation.findings.reduce((count, finding) => count + finding.evidence_refs.length, 0), 0);
-	const invalidCitationCount = citationObservations.reduce((sum, observation) => sum + observation.score.unlabeledEvidence.length, 0);
-	const trustedNegative = observations.filter((observation) => observation.labelState === "trusted-negative");
-	const completedTrustedNegative = trustedNegative.filter((observation) => !observation.error);
-	const unlabeled = observations.filter((observation) => observation.labelState === "unlabeled");
-	const completedUnlabeled = unlabeled.filter((observation) => !observation.error);
-	const resolvedCitations = completed.reduce((sum, observation) => sum + (observation.evidenceResolution?.resolved ?? 0), 0);
-	const unresolvedCitations = completed.reduce((sum, observation) => sum + (observation.evidenceResolution?.unresolvedEvidence.length ?? 0), 0);
-	const citationResolutionErrors = completed.reduce((sum, observation) => sum + (observation.evidenceResolution?.errors.length ?? 0), 0);
-	const resolutionAttempts = completed.filter((observation) => observation.findings.some((finding) => finding.evidence_refs.length > 0));
-	const citationResolutionUnknownRuns = resolutionAttempts.filter((observation) => !observation.evidenceResolution || observation.evidenceResolution.errors.length > 0).length;
-	const usages = observations.map((observation) => observation.usage);
-	const knownCostUsd = stableSum(usages.map((usage) => {
-		if (!usage) return 0;
-		return usage.cost.kind === "uncaptured" ? usage.knownCostUsd ?? 0 : usage.cost.usd;
-	}));
-	const predictionAgreement = repeatedObservationAgreement(observations, predictionSignature);
-	const matchedLabelAgreement = repeatedObservationAgreement(observations.filter((observation) => observation.labelState === "positive"), matchedLabelSignature);
-	return {
-		runnerId,
-		plannedRuns: observations.length,
-		completedRuns: observations.filter((observation) => !observation.error).length,
-		failedRuns: observations.filter((observation) => Boolean(observation.error)).length,
-		issueBearingRuns: issueBearing.length,
-		trustedNegativeRuns: trustedNegative.length,
-		unlabeledRuns: unlabeled.length,
-		issueRecall,
-		findingPrecision,
-		f1: findingPrecision === null || issueRecall === null ? null : harmonicMeanScore(findingPrecision, issueRecall),
-		macroIssueRecall,
-		macroFindingPrecision,
-		macroF1,
-		criticalStepAccuracy: critical.length === 0 ? null : mean(critical),
-		citationCoverage: allFindings === 0 ? null : findingsWithEvidence / allFindings,
-		citationExcerptCoverage: allCitations.length === 0 ? null : allCitations.filter((evidence) => Boolean(evidence.excerpt?.trim())).length / allCitations.length,
-		citationLabelAgreement: citationObservations.length === 0 ? null : citationCount === 0 ? 0 : (citationCount - invalidCitationCount) / citationCount,
-		citationResolution: resolutionAttempts.length === 0 || citationResolutionUnknownRuns > 0 || resolvedCitations + unresolvedCitations === 0 ? null : resolvedCitations / (resolvedCitations + unresolvedCitations),
-		citationResolutionUnknownRuns,
-		unresolvedCitations,
-		citationResolutionErrors,
-		trustedNegativeFalsePositiveRate: completedTrustedNegative.length === 0 ? null : completedTrustedNegative.filter((observation) => observation.score.predictionOnLabelEmptyCase).length / completedTrustedNegative.length,
-		trustedNegativeFailureRate: trustedNegative.length === 0 ? null : trustedNegative.filter((observation) => Boolean(observation.error)).length / trustedNegative.length,
-		unlabeledPredictionRate: completedUnlabeled.length === 0 ? null : completedUnlabeled.filter((observation) => observation.findings.length > 0).length / completedUnlabeled.length,
-		unlabeledFailureRate: unlabeled.length === 0 ? null : unlabeled.filter((observation) => Boolean(observation.error)).length / unlabeled.length,
-		predictionAgreement: predictionAgreement.value,
-		predictionAgreementCases: predictionAgreement.cases,
-		matchedLabelAgreement: matchedLabelAgreement.value,
-		matchedLabelAgreementCases: matchedLabelAgreement.cases,
-		latencyMs: latencyDistribution(observations.map((observation) => observation.latencyMs).filter((value) => value !== null)),
-		benchmarkClockLatencyRuns: observations.filter((observation) => observation.latencySource === "benchmark-clock").length,
-		runnerReportedLatencyRuns: observations.filter((observation) => observation.latencySource === "runner-reported").length,
-		latencyUnknownRuns: observations.filter((observation) => observation.latencySource === "uncaptured").length,
-		calls: usages.reduce((sum, usage) => sum + (usage?.calls ?? 0), 0),
-		callsUnknownRuns: usages.filter((usage) => !usage || usage.calls === null).length,
-		inputTokens: usages.reduce((sum, usage) => sum + (usage?.tokens?.input ?? 0), 0),
-		outputTokens: usages.reduce((sum, usage) => sum + (usage?.tokens?.output ?? 0), 0),
-		reasoningTokens: usages.reduce((sum, usage) => sum + (usage?.tokens?.reasoning ?? 0), 0),
-		cachedTokens: usages.reduce((sum, usage) => sum + (usage?.tokens?.cached ?? 0), 0),
-		cacheWriteTokens: usages.reduce((sum, usage) => sum + (usage?.tokens?.cacheWrite ?? 0), 0),
-		tokenUsageUnknownRuns: usages.filter((usage) => !usage?.tokens).length,
-		reasoningTokenUsageUnknownRuns: usages.filter((usage) => usage?.tokens?.reasoning === void 0).length,
-		cachedTokenUsageUnknownRuns: usages.filter((usage) => usage?.tokens?.cached === void 0).length,
-		cacheWriteTokenUsageUnknownRuns: usages.filter((usage) => usage?.tokens?.cacheWrite === void 0).length,
-		knownCostUsd,
-		costUnknownRuns: usages.filter((usage) => !usage || usage.cost.kind === "uncaptured").length
-	};
-}
-function mean(values) {
-	return values.length === 0 ? 0 : stableSum(values) / values.length;
-}
-function stableSum(values) {
-	const ordered = [...values].sort((left, right) => Math.abs(left) - Math.abs(right) || left - right);
-	let sum = 0;
-	let correction = 0;
-	for (const value of ordered) {
-		const next = sum + value;
-		correction += Math.abs(sum) >= Math.abs(value) ? sum - next + value : value - next + sum;
-		sum = next;
-	}
-	return sum + correction;
-}
-function latencyDistribution(values) {
-	if (values.length === 0) return null;
-	const sorted = [...values].sort((a, b) => a - b);
-	return {
-		min: sorted[0],
-		mean: mean(sorted),
-		p50: percentile(sorted, .5),
-		p95: percentile(sorted, .95),
-		max: sorted.at(-1)
-	};
-}
-function percentile(sorted, quantile) {
-	if (sorted.length === 0) return 0;
-	return sorted[Math.ceil(quantile * sorted.length) - 1] ?? sorted.at(-1) ?? 0;
-}
-function repeatedObservationAgreement(observations, signature) {
-	const byCase = /* @__PURE__ */ new Map();
-	for (const observation of observations) {
-		const rows = byCase.get(observation.caseId) ?? [];
-		rows.push(observation);
-		byCase.set(observation.caseId, rows);
-	}
-	const caseAgreements = [];
-	for (const rows of byCase.values()) {
-		const agreements = [];
-		for (let left = 0; left < rows.length; left++) for (let right = left + 1; right < rows.length; right++) agreements.push(jaccard(signature(rows[left]), signature(rows[right])));
-		if (agreements.length > 0) caseAgreements.push(mean(agreements));
-	}
-	return {
-		value: caseAgreements.length === 0 ? null : mean(caseAgreements),
-		cases: caseAgreements.length
-	};
-}
-function matchedLabelSignature(observation) {
-	if (observation.error) return [`error:${observation.error.class}`];
-	return observation.score.matchedIssueIds;
-}
-function predictionSignature(observation) {
-	if (observation.error) return [`error:${observation.error.class}`];
-	return observation.findings.map((finding) => JSON.stringify([finding.finding_id, finding.evidence_refs.map((evidence) => [
-		evidence.kind,
-		evidence.uri,
-		evidence.excerpt ?? null
-	]).sort((left, right) => compareCodeUnits(JSON.stringify(left), JSON.stringify(right)))])).sort();
-}
-function jaccard(left, right) {
-	const a = new Set(left);
-	const b = new Set(right);
-	const union = /* @__PURE__ */ new Set([...a, ...b]);
-	if (union.size === 0) return 1;
-	let intersection = 0;
-	for (const value of a) if (b.has(value)) intersection += 1;
-	return intersection / union.size;
-}
-//#endregion
-//#region src/analyst/benchmark.ts
-/**
-* Resolve canonical `trace://<trace>/span/<span>` evidence against a trace store.
-* Other evidence kinds and URI schemes require a caller-supplied resolver.
-*/
-function traceStoreEvidenceResolver(getStore) {
-	return async ({ caseInput, evidence, signal }) => {
-		if (evidence.kind !== "span") return false;
-		const location = parseTraceSpanUri(evidence.uri);
-		if (!location) return false;
-		const result = await getStore(caseInput).viewSpans({
-			trace_id: location.traceId,
-			span_ids: [location.spanId]
-		}, signal ? { signal } : void 0);
-		return result.trace_id === location.traceId && result.missing_span_ids.length === 0 && result.spans.some((span) => span.span_id === location.spanId);
-	};
-}
-async function runAnalystBenchmark(options) {
-	validateBenchmarkOptions(options);
-	const startedAt = (/* @__PURE__ */ new Date()).toISOString();
-	const repetitions = options.repetitions ?? 1;
-	const runnerOrderSeed = options.runnerOrderSeed ?? 0;
-	const allJobs = benchmarkJobs(options.cases, options.runners, repetitions, runnerOrderSeed);
-	const maxConcurrency = Math.min(options.maxConcurrency ?? 1, allJobs.length);
-	const initialObservations = validateInitialObservations(options.initialObservations ?? [], allJobs);
-	const completed = new Set(initialObservations.map(observationKey));
-	const jobs = allJobs.filter((job) => !completed.has(jobKey(job)));
-	const observations = [...initialObservations];
-	let cursor = 0;
-	const worker = async () => {
-		while (cursor < jobs.length) {
-			options.signal?.throwIfAborted();
-			const job = jobs[cursor++];
-			const observation = await runBenchmarkJob(job, options.signal, options.resolveEvidence);
-			options.signal?.throwIfAborted();
-			await options.onObservation?.(observation);
-			options.signal?.throwIfAborted();
-			observations.push(observation);
-		}
-	};
-	await Promise.all(Array.from({ length: Math.min(maxConcurrency, jobs.length) }, worker));
-	options.signal?.throwIfAborted();
-	const runnerOrder = new Map(options.runners.map((runner, index) => [runner.id, index]));
-	const caseOrder = new Map(options.cases.map((testCase, index) => [testCase.id, index]));
-	observations.sort((a, b) => (runnerOrder.get(a.runnerId) ?? 0) - (runnerOrder.get(b.runnerId) ?? 0) || (caseOrder.get(a.caseId) ?? 0) - (caseOrder.get(b.caseId) ?? 0) || a.repetition - b.repetition);
-	return {
-		provenance: {
-			...options.benchmark,
-			startedAt,
-			endedAt: (/* @__PURE__ */ new Date()).toISOString(),
-			caseCount: options.cases.length,
-			runnerIds: options.runners.map((runner) => runner.id),
-			repetitions,
-			maxConcurrency,
-			runnerOrderSeed
-		},
-		observations,
-		summaries: options.runners.map((runner) => summarizeAnalystBenchmarkRunner(runner.id, observations.filter((observation) => observation.runnerId === runner.id)))
-	};
-}
-function registryBenchmarkRunner(options) {
-	return {
-		id: options.id,
-		async analyze(input, context) {
-			const result = await options.registry.run(`${options.id}:${context.caseId}:${context.repetition}`, input, {
-				...options.runOptions,
-				signal: context.signal
-			});
-			return {
-				findings: result.findings,
-				usage: mergeRegistryUsage(result),
-				metadata: { analystRun: result },
-				...options.failOnAnalystFailure ? { error: registryRunFailure(result) } : {}
-			};
-		}
-	};
-}
-async function runBenchmarkJob(job, signal, resolveEvidence) {
-	const started = performance.now();
-	try {
-		const output = await job.runner.analyze(job.testCase.input, {
-			caseId: job.testCase.id,
-			repetition: job.repetition,
-			signal
-		});
-		if (output.usage) assertValidAnalystUsageReceipt(output.usage, "analyst benchmark usage");
-		const benchmarkLatencyMs = performance.now() - started;
-		const latency = resolveBenchmarkLatency(output.observedLatencyMs, benchmarkLatencyMs);
-		const scoredFindings = output.error ? [] : output.findings;
-		return {
-			runnerId: job.runner.id,
-			caseId: job.testCase.id,
-			clusterId: job.testCase.clusterId,
-			labelState: job.testCase.labelState,
-			repetition: job.repetition,
-			executionIndex: job.executionIndex,
-			latencyMs: latency.value,
-			latencySource: latency.source,
-			findings: output.findings,
-			score: scoreAnalystFindings(job.testCase, scoredFindings),
-			evidenceResolution: resolveEvidence ? await resolveFindingEvidence(job.testCase, output.findings, resolveEvidence, signal) : void 0,
-			caseTags: [...job.testCase.tags ?? []],
-			caseMetadata: job.testCase.metadata,
-			usage: output.usage,
-			runnerMetadata: output.metadata,
-			...output.error ? { error: output.error } : {}
-		};
-	} catch (error) {
-		if (signal?.aborted) throw error;
-		const findings = [];
-		return {
-			runnerId: job.runner.id,
-			caseId: job.testCase.id,
-			clusterId: job.testCase.clusterId,
-			labelState: job.testCase.labelState,
-			repetition: job.repetition,
-			executionIndex: job.executionIndex,
-			latencyMs: performance.now() - started,
-			latencySource: "benchmark-clock",
-			findings,
-			score: scoreAnalystFindings(job.testCase, findings),
-			caseTags: [...job.testCase.tags ?? []],
-			caseMetadata: job.testCase.metadata,
-			error: {
-				class: error instanceof Error ? error.constructor.name : "Error",
-				message: error instanceof Error ? error.message : String(error)
-			}
-		};
-	}
-}
-function resolveBenchmarkLatency(observedLatencyMs, fallbackMs) {
-	if (observedLatencyMs === void 0) return {
-		value: fallbackMs,
-		source: "benchmark-clock"
-	};
-	if (observedLatencyMs === null) return {
-		value: null,
-		source: "uncaptured"
-	};
-	if (!Number.isFinite(observedLatencyMs) || observedLatencyMs < 0) throw new RangeError("analyst benchmark observedLatencyMs must be finite and non-negative");
-	return {
-		value: observedLatencyMs,
-		source: "runner-reported"
-	};
-}
-function registryRunFailure(result) {
-	const failed = result.per_analyst.filter((summary) => summary.status === "failed");
-	if (failed.length === 0) return void 0;
-	return {
-		class: "AnalystRunFailure",
-		message: failed.map((summary) => `${summary.analyst_id}: ${summary.error?.class ?? "Error"}: ${summary.error?.message ?? "analyst failed"}`).join("; ")
-	};
-}
-function benchmarkJobs(cases, runners, repetitions, runnerOrderSeed) {
-	const seededRunners = [...runners].sort((left, right) => stableHash(`${runnerOrderSeed}\u0000${left.id}`) - stableHash(`${runnerOrderSeed}\u0000${right.id}`) || left.id.localeCompare(right.id));
-	let executionIndex = 0;
-	return cases.flatMap((testCase, caseIndex) => Array.from({ length: repetitions }, (_, repetition) => {
-		const rotation = (caseIndex * repetitions + repetition) % seededRunners.length;
-		return [...seededRunners.slice(rotation), ...seededRunners.slice(0, rotation)].map((runner) => ({
-			runner,
-			testCase,
-			repetition,
-			executionIndex: executionIndex++
-		}));
-	}).flat());
-}
-function validateInitialObservations(observations, jobs) {
-	const expected = new Map(jobs.map((job) => [jobKey(job), job]));
-	const seen = /* @__PURE__ */ new Set();
-	return observations.map((observation) => {
-		const key = observationKey(observation);
-		if (seen.has(key)) throw new TypeError(`duplicate initial analyst benchmark observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}'`);
-		seen.add(key);
-		const job = expected.get(key);
-		if (!job) throw new TypeError(`initial analyst benchmark observation does not match a planned job: '${observation.runnerId}/${observation.caseId}/${observation.repetition}'`);
-		if (observation.executionIndex !== job.executionIndex) throw new TypeError(`initial analyst benchmark observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}' has executionIndex ${observation.executionIndex}; expected ${job.executionIndex}`);
-		if (observation.clusterId !== job.testCase.clusterId || observation.labelState !== job.testCase.labelState) throw new TypeError(`initial analyst benchmark observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}' does not match the current case labels`);
-		if (JSON.stringify(observation.caseTags) !== JSON.stringify(job.testCase.tags ?? []) || JSON.stringify(observation.caseMetadata) !== JSON.stringify(job.testCase.metadata)) throw new TypeError(`initial analyst benchmark observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}' does not match the current case metadata`);
-		const expectedScore = scoreAnalystFindings(job.testCase, observation.error ? [] : observation.findings);
-		if (JSON.stringify(observation.score) !== JSON.stringify(expectedScore)) throw new TypeError(`initial analyst benchmark observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}' has stale or invalid scores`);
-		if (observation.usage) assertValidAnalystUsageReceipt(observation.usage, `initial analyst benchmark observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}' usage`);
-		if (![
-			"benchmark-clock",
-			"runner-reported",
-			"uncaptured"
-		].includes(observation.latencySource) || observation.latencySource === "uncaptured" && observation.latencyMs !== null || observation.latencySource !== "uncaptured" && (observation.latencyMs === null || !Number.isFinite(observation.latencyMs) || observation.latencyMs < 0)) throw new TypeError(`initial analyst benchmark observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}' has invalid latency`);
-		return { ...observation };
-	});
-}
-function jobKey(job) {
-	return `${job.runner.id}\u0000${job.testCase.id}\u0000${job.repetition}`;
-}
-function observationKey(observation) {
-	return `${observation.runnerId}\u0000${observation.caseId}\u0000${observation.repetition}`;
-}
-async function resolveFindingEvidence(testCase, findings, resolver, signal) {
-	const evidence = findings.flatMap((finding) => finding.evidence_refs);
-	const resolved = [];
-	const unresolvedEvidence = [];
-	const errors = [];
-	for (const ref of evidence) {
-		signal?.throwIfAborted();
-		try {
-			if (await resolver({
-				caseId: testCase.id,
-				caseInput: testCase.input,
-				evidence: ref,
-				signal
-			})) resolved.push(ref);
-			else unresolvedEvidence.push(ref);
-		} catch (error) {
-			if (signal?.aborted) throw error;
-			errors.push({
-				evidence: ref,
-				class: error instanceof Error ? error.constructor.name : "Error",
-				message: error instanceof Error ? error.message : String(error)
-			});
-		}
-	}
-	return {
-		checked: evidence.length,
-		resolved: resolved.length,
-		unresolvedEvidence,
-		errors,
-		validity: evidence.length === 0 || errors.length > 0 ? null : resolved.length / evidence.length
-	};
-}
-function parseTraceSpanUri(uri) {
-	const match = /^trace:\/\/([^/]+)\/span\/([^/]+)$/.exec(uri);
-	if (!match) return null;
-	try {
-		const traceId = decodeURIComponent(match[1]);
-		const spanId = decodeURIComponent(match[2]);
-		return traceId && spanId ? {
-			traceId,
-			spanId
-		} : null;
-	} catch {
-		return null;
-	}
-}
-function validateBenchmarkOptions(options) {
-	if (options.cases.length === 0) throw new TypeError("runAnalystBenchmark requires cases");
-	if (options.runners.length === 0) throw new TypeError("runAnalystBenchmark requires runners");
-	const repetitions = options.repetitions ?? 1;
-	const maxConcurrency = options.maxConcurrency ?? 1;
-	if (!Number.isSafeInteger(repetitions) || repetitions < 1) throw new RangeError("runAnalystBenchmark repetitions must be a positive safe integer");
-	if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1) throw new RangeError("runAnalystBenchmark maxConcurrency must be a positive safe integer");
-	if (!Number.isSafeInteger(options.runnerOrderSeed ?? 0)) throw new RangeError("runAnalystBenchmark runnerOrderSeed must be a safe integer");
-	assertUniqueNonEmpty(options.cases.map((testCase) => testCase.id), "case");
-	assertUniqueNonEmpty(options.runners.map((runner) => runner.id), "runner");
-	for (const testCase of options.cases) validateBenchmarkCase(testCase);
-}
-function validateBenchmarkCase(testCase) {
-	assertValidAnalystScoringCase(testCase);
-	if (!testCase.clusterId.trim()) throw new TypeError(`${testCase.id}: analyst benchmark clusterId must not be empty`);
-	if (testCase.labelState !== "positive" && testCase.labelState !== "trusted-negative" && testCase.labelState !== "unlabeled") throw new TypeError(`${testCase.id}: analyst benchmark labelState is invalid`);
-	if (testCase.labelState === "positive" && testCase.expectedIssues.length === 0) throw new TypeError(`${testCase.id}: positive case requires at least one expected issue`);
-	if (testCase.labelState !== "positive" && testCase.expectedIssues.length > 0) throw new TypeError(`${testCase.id}: ${testCase.labelState} case cannot contain expected issues`);
-}
-function assertUniqueNonEmpty(values, label) {
-	const seen = /* @__PURE__ */ new Set();
-	for (const value of values) {
-		if (!value.trim()) throw new TypeError(`analyst benchmark ${label} id must not be empty`);
-		if (seen.has(value)) throw new TypeError(`duplicate analyst benchmark ${label} id '${value}'`);
-		seen.add(value);
-	}
-}
-/** 32-bit FNV-1a over raw UTF-16 code units, left SIGNED so subtracting two
-*  results is a valid comparator.
-*
-*  Frozen: it fixes the runner order a benchmark reports. Adding the `>>> 0`
-*  the other variants carry would change that order, which is why this one
-*  does not have it. */
-function stableHash(value) {
-	let hash = 2166136261;
-	for (let index = 0; index < value.length; index += 1) {
-		hash ^= value.charCodeAt(index);
-		hash = Math.imul(hash, 16777619);
-	}
-	return hash >>> 0;
-}
-function mergeRegistryUsage(result) {
-	const usages = result.per_analyst.map((summary) => summary.usage);
-	const calls = usages.every((usage) => usage.calls !== null) ? usages.reduce((sum, usage) => sum + (usage.calls ?? 0), 0) : null;
-	const tokens = usages.every((usage) => usage.tokens !== null) ? usages.reduce((sum, usage) => ({
-		input: sum.input + (usage.tokens?.input ?? 0),
-		output: sum.output + (usage.tokens?.output ?? 0),
-		reasoning: sum.reasoning + (usage.tokens?.reasoning ?? 0),
-		cached: sum.cached + (usage.tokens?.cached ?? 0),
-		cacheWrite: sum.cacheWrite + (usage.tokens?.cacheWrite ?? 0)
-	}), {
-		input: 0,
-		output: 0,
-		reasoning: 0,
-		cached: 0,
-		cacheWrite: 0
-	}) : null;
-	const knownCostUsd = usages.reduce((sum, usage) => sum + (usage.cost.kind === "uncaptured" ? usage.knownCostUsd ?? 0 : usage.cost.usd), 0);
-	const cost = usages.some((usage) => usage.cost.kind === "uncaptured") ? {
-		kind: "uncaptured",
-		usd: null
-	} : usages.some((usage) => usage.cost.kind === "estimated") ? {
-		kind: "estimated",
-		usd: knownCostUsd
-	} : {
-		kind: "observed",
-		usd: knownCostUsd
-	};
-	return {
-		calls,
-		tokens,
-		cost,
-		...cost.kind === "uncaptured" ? { knownCostUsd } : {}
-	};
-}
-//#endregion
-export { scoreAnalystFindings as a, summarizeAnalystBenchmarkRunner as i, runAnalystBenchmark as n, traceStoreEvidenceResolver as r, registryBenchmarkRunner as t };
-
-//# sourceMappingURL=benchmark-BywPeAUo.js.map