@tangle-network/agent-eval 0.136.0 → 0.138.0

package/dist/benchmark-D8dkki-J.js

@@ -0,0 +1,554 @@
+import { l as assertValidAnalystUsageReceipt } from "./single-run-lock-D_bS5xhj.js";
+import { performance } from "node:perf_hooks";
+import { linearSumAssignment } from "linear-sum-assignment";
+//#region src/analyst/benchmark-scoring.ts
+function scoreAnalystFindings(testCase, findings) {
+	assertValidAnalystScoringCase(testCase);
+	const matchedFindingByIssue = matchFindingsToIssues(testCase.expectedIssues, findings);
+	const matchedIssueIds = testCase.expectedIssues.filter((_, index) => matchedFindingByIssue.has(index)).map((issue) => issue.id);
+	const missedIssueIds = testCase.expectedIssues.filter((_, index) => !matchedFindingByIssue.has(index)).map((issue) => issue.id);
+	const supportedFindingIndexes = new Set(matchedFindingByIssue.values());
+	const unsupportedFindingIndexes = findings.map((_, index) => index).filter((index) => !supportedFindingIndexes.has(index));
+	const expectedIssueCount = testCase.expectedIssues.length;
+	const issueRecall = expectedIssueCount === 0 ? 1 : matchedIssueIds.length / expectedIssueCount;
+	const findingPrecision = findings.length === 0 ? expectedIssueCount === 0 ? 1 : 0 : supportedFindingIndexes.size / findings.length;
+	const f1 = harmonicMeanScore(findingPrecision, issueRecall);
+	const allEvidence = findings.flatMap((finding) => finding.evidence_refs);
+	const criticalIssues = testCase.expectedIssues.filter((issue) => (issue.criticalEvidence?.length ?? 0) > 0);
+	const criticalHits = testCase.expectedIssues.filter((issue) => {
+		if ((issue.criticalEvidence?.length ?? 0) === 0) return false;
+		return matchesEvidence(allEvidence, issue.criticalEvidence ?? [], "any");
+	}).length;
+	const findingsWithEvidence = findings.filter((finding) => finding.evidence_refs.length > 0).length;
+	const unlabeledEvidence = testCase.labeledEvidence ? allEvidence.filter((ref) => !testCase.labeledEvidence.some((expected) => evidenceMatches(ref, expected))) : [];
+	return {
+		expectedIssueCount,
+		matchedIssueIds,
+		missedIssueIds,
+		supportedFindingIndexes: [...supportedFindingIndexes].sort((a, b) => a - b),
+		unsupportedFindingIndexes,
+		unlabeledEvidence,
+		issueRecall,
+		findingPrecision,
+		f1,
+		criticalStepAccuracy: criticalIssues.length === 0 ? null : criticalHits / criticalIssues.length,
+		citationCoverage: findings.length === 0 ? null : findingsWithEvidence / findings.length,
+		citationExcerptCoverage: allEvidence.length === 0 ? null : allEvidence.filter((evidence) => Boolean(evidence.excerpt?.trim())).length / allEvidence.length,
+		citationLabelAgreement: testCase.labeledEvidence === void 0 ? null : allEvidence.length === 0 ? findings.length === 0 ? null : 0 : (allEvidence.length - unlabeledEvidence.length) / allEvidence.length,
+		predictionOnLabelEmptyCase: expectedIssueCount === 0 && findings.length > 0
+	};
+}
+function assertValidAnalystScoringCase(testCase) {
+	if (!testCase.id.trim()) throw new TypeError("analyst benchmark case id must not be empty");
+	const ids = /* @__PURE__ */ new Set();
+	for (const issue of testCase.expectedIssues) {
+		if (!issue.id.trim()) throw new TypeError(`${testCase.id}: expected issue id must not be empty`);
+		if (ids.has(issue.id)) throw new TypeError(`${testCase.id}: duplicate expected issue id '${issue.id}'`);
+		ids.add(issue.id);
+		if (!issue.findingIds?.length && !issue.areas?.length && !issue.subjects?.length && !issue.evidence?.length) throw new TypeError(`${testCase.id}/${issue.id}: expected issue must identify a finding by id, area, subject, or evidence`);
+	}
+	for (const ref of testCase.labeledEvidence ?? []) if (!ref.uri.trim()) throw new TypeError(`${testCase.id}: labeled evidence URI must not be empty`);
+}
+function harmonicMeanScore(a, b) {
+	return a + b === 0 ? 0 : 2 * a * b / (a + b);
+}
+function matchFindingsToIssues(issues, findings) {
+	if (issues.length === 0) return /* @__PURE__ */ new Map();
+	const cardinalityWeight = issues.length + 1;
+	const assignment = linearSumAssignment(issues.map((issue) => [...findings.map((finding) => {
+		if (!findingMatchesIssue(finding, issue)) return -1;
+		const criticalHit = (issue.criticalEvidence?.length ?? 0) > 0 && matchesEvidence(finding.evidence_refs, issue.criticalEvidence ?? [], "any");
+		return cardinalityWeight + Number(criticalHit);
+	}), ...Array.from({ length: issues.length }, () => 0)]), { maximaze: true }).rowAssignments;
+	const matches = /* @__PURE__ */ new Map();
+	for (const [issueIndex, column] of assignment.entries()) {
+		if (column < 0 || column >= findings.length) continue;
+		if (!findingMatchesIssue(findings[column], issues[issueIndex])) continue;
+		matches.set(issueIndex, column);
+	}
+	return matches;
+}
+function findingMatchesIssue(finding, issue) {
+	if (issue.findingIds && !issue.findingIds.includes(finding.finding_id)) return false;
+	if (issue.areas && !issue.areas.includes(finding.area)) return false;
+	if (issue.subjects && (!finding.subject || !issue.subjects.includes(finding.subject))) return false;
+	if (issue.evidence && !matchesEvidence(finding.evidence_refs, issue.evidence, issue.evidenceMode ?? "any")) return false;
+	return true;
+}
+function matchesEvidence(actual, expected, mode) {
+	if (expected.length === 0) return true;
+	const match = (target) => actual.some((ref) => evidenceMatches(ref, target));
+	return mode === "all" ? expected.every(match) : expected.some(match);
+}
+function evidenceMatches(actual, expected) {
+	return actual.uri === expected.uri && (expected.kind === void 0 || actual.kind === expected.kind);
+}
+//#endregion
+//#region src/analyst/benchmark-summary.ts
+function summarizeAnalystBenchmarkRunner(runnerId, observations) {
+	const issueBearing = observations.filter((observation) => observation.labelState === "positive");
+	const completed = observations.filter((observation) => !observation.error);
+	const expectedIssues = issueBearing.reduce((sum, observation) => sum + observation.score.expectedIssueCount, 0);
+	const matchedIssues = issueBearing.reduce((sum, observation) => sum + observation.score.matchedIssueIds.length, 0);
+	const issueFindings = issueBearing.reduce((sum, observation) => sum + (observation.error ? 0 : observation.findings.length), 0);
+	const supportedFindings = issueBearing.reduce((sum, observation) => sum + observation.score.supportedFindingIndexes.length, 0);
+	const issueRecall = expectedIssues === 0 ? null : matchedIssues / expectedIssues;
+	const findingPrecision = expectedIssues === 0 ? null : issueFindings === 0 ? 0 : supportedFindings / issueFindings;
+	const macroIssueRecall = issueBearing.length === 0 ? null : mean(issueBearing.map((observation) => observation.score.issueRecall));
+	const macroFindingPrecision = issueBearing.length === 0 ? null : mean(issueBearing.map((observation) => observation.score.findingPrecision));
+	const macroF1 = issueBearing.length === 0 ? null : mean(issueBearing.map((observation) => observation.score.f1));
+	const critical = observations.map((observation) => observation.score.criticalStepAccuracy).filter((value) => value !== null);
+	const findingsWithEvidence = completed.reduce((sum, observation) => sum + observation.findings.filter((finding) => finding.evidence_refs.length > 0).length, 0);
+	const allFindings = completed.reduce((sum, observation) => sum + observation.findings.length, 0);
+	const allCitations = completed.flatMap((observation) => observation.findings.flatMap((finding) => finding.evidence_refs));
+	const citationObservations = completed.filter((observation) => observation.score.citationLabelAgreement !== null);
+	const citationCount = citationObservations.reduce((sum, observation) => sum + observation.findings.reduce((count, finding) => count + finding.evidence_refs.length, 0), 0);
+	const invalidCitationCount = citationObservations.reduce((sum, observation) => sum + observation.score.unlabeledEvidence.length, 0);
+	const trustedNegative = observations.filter((observation) => observation.labelState === "trusted-negative");
+	const completedTrustedNegative = trustedNegative.filter((observation) => !observation.error);
+	const unlabeled = observations.filter((observation) => observation.labelState === "unlabeled");
+	const completedUnlabeled = unlabeled.filter((observation) => !observation.error);
+	const resolvedCitations = completed.reduce((sum, observation) => sum + (observation.evidenceResolution?.resolved ?? 0), 0);
+	const unresolvedCitations = completed.reduce((sum, observation) => sum + (observation.evidenceResolution?.unresolvedEvidence.length ?? 0), 0);
+	const citationResolutionErrors = completed.reduce((sum, observation) => sum + (observation.evidenceResolution?.errors.length ?? 0), 0);
+	const resolutionAttempts = completed.filter((observation) => observation.findings.some((finding) => finding.evidence_refs.length > 0));
+	const citationResolutionUnknownRuns = resolutionAttempts.filter((observation) => !observation.evidenceResolution || observation.evidenceResolution.errors.length > 0).length;
+	const usages = observations.map((observation) => observation.usage);
+	const knownCostUsd = stableSum(usages.map((usage) => {
+		if (!usage) return 0;
+		return usage.cost.kind === "uncaptured" ? usage.knownCostUsd ?? 0 : usage.cost.usd;
+	}));
+	const predictionAgreement = repeatedObservationAgreement(observations, predictionSignature);
+	const matchedLabelAgreement = repeatedObservationAgreement(observations.filter((observation) => observation.labelState === "positive"), matchedLabelSignature);
+	return {
+		runnerId,
+		plannedRuns: observations.length,
+		completedRuns: observations.filter((observation) => !observation.error).length,
+		failedRuns: observations.filter((observation) => Boolean(observation.error)).length,
+		issueBearingRuns: issueBearing.length,
+		trustedNegativeRuns: trustedNegative.length,
+		unlabeledRuns: unlabeled.length,
+		issueRecall,
+		findingPrecision,
+		f1: findingPrecision === null || issueRecall === null ? null : harmonicMeanScore(findingPrecision, issueRecall),
+		macroIssueRecall,
+		macroFindingPrecision,
+		macroF1,
+		criticalStepAccuracy: critical.length === 0 ? null : mean(critical),
+		citationCoverage: allFindings === 0 ? null : findingsWithEvidence / allFindings,
+		citationExcerptCoverage: allCitations.length === 0 ? null : allCitations.filter((evidence) => Boolean(evidence.excerpt?.trim())).length / allCitations.length,
+		citationLabelAgreement: citationObservations.length === 0 ? null : citationCount === 0 ? 0 : (citationCount - invalidCitationCount) / citationCount,
+		citationResolution: resolutionAttempts.length === 0 || citationResolutionUnknownRuns > 0 || resolvedCitations + unresolvedCitations === 0 ? null : resolvedCitations / (resolvedCitations + unresolvedCitations),
+		citationResolutionUnknownRuns,
+		unresolvedCitations,
+		citationResolutionErrors,
+		trustedNegativeFalsePositiveRate: completedTrustedNegative.length === 0 ? null : completedTrustedNegative.filter((observation) => observation.score.predictionOnLabelEmptyCase).length / completedTrustedNegative.length,
+		trustedNegativeFailureRate: trustedNegative.length === 0 ? null : trustedNegative.filter((observation) => Boolean(observation.error)).length / trustedNegative.length,
+		unlabeledPredictionRate: completedUnlabeled.length === 0 ? null : completedUnlabeled.filter((observation) => observation.findings.length > 0).length / completedUnlabeled.length,
+		unlabeledFailureRate: unlabeled.length === 0 ? null : unlabeled.filter((observation) => Boolean(observation.error)).length / unlabeled.length,
+		predictionAgreement: predictionAgreement.value,
+		predictionAgreementCases: predictionAgreement.cases,
+		matchedLabelAgreement: matchedLabelAgreement.value,
+		matchedLabelAgreementCases: matchedLabelAgreement.cases,
+		latencyMs: latencyDistribution(observations.map((observation) => observation.latencyMs).filter((value) => value !== null)),
+		benchmarkClockLatencyRuns: observations.filter((observation) => observation.latencySource === "benchmark-clock").length,
+		runnerReportedLatencyRuns: observations.filter((observation) => observation.latencySource === "runner-reported").length,
+		latencyUnknownRuns: observations.filter((observation) => observation.latencySource === "uncaptured").length,
+		calls: usages.reduce((sum, usage) => sum + (usage?.calls ?? 0), 0),
+		callsUnknownRuns: usages.filter((usage) => !usage || usage.calls === null).length,
+		inputTokens: usages.reduce((sum, usage) => sum + (usage?.tokens?.input ?? 0), 0),
+		outputTokens: usages.reduce((sum, usage) => sum + (usage?.tokens?.output ?? 0), 0),
+		reasoningTokens: usages.reduce((sum, usage) => sum + (usage?.tokens?.reasoning ?? 0), 0),
+		cachedTokens: usages.reduce((sum, usage) => sum + (usage?.tokens?.cached ?? 0), 0),
+		cacheWriteTokens: usages.reduce((sum, usage) => sum + (usage?.tokens?.cacheWrite ?? 0), 0),
+		tokenUsageUnknownRuns: usages.filter((usage) => !usage?.tokens).length,
+		reasoningTokenUsageUnknownRuns: usages.filter((usage) => usage?.tokens?.reasoning === void 0).length,
+		cachedTokenUsageUnknownRuns: usages.filter((usage) => usage?.tokens?.cached === void 0).length,
+		cacheWriteTokenUsageUnknownRuns: usages.filter((usage) => usage?.tokens?.cacheWrite === void 0).length,
+		knownCostUsd,
+		costUnknownRuns: usages.filter((usage) => !usage || usage.cost.kind === "uncaptured").length
+	};
+}
+function mean(values) {
+	return values.length === 0 ? 0 : stableSum(values) / values.length;
+}
+function stableSum(values) {
+	const ordered = [...values].sort((left, right) => Math.abs(left) - Math.abs(right) || left - right);
+	let sum = 0;
+	let correction = 0;
+	for (const value of ordered) {
+		const next = sum + value;
+		correction += Math.abs(sum) >= Math.abs(value) ? sum - next + value : value - next + sum;
+		sum = next;
+	}
+	return sum + correction;
+}
+function latencyDistribution(values) {
+	if (values.length === 0) return null;
+	const sorted = [...values].sort((a, b) => a - b);
+	return {
+		min: sorted[0],
+		mean: mean(sorted),
+		p50: percentile(sorted, .5),
+		p95: percentile(sorted, .95),
+		max: sorted.at(-1)
+	};
+}
+function percentile(sorted, quantile) {
+	if (sorted.length === 0) return 0;
+	return sorted[Math.ceil(quantile * sorted.length) - 1] ?? sorted.at(-1) ?? 0;
+}
+function repeatedObservationAgreement(observations, signature) {
+	const byCase = /* @__PURE__ */ new Map();
+	for (const observation of observations) {
+		const rows = byCase.get(observation.caseId) ?? [];
+		rows.push(observation);
+		byCase.set(observation.caseId, rows);
+	}
+	const caseAgreements = [];
+	for (const rows of byCase.values()) {
+		const agreements = [];
+		for (let left = 0; left < rows.length; left++) for (let right = left + 1; right < rows.length; right++) agreements.push(jaccard(signature(rows[left]), signature(rows[right])));
+		if (agreements.length > 0) caseAgreements.push(mean(agreements));
+	}
+	return {
+		value: caseAgreements.length === 0 ? null : mean(caseAgreements),
+		cases: caseAgreements.length
+	};
+}
+function matchedLabelSignature(observation) {
+	if (observation.error) return [`error:${observation.error.class}`];
+	return observation.score.matchedIssueIds;
+}
+function predictionSignature(observation) {
+	if (observation.error) return [`error:${observation.error.class}`];
+	return observation.findings.map((finding) => JSON.stringify([finding.finding_id, finding.evidence_refs.map((evidence) => [
+		evidence.kind,
+		evidence.uri,
+		evidence.excerpt ?? null
+	]).sort((left, right) => JSON.stringify(left).localeCompare(JSON.stringify(right)))])).sort();
+}
+function jaccard(left, right) {
+	const a = new Set(left);
+	const b = new Set(right);
+	const union = /* @__PURE__ */ new Set([...a, ...b]);
+	if (union.size === 0) return 1;
+	let intersection = 0;
+	for (const value of a) if (b.has(value)) intersection += 1;
+	return intersection / union.size;
+}
+//#endregion
+//#region src/analyst/benchmark.ts
+/**
+* Resolve canonical `trace://<trace>/span/<span>` evidence against a trace store.
+* Other evidence kinds and URI schemes require a caller-supplied resolver.
+*/
+function traceStoreEvidenceResolver(getStore) {
+	return async ({ caseInput, evidence, signal }) => {
+		if (evidence.kind !== "span") return false;
+		const location = parseTraceSpanUri(evidence.uri);
+		if (!location) return false;
+		const result = await getStore(caseInput).viewSpans({
+			trace_id: location.traceId,
+			span_ids: [location.spanId]
+		}, signal ? { signal } : void 0);
+		return result.trace_id === location.traceId && result.missing_span_ids.length === 0 && result.spans.some((span) => span.span_id === location.spanId);
+	};
+}
+async function runAnalystBenchmark(options) {
+	validateBenchmarkOptions(options);
+	const startedAt = (/* @__PURE__ */ new Date()).toISOString();
+	const repetitions = options.repetitions ?? 1;
+	const runnerOrderSeed = options.runnerOrderSeed ?? 0;
+	const allJobs = benchmarkJobs(options.cases, options.runners, repetitions, runnerOrderSeed);
+	const maxConcurrency = Math.min(options.maxConcurrency ?? 1, allJobs.length);
+	const initialObservations = validateInitialObservations(options.initialObservations ?? [], allJobs);
+	const completed = new Set(initialObservations.map(observationKey));
+	const jobs = allJobs.filter((job) => !completed.has(jobKey(job)));
+	const observations = [...initialObservations];
+	let cursor = 0;
+	const worker = async () => {
+		while (cursor < jobs.length) {
+			options.signal?.throwIfAborted();
+			const job = jobs[cursor++];
+			const observation = await runBenchmarkJob(job, options.signal, options.resolveEvidence);
+			options.signal?.throwIfAborted();
+			await options.onObservation?.(observation);
+			options.signal?.throwIfAborted();
+			observations.push(observation);
+		}
+	};
+	await Promise.all(Array.from({ length: Math.min(maxConcurrency, jobs.length) }, worker));
+	options.signal?.throwIfAborted();
+	const runnerOrder = new Map(options.runners.map((runner, index) => [runner.id, index]));
+	const caseOrder = new Map(options.cases.map((testCase, index) => [testCase.id, index]));
+	observations.sort((a, b) => (runnerOrder.get(a.runnerId) ?? 0) - (runnerOrder.get(b.runnerId) ?? 0) || (caseOrder.get(a.caseId) ?? 0) - (caseOrder.get(b.caseId) ?? 0) || a.repetition - b.repetition);
+	return {
+		provenance: {
+			...options.benchmark,
+			startedAt,
+			endedAt: (/* @__PURE__ */ new Date()).toISOString(),
+			caseCount: options.cases.length,
+			runnerIds: options.runners.map((runner) => runner.id),
+			repetitions,
+			maxConcurrency,
+			runnerOrderSeed
+		},
+		observations,
+		summaries: options.runners.map((runner) => summarizeAnalystBenchmarkRunner(runner.id, observations.filter((observation) => observation.runnerId === runner.id)))
+	};
+}
+function registryBenchmarkRunner(options) {
+	return {
+		id: options.id,
+		async analyze(input, context) {
+			const result = await options.registry.run(`${options.id}:${context.caseId}:${context.repetition}`, input, {
+				...options.runOptions,
+				signal: context.signal
+			});
+			return {
+				findings: result.findings,
+				usage: mergeRegistryUsage(result),
+				metadata: { analystRun: result },
+				...options.failOnAnalystFailure ? { error: registryRunFailure(result) } : {}
+			};
+		}
+	};
+}
+async function runBenchmarkJob(job, signal, resolveEvidence) {
+	const started = performance.now();
+	try {
+		const output = await job.runner.analyze(job.testCase.input, {
+			caseId: job.testCase.id,
+			repetition: job.repetition,
+			signal
+		});
+		if (output.usage) assertValidAnalystUsageReceipt(output.usage, "analyst benchmark usage");
+		const benchmarkLatencyMs = performance.now() - started;
+		const latency = resolveBenchmarkLatency(output.observedLatencyMs, benchmarkLatencyMs);
+		const scoredFindings = output.error ? [] : output.findings;
+		return {
+			runnerId: job.runner.id,
+			caseId: job.testCase.id,
+			clusterId: job.testCase.clusterId,
+			labelState: job.testCase.labelState,
+			repetition: job.repetition,
+			executionIndex: job.executionIndex,
+			latencyMs: latency.value,
+			latencySource: latency.source,
+			findings: output.findings,
+			score: scoreAnalystFindings(job.testCase, scoredFindings),
+			evidenceResolution: resolveEvidence ? await resolveFindingEvidence(job.testCase, output.findings, resolveEvidence, signal) : void 0,
+			caseTags: [...job.testCase.tags ?? []],
+			caseMetadata: job.testCase.metadata,
+			usage: output.usage,
+			runnerMetadata: output.metadata,
+			...output.error ? { error: output.error } : {}
+		};
+	} catch (error) {
+		if (signal?.aborted) throw error;
+		const findings = [];
+		return {
+			runnerId: job.runner.id,
+			caseId: job.testCase.id,
+			clusterId: job.testCase.clusterId,
+			labelState: job.testCase.labelState,
+			repetition: job.repetition,
+			executionIndex: job.executionIndex,
+			latencyMs: performance.now() - started,
+			latencySource: "benchmark-clock",
+			findings,
+			score: scoreAnalystFindings(job.testCase, findings),
+			caseTags: [...job.testCase.tags ?? []],
+			caseMetadata: job.testCase.metadata,
+			error: {
+				class: error instanceof Error ? error.constructor.name : "Error",
+				message: error instanceof Error ? error.message : String(error)
+			}
+		};
+	}
+}
+function resolveBenchmarkLatency(observedLatencyMs, fallbackMs) {
+	if (observedLatencyMs === void 0) return {
+		value: fallbackMs,
+		source: "benchmark-clock"
+	};
+	if (observedLatencyMs === null) return {
+		value: null,
+		source: "uncaptured"
+	};
+	if (!Number.isFinite(observedLatencyMs) || observedLatencyMs < 0) throw new RangeError("analyst benchmark observedLatencyMs must be finite and non-negative");
+	return {
+		value: observedLatencyMs,
+		source: "runner-reported"
+	};
+}
+function registryRunFailure(result) {
+	const failed = result.per_analyst.filter((summary) => summary.status === "failed");
+	if (failed.length === 0) return void 0;
+	return {
+		class: "AnalystRunFailure",
+		message: failed.map((summary) => `${summary.analyst_id}: ${summary.error?.class ?? "Error"}: ${summary.error?.message ?? "analyst failed"}`).join("; ")
+	};
+}
+function benchmarkJobs(cases, runners, repetitions, runnerOrderSeed) {
+	const seededRunners = [...runners].sort((left, right) => stableHash(`${runnerOrderSeed}\u0000${left.id}`) - stableHash(`${runnerOrderSeed}\u0000${right.id}`) || left.id.localeCompare(right.id));
+	let executionIndex = 0;
+	return cases.flatMap((testCase, caseIndex) => Array.from({ length: repetitions }, (_, repetition) => {
+		const rotation = (caseIndex * repetitions + repetition) % seededRunners.length;
+		return [...seededRunners.slice(rotation), ...seededRunners.slice(0, rotation)].map((runner) => ({
+			runner,
+			testCase,
+			repetition,
+			executionIndex: executionIndex++
+		}));
+	}).flat());
+}
+function validateInitialObservations(observations, jobs) {
+	const expected = new Map(jobs.map((job) => [jobKey(job), job]));
+	const seen = /* @__PURE__ */ new Set();
+	return observations.map((observation) => {
+		const key = observationKey(observation);
+		if (seen.has(key)) throw new TypeError(`duplicate initial analyst benchmark observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}'`);
+		seen.add(key);
+		const job = expected.get(key);
+		if (!job) throw new TypeError(`initial analyst benchmark observation does not match a planned job: '${observation.runnerId}/${observation.caseId}/${observation.repetition}'`);
+		if (observation.executionIndex !== job.executionIndex) throw new TypeError(`initial analyst benchmark observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}' has executionIndex ${observation.executionIndex}; expected ${job.executionIndex}`);
+		if (observation.clusterId !== job.testCase.clusterId || observation.labelState !== job.testCase.labelState) throw new TypeError(`initial analyst benchmark observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}' does not match the current case labels`);
+		if (JSON.stringify(observation.caseTags) !== JSON.stringify(job.testCase.tags ?? []) || JSON.stringify(observation.caseMetadata) !== JSON.stringify(job.testCase.metadata)) throw new TypeError(`initial analyst benchmark observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}' does not match the current case metadata`);
+		const expectedScore = scoreAnalystFindings(job.testCase, observation.error ? [] : observation.findings);
+		if (JSON.stringify(observation.score) !== JSON.stringify(expectedScore)) throw new TypeError(`initial analyst benchmark observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}' has stale or invalid scores`);
+		if (observation.usage) assertValidAnalystUsageReceipt(observation.usage, `initial analyst benchmark observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}' usage`);
+		if (![
+			"benchmark-clock",
+			"runner-reported",
+			"uncaptured"
+		].includes(observation.latencySource) || observation.latencySource === "uncaptured" && observation.latencyMs !== null || observation.latencySource !== "uncaptured" && (observation.latencyMs === null || !Number.isFinite(observation.latencyMs) || observation.latencyMs < 0)) throw new TypeError(`initial analyst benchmark observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}' has invalid latency`);
+		return { ...observation };
+	});
+}
+function jobKey(job) {
+	return `${job.runner.id}\u0000${job.testCase.id}\u0000${job.repetition}`;
+}
+function observationKey(observation) {
+	return `${observation.runnerId}\u0000${observation.caseId}\u0000${observation.repetition}`;
+}
+async function resolveFindingEvidence(testCase, findings, resolver, signal) {
+	const evidence = findings.flatMap((finding) => finding.evidence_refs);
+	const resolved = [];
+	const unresolvedEvidence = [];
+	const errors = [];
+	for (const ref of evidence) {
+		signal?.throwIfAborted();
+		try {
+			if (await resolver({
+				caseId: testCase.id,
+				caseInput: testCase.input,
+				evidence: ref,
+				signal
+			})) resolved.push(ref);
+			else unresolvedEvidence.push(ref);
+		} catch (error) {
+			if (signal?.aborted) throw error;
+			errors.push({
+				evidence: ref,
+				class: error instanceof Error ? error.constructor.name : "Error",
+				message: error instanceof Error ? error.message : String(error)
+			});
+		}
+	}
+	return {
+		checked: evidence.length,
+		resolved: resolved.length,
+		unresolvedEvidence,
+		errors,
+		validity: evidence.length === 0 || errors.length > 0 ? null : resolved.length / evidence.length
+	};
+}
+function parseTraceSpanUri(uri) {
+	const match = /^trace:\/\/([^/]+)\/span\/([^/]+)$/.exec(uri);
+	if (!match) return null;
+	try {
+		const traceId = decodeURIComponent(match[1]);
+		const spanId = decodeURIComponent(match[2]);
+		return traceId && spanId ? {
+			traceId,
+			spanId
+		} : null;
+	} catch {
+		return null;
+	}
+}
+function validateBenchmarkOptions(options) {
+	if (options.cases.length === 0) throw new TypeError("runAnalystBenchmark requires cases");
+	if (options.runners.length === 0) throw new TypeError("runAnalystBenchmark requires runners");
+	const repetitions = options.repetitions ?? 1;
+	const maxConcurrency = options.maxConcurrency ?? 1;
+	if (!Number.isSafeInteger(repetitions) || repetitions < 1) throw new RangeError("runAnalystBenchmark repetitions must be a positive safe integer");
+	if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1) throw new RangeError("runAnalystBenchmark maxConcurrency must be a positive safe integer");
+	if (!Number.isSafeInteger(options.runnerOrderSeed ?? 0)) throw new RangeError("runAnalystBenchmark runnerOrderSeed must be a safe integer");
+	assertUniqueNonEmpty(options.cases.map((testCase) => testCase.id), "case");
+	assertUniqueNonEmpty(options.runners.map((runner) => runner.id), "runner");
+	for (const testCase of options.cases) validateBenchmarkCase(testCase);
+}
+function validateBenchmarkCase(testCase) {
+	assertValidAnalystScoringCase(testCase);
+	if (!testCase.clusterId.trim()) throw new TypeError(`${testCase.id}: analyst benchmark clusterId must not be empty`);
+	if (testCase.labelState !== "positive" && testCase.labelState !== "trusted-negative" && testCase.labelState !== "unlabeled") throw new TypeError(`${testCase.id}: analyst benchmark labelState is invalid`);
+	if (testCase.labelState === "positive" && testCase.expectedIssues.length === 0) throw new TypeError(`${testCase.id}: positive case requires at least one expected issue`);
+	if (testCase.labelState !== "positive" && testCase.expectedIssues.length > 0) throw new TypeError(`${testCase.id}: ${testCase.labelState} case cannot contain expected issues`);
+}
+function assertUniqueNonEmpty(values, label) {
+	const seen = /* @__PURE__ */ new Set();
+	for (const value of values) {
+		if (!value.trim()) throw new TypeError(`analyst benchmark ${label} id must not be empty`);
+		if (seen.has(value)) throw new TypeError(`duplicate analyst benchmark ${label} id '${value}'`);
+		seen.add(value);
+	}
+}
+function stableHash(value) {
+	let hash = 2166136261;
+	for (let index = 0; index < value.length; index += 1) {
+		hash ^= value.charCodeAt(index);
+		hash = Math.imul(hash, 16777619);
+	}
+	return hash >>> 0;
+}
+function mergeRegistryUsage(result) {
+	const usages = result.per_analyst.map((summary) => summary.usage);
+	const calls = usages.every((usage) => usage.calls !== null) ? usages.reduce((sum, usage) => sum + (usage.calls ?? 0), 0) : null;
+	const tokens = usages.every((usage) => usage.tokens !== null) ? usages.reduce((sum, usage) => ({
+		input: sum.input + (usage.tokens?.input ?? 0),
+		output: sum.output + (usage.tokens?.output ?? 0),
+		reasoning: sum.reasoning + (usage.tokens?.reasoning ?? 0),
+		cached: sum.cached + (usage.tokens?.cached ?? 0),
+		cacheWrite: sum.cacheWrite + (usage.tokens?.cacheWrite ?? 0)
+	}), {
+		input: 0,
+		output: 0,
+		reasoning: 0,
+		cached: 0,
+		cacheWrite: 0
+	}) : null;
+	const knownCostUsd = usages.reduce((sum, usage) => sum + (usage.cost.kind === "uncaptured" ? usage.knownCostUsd ?? 0 : usage.cost.usd), 0);
+	const cost = usages.some((usage) => usage.cost.kind === "uncaptured") ? {
+		kind: "uncaptured",
+		usd: null
+	} : usages.some((usage) => usage.cost.kind === "estimated") ? {
+		kind: "estimated",
+		usd: knownCostUsd
+	} : {
+		kind: "observed",
+		usd: knownCostUsd
+	};
+	return {
+		calls,
+		tokens,
+		cost,
+		...cost.kind === "uncaptured" ? { knownCostUsd } : {}
+	};
+}
+//#endregion
+export { scoreAnalystFindings as a, summarizeAnalystBenchmarkRunner as i, runAnalystBenchmark as n, traceStoreEvidenceResolver as r, registryBenchmarkRunner as t };
+
+//# sourceMappingURL=benchmark-D8dkki-J.js.map