@tangle-network/agent-eval 0.137.0 → 0.139.0

package/dist/{benchmark-YDrpumqB.js → benchmark-CYtcIF2V.js}

@@ -1,24 +1,9 @@
+import { t as assertValidAnalystUsageReceipt } from "./usage-receipt-CgxMEBZq.js";
 import { performance } from "node:perf_hooks";
 import { linearSumAssignment } from "linear-sum-assignment";
-//#region src/analyst/benchmark.ts
-/**
-* Resolve canonical `trace://<trace>/span/<span>` evidence against a trace store.
-* Other evidence kinds and URI schemes require a caller-supplied resolver.
-*/
-function traceStoreEvidenceResolver(getStore) {
-	return async ({ caseInput, evidence }) => {
-		if (evidence.kind !== "span") return false;
-		const location = parseTraceSpanUri(evidence.uri);
-		if (!location) return false;
-		const result = await getStore(caseInput).viewSpans({
-			trace_id: location.traceId,
-			span_ids: [location.spanId]
-		});
-		return result.trace_id === location.traceId && result.missing_span_ids.length === 0 && result.spans.some((span) => span.span_id === location.spanId);
-	};
-}
+//#region src/analyst/benchmark-scoring.ts
 function scoreAnalystFindings(testCase, findings) {
-	validateCase(testCase);
+	assertValidAnalystScoringCase(testCase);
 	const matchedFindingByIssue = matchFindingsToIssues(testCase.expectedIssues, findings);
 	const matchedIssueIds = testCase.expectedIssues.filter((_, index) => matchedFindingByIssue.has(index)).map((issue) => issue.id);
 	const missedIssueIds = testCase.expectedIssues.filter((_, index) => !matchedFindingByIssue.has(index)).map((issue) => issue.id);
@@ -27,7 +12,7 @@ function scoreAnalystFindings(testCase, findings) {
 	const expectedIssueCount = testCase.expectedIssues.length;
 	const issueRecall = expectedIssueCount === 0 ? 1 : matchedIssueIds.length / expectedIssueCount;
 	const findingPrecision = findings.length === 0 ? expectedIssueCount === 0 ? 1 : 0 : supportedFindingIndexes.size / findings.length;
-	const f1 = harmonicMean(findingPrecision, issueRecall);
+	const f1 = harmonicMeanScore(findingPrecision, issueRecall);
 	const allEvidence = findings.flatMap((finding) => finding.evidence_refs);
 	const criticalIssues = testCase.expectedIssues.filter((issue) => (issue.criticalEvidence?.length ?? 0) > 0);
 	const criticalHits = testCase.expectedIssues.filter((issue) => {
@@ -48,10 +33,25 @@ function scoreAnalystFindings(testCase, findings) {
 		f1,
 		criticalStepAccuracy: criticalIssues.length === 0 ? null : criticalHits / criticalIssues.length,
 		citationCoverage: findings.length === 0 ? null : findingsWithEvidence / findings.length,
+		citationExcerptCoverage: allEvidence.length === 0 ? null : allEvidence.filter((evidence) => Boolean(evidence.excerpt?.trim())).length / allEvidence.length,
 		citationLabelAgreement: testCase.labeledEvidence === void 0 ? null : allEvidence.length === 0 ? findings.length === 0 ? null : 0 : (allEvidence.length - unlabeledEvidence.length) / allEvidence.length,
-		cleanFalsePositive: expectedIssueCount === 0 && findings.length > 0
+		predictionOnLabelEmptyCase: expectedIssueCount === 0 && findings.length > 0
 	};
 }
+function assertValidAnalystScoringCase(testCase) {
+	if (!testCase.id.trim()) throw new TypeError("analyst benchmark case id must not be empty");
+	const ids = /* @__PURE__ */ new Set();
+	for (const issue of testCase.expectedIssues) {
+		if (!issue.id.trim()) throw new TypeError(`${testCase.id}: expected issue id must not be empty`);
+		if (ids.has(issue.id)) throw new TypeError(`${testCase.id}: duplicate expected issue id '${issue.id}'`);
+		ids.add(issue.id);
+		if (!issue.findingIds?.length && !issue.areas?.length && !issue.subjects?.length && !issue.evidence?.length) throw new TypeError(`${testCase.id}/${issue.id}: expected issue must identify a finding by id, area, subject, or evidence`);
+	}
+	for (const ref of testCase.labeledEvidence ?? []) if (!ref.uri.trim()) throw new TypeError(`${testCase.id}: labeled evidence URI must not be empty`);
+}
+function harmonicMeanScore(a, b) {
+	return a + b === 0 ? 0 : 2 * a * b / (a + b);
+}
 function matchFindingsToIssues(issues, findings) {
 	if (issues.length === 0) return /* @__PURE__ */ new Map();
 	const cardinalityWeight = issues.length + 1;
@@ -68,23 +68,218 @@ function matchFindingsToIssues(issues, findings) {
 	}
 	return matches;
 }
+function findingMatchesIssue(finding, issue) {
+	if (issue.findingIds && !issue.findingIds.includes(finding.finding_id)) return false;
+	if (issue.areas && !issue.areas.includes(finding.area)) return false;
+	if (issue.subjects && (!finding.subject || !issue.subjects.includes(finding.subject))) return false;
+	if (issue.evidence && !matchesEvidence(finding.evidence_refs, issue.evidence, issue.evidenceMode ?? "any")) return false;
+	return true;
+}
+function matchesEvidence(actual, expected, mode) {
+	if (expected.length === 0) return true;
+	const match = (target) => actual.some((ref) => evidenceMatches(ref, target));
+	return mode === "all" ? expected.every(match) : expected.some(match);
+}
+function evidenceMatches(actual, expected) {
+	return actual.uri === expected.uri && (expected.kind === void 0 || actual.kind === expected.kind);
+}
+//#endregion
+//#region src/analyst/benchmark-summary.ts
+function summarizeAnalystBenchmarkRunner(runnerId, observations) {
+	const issueBearing = observations.filter((observation) => observation.labelState === "positive");
+	const completed = observations.filter((observation) => !observation.error);
+	const expectedIssues = issueBearing.reduce((sum, observation) => sum + observation.score.expectedIssueCount, 0);
+	const matchedIssues = issueBearing.reduce((sum, observation) => sum + observation.score.matchedIssueIds.length, 0);
+	const issueFindings = issueBearing.reduce((sum, observation) => sum + (observation.error ? 0 : observation.findings.length), 0);
+	const supportedFindings = issueBearing.reduce((sum, observation) => sum + observation.score.supportedFindingIndexes.length, 0);
+	const issueRecall = expectedIssues === 0 ? null : matchedIssues / expectedIssues;
+	const findingPrecision = expectedIssues === 0 ? null : issueFindings === 0 ? 0 : supportedFindings / issueFindings;
+	const macroIssueRecall = issueBearing.length === 0 ? null : mean(issueBearing.map((observation) => observation.score.issueRecall));
+	const macroFindingPrecision = issueBearing.length === 0 ? null : mean(issueBearing.map((observation) => observation.score.findingPrecision));
+	const macroF1 = issueBearing.length === 0 ? null : mean(issueBearing.map((observation) => observation.score.f1));
+	const critical = observations.map((observation) => observation.score.criticalStepAccuracy).filter((value) => value !== null);
+	const findingsWithEvidence = completed.reduce((sum, observation) => sum + observation.findings.filter((finding) => finding.evidence_refs.length > 0).length, 0);
+	const allFindings = completed.reduce((sum, observation) => sum + observation.findings.length, 0);
+	const allCitations = completed.flatMap((observation) => observation.findings.flatMap((finding) => finding.evidence_refs));
+	const citationObservations = completed.filter((observation) => observation.score.citationLabelAgreement !== null);
+	const citationCount = citationObservations.reduce((sum, observation) => sum + observation.findings.reduce((count, finding) => count + finding.evidence_refs.length, 0), 0);
+	const invalidCitationCount = citationObservations.reduce((sum, observation) => sum + observation.score.unlabeledEvidence.length, 0);
+	const trustedNegative = observations.filter((observation) => observation.labelState === "trusted-negative");
+	const completedTrustedNegative = trustedNegative.filter((observation) => !observation.error);
+	const unlabeled = observations.filter((observation) => observation.labelState === "unlabeled");
+	const completedUnlabeled = unlabeled.filter((observation) => !observation.error);
+	const resolvedCitations = completed.reduce((sum, observation) => sum + (observation.evidenceResolution?.resolved ?? 0), 0);
+	const unresolvedCitations = completed.reduce((sum, observation) => sum + (observation.evidenceResolution?.unresolvedEvidence.length ?? 0), 0);
+	const citationResolutionErrors = completed.reduce((sum, observation) => sum + (observation.evidenceResolution?.errors.length ?? 0), 0);
+	const resolutionAttempts = completed.filter((observation) => observation.findings.some((finding) => finding.evidence_refs.length > 0));
+	const citationResolutionUnknownRuns = resolutionAttempts.filter((observation) => !observation.evidenceResolution || observation.evidenceResolution.errors.length > 0).length;
+	const usages = observations.map((observation) => observation.usage);
+	const knownCostUsd = stableSum(usages.map((usage) => {
+		if (!usage) return 0;
+		return usage.cost.kind === "uncaptured" ? usage.knownCostUsd ?? 0 : usage.cost.usd;
+	}));
+	const predictionAgreement = repeatedObservationAgreement(observations, predictionSignature);
+	const matchedLabelAgreement = repeatedObservationAgreement(observations.filter((observation) => observation.labelState === "positive"), matchedLabelSignature);
+	return {
+		runnerId,
+		plannedRuns: observations.length,
+		completedRuns: observations.filter((observation) => !observation.error).length,
+		failedRuns: observations.filter((observation) => Boolean(observation.error)).length,
+		issueBearingRuns: issueBearing.length,
+		trustedNegativeRuns: trustedNegative.length,
+		unlabeledRuns: unlabeled.length,
+		issueRecall,
+		findingPrecision,
+		f1: findingPrecision === null || issueRecall === null ? null : harmonicMeanScore(findingPrecision, issueRecall),
+		macroIssueRecall,
+		macroFindingPrecision,
+		macroF1,
+		criticalStepAccuracy: critical.length === 0 ? null : mean(critical),
+		citationCoverage: allFindings === 0 ? null : findingsWithEvidence / allFindings,
+		citationExcerptCoverage: allCitations.length === 0 ? null : allCitations.filter((evidence) => Boolean(evidence.excerpt?.trim())).length / allCitations.length,
+		citationLabelAgreement: citationObservations.length === 0 ? null : citationCount === 0 ? 0 : (citationCount - invalidCitationCount) / citationCount,
+		citationResolution: resolutionAttempts.length === 0 || citationResolutionUnknownRuns > 0 || resolvedCitations + unresolvedCitations === 0 ? null : resolvedCitations / (resolvedCitations + unresolvedCitations),
+		citationResolutionUnknownRuns,
+		unresolvedCitations,
+		citationResolutionErrors,
+		trustedNegativeFalsePositiveRate: completedTrustedNegative.length === 0 ? null : completedTrustedNegative.filter((observation) => observation.score.predictionOnLabelEmptyCase).length / completedTrustedNegative.length,
+		trustedNegativeFailureRate: trustedNegative.length === 0 ? null : trustedNegative.filter((observation) => Boolean(observation.error)).length / trustedNegative.length,
+		unlabeledPredictionRate: completedUnlabeled.length === 0 ? null : completedUnlabeled.filter((observation) => observation.findings.length > 0).length / completedUnlabeled.length,
+		unlabeledFailureRate: unlabeled.length === 0 ? null : unlabeled.filter((observation) => Boolean(observation.error)).length / unlabeled.length,
+		predictionAgreement: predictionAgreement.value,
+		predictionAgreementCases: predictionAgreement.cases,
+		matchedLabelAgreement: matchedLabelAgreement.value,
+		matchedLabelAgreementCases: matchedLabelAgreement.cases,
+		latencyMs: latencyDistribution(observations.map((observation) => observation.latencyMs).filter((value) => value !== null)),
+		benchmarkClockLatencyRuns: observations.filter((observation) => observation.latencySource === "benchmark-clock").length,
+		runnerReportedLatencyRuns: observations.filter((observation) => observation.latencySource === "runner-reported").length,
+		latencyUnknownRuns: observations.filter((observation) => observation.latencySource === "uncaptured").length,
+		calls: usages.reduce((sum, usage) => sum + (usage?.calls ?? 0), 0),
+		callsUnknownRuns: usages.filter((usage) => !usage || usage.calls === null).length,
+		inputTokens: usages.reduce((sum, usage) => sum + (usage?.tokens?.input ?? 0), 0),
+		outputTokens: usages.reduce((sum, usage) => sum + (usage?.tokens?.output ?? 0), 0),
+		reasoningTokens: usages.reduce((sum, usage) => sum + (usage?.tokens?.reasoning ?? 0), 0),
+		cachedTokens: usages.reduce((sum, usage) => sum + (usage?.tokens?.cached ?? 0), 0),
+		cacheWriteTokens: usages.reduce((sum, usage) => sum + (usage?.tokens?.cacheWrite ?? 0), 0),
+		tokenUsageUnknownRuns: usages.filter((usage) => !usage?.tokens).length,
+		reasoningTokenUsageUnknownRuns: usages.filter((usage) => usage?.tokens?.reasoning === void 0).length,
+		cachedTokenUsageUnknownRuns: usages.filter((usage) => usage?.tokens?.cached === void 0).length,
+		cacheWriteTokenUsageUnknownRuns: usages.filter((usage) => usage?.tokens?.cacheWrite === void 0).length,
+		knownCostUsd,
+		costUnknownRuns: usages.filter((usage) => !usage || usage.cost.kind === "uncaptured").length
+	};
+}
+function mean(values) {
+	return values.length === 0 ? 0 : stableSum(values) / values.length;
+}
+function stableSum(values) {
+	const ordered = [...values].sort((left, right) => Math.abs(left) - Math.abs(right) || left - right);
+	let sum = 0;
+	let correction = 0;
+	for (const value of ordered) {
+		const next = sum + value;
+		correction += Math.abs(sum) >= Math.abs(value) ? sum - next + value : value - next + sum;
+		sum = next;
+	}
+	return sum + correction;
+}
+function latencyDistribution(values) {
+	if (values.length === 0) return null;
+	const sorted = [...values].sort((a, b) => a - b);
+	return {
+		min: sorted[0],
+		mean: mean(sorted),
+		p50: percentile(sorted, .5),
+		p95: percentile(sorted, .95),
+		max: sorted.at(-1)
+	};
+}
+function percentile(sorted, quantile) {
+	if (sorted.length === 0) return 0;
+	return sorted[Math.ceil(quantile * sorted.length) - 1] ?? sorted.at(-1) ?? 0;
+}
+function repeatedObservationAgreement(observations, signature) {
+	const byCase = /* @__PURE__ */ new Map();
+	for (const observation of observations) {
+		const rows = byCase.get(observation.caseId) ?? [];
+		rows.push(observation);
+		byCase.set(observation.caseId, rows);
+	}
+	const caseAgreements = [];
+	for (const rows of byCase.values()) {
+		const agreements = [];
+		for (let left = 0; left < rows.length; left++) for (let right = left + 1; right < rows.length; right++) agreements.push(jaccard(signature(rows[left]), signature(rows[right])));
+		if (agreements.length > 0) caseAgreements.push(mean(agreements));
+	}
+	return {
+		value: caseAgreements.length === 0 ? null : mean(caseAgreements),
+		cases: caseAgreements.length
+	};
+}
+function matchedLabelSignature(observation) {
+	if (observation.error) return [`error:${observation.error.class}`];
+	return observation.score.matchedIssueIds;
+}
+function predictionSignature(observation) {
+	if (observation.error) return [`error:${observation.error.class}`];
+	return observation.findings.map((finding) => JSON.stringify([finding.finding_id, finding.evidence_refs.map((evidence) => [
+		evidence.kind,
+		evidence.uri,
+		evidence.excerpt ?? null
+	]).sort((left, right) => JSON.stringify(left).localeCompare(JSON.stringify(right)))])).sort();
+}
+function jaccard(left, right) {
+	const a = new Set(left);
+	const b = new Set(right);
+	const union = /* @__PURE__ */ new Set([...a, ...b]);
+	if (union.size === 0) return 1;
+	let intersection = 0;
+	for (const value of a) if (b.has(value)) intersection += 1;
+	return intersection / union.size;
+}
+//#endregion
+//#region src/analyst/benchmark.ts
+/**
+* Resolve canonical `trace://<trace>/span/<span>` evidence against a trace store.
+* Other evidence kinds and URI schemes require a caller-supplied resolver.
+*/
+function traceStoreEvidenceResolver(getStore) {
+	return async ({ caseInput, evidence, signal }) => {
+		if (evidence.kind !== "span") return false;
+		const location = parseTraceSpanUri(evidence.uri);
+		if (!location) return false;
+		const result = await getStore(caseInput).viewSpans({
+			trace_id: location.traceId,
+			span_ids: [location.spanId]
+		}, signal ? { signal } : void 0);
+		return result.trace_id === location.traceId && result.missing_span_ids.length === 0 && result.spans.some((span) => span.span_id === location.spanId);
+	};
+}
 async function runAnalystBenchmark(options) {
 	validateBenchmarkOptions(options);
 	const startedAt = (/* @__PURE__ */ new Date()).toISOString();
 	const repetitions = options.repetitions ?? 1;
 	const runnerOrderSeed = options.runnerOrderSeed ?? 0;
-	const maxConcurrency = Math.min(options.maxConcurrency ?? 1, options.runners.length * options.cases.length * repetitions);
-	const jobs = benchmarkJobs(options.cases, options.runners, repetitions, runnerOrderSeed);
-	const observations = [];
+	const allJobs = benchmarkJobs(options.cases, options.runners, repetitions, runnerOrderSeed);
+	const maxConcurrency = Math.min(options.maxConcurrency ?? 1, allJobs.length);
+	const initialObservations = validateInitialObservations(options.initialObservations ?? [], allJobs);
+	const completed = new Set(initialObservations.map(observationKey));
+	const jobs = allJobs.filter((job) => !completed.has(jobKey(job)));
+	const observations = [...initialObservations];
 	let cursor = 0;
 	const worker = async () => {
 		while (cursor < jobs.length) {
 			options.signal?.throwIfAborted();
 			const job = jobs[cursor++];
-			observations.push(await runBenchmarkJob(job, options.signal, options.resolveEvidence));
+			const observation = await runBenchmarkJob(job, options.signal, options.resolveEvidence);
+			options.signal?.throwIfAborted();
+			await options.onObservation?.(observation);
+			options.signal?.throwIfAborted();
+			observations.push(observation);
 		}
 	};
-	await Promise.all(Array.from({ length: maxConcurrency }, worker));
+	await Promise.all(Array.from({ length: Math.min(maxConcurrency, jobs.length) }, worker));
+	options.signal?.throwIfAborted();
 	const runnerOrder = new Map(options.runners.map((runner, index) => [runner.id, index]));
 	const caseOrder = new Map(options.cases.map((testCase, index) => [testCase.id, index]));
 	observations.sort((a, b) => (runnerOrder.get(a.runnerId) ?? 0) - (runnerOrder.get(b.runnerId) ?? 0) || (caseOrder.get(a.caseId) ?? 0) - (caseOrder.get(b.caseId) ?? 0) || a.repetition - b.repetition);
@@ -100,7 +295,7 @@ async function runAnalystBenchmark(options) {
 			runnerOrderSeed
 		},
 		observations,
-		summaries: options.runners.map((runner) => summarizeRunner(runner.id, observations.filter((observation) => observation.runnerId === runner.id)))
+		summaries: options.runners.map((runner) => summarizeAnalystBenchmarkRunner(runner.id, observations.filter((observation) => observation.runnerId === runner.id)))
 	};
 }
 function registryBenchmarkRunner(options) {
@@ -114,7 +309,8 @@ function registryBenchmarkRunner(options) {
 			return {
 				findings: result.findings,
 				usage: mergeRegistryUsage(result),
-				metadata: { analystRun: result }
+				metadata: { analystRun: result },
+				...options.failOnAnalystFailure ? { error: registryRunFailure(result) } : {}
 			};
 		}
 	};
@@ -127,19 +323,27 @@ async function runBenchmarkJob(job, signal, resolveEvidence) {
 			repetition: job.repetition,
 			signal
 		});
+		if (output.usage) assertValidAnalystUsageReceipt(output.usage, "analyst benchmark usage");
+		const benchmarkLatencyMs = performance.now() - started;
+		const latency = resolveBenchmarkLatency(output.observedLatencyMs, benchmarkLatencyMs);
+		const scoredFindings = output.error ? [] : output.findings;
 		return {
 			runnerId: job.runner.id,
 			caseId: job.testCase.id,
+			clusterId: job.testCase.clusterId,
+			labelState: job.testCase.labelState,
 			repetition: job.repetition,
 			executionIndex: job.executionIndex,
-			latencyMs: performance.now() - started,
+			latencyMs: latency.value,
+			latencySource: latency.source,
 			findings: output.findings,
-			score: scoreAnalystFindings(job.testCase, output.findings),
+			score: scoreAnalystFindings(job.testCase, scoredFindings),
 			evidenceResolution: resolveEvidence ? await resolveFindingEvidence(job.testCase, output.findings, resolveEvidence, signal) : void 0,
 			caseTags: [...job.testCase.tags ?? []],
 			caseMetadata: job.testCase.metadata,
 			usage: output.usage,
-			runnerMetadata: output.metadata
+			runnerMetadata: output.metadata,
+			...output.error ? { error: output.error } : {}
 		};
 	} catch (error) {
 		if (signal?.aborted) throw error;
@@ -147,9 +351,12 @@ async function runBenchmarkJob(job, signal, resolveEvidence) {
 		return {
 			runnerId: job.runner.id,
 			caseId: job.testCase.id,
+			clusterId: job.testCase.clusterId,
+			labelState: job.testCase.labelState,
 			repetition: job.repetition,
 			executionIndex: job.executionIndex,
 			latencyMs: performance.now() - started,
+			latencySource: "benchmark-clock",
 			findings,
 			score: scoreAnalystFindings(job.testCase, findings),
 			caseTags: [...job.testCase.tags ?? []],
@@ -161,6 +368,29 @@ async function runBenchmarkJob(job, signal, resolveEvidence) {
 		};
 	}
 }
+function resolveBenchmarkLatency(observedLatencyMs, fallbackMs) {
+	if (observedLatencyMs === void 0) return {
+		value: fallbackMs,
+		source: "benchmark-clock"
+	};
+	if (observedLatencyMs === null) return {
+		value: null,
+		source: "uncaptured"
+	};
+	if (!Number.isFinite(observedLatencyMs) || observedLatencyMs < 0) throw new RangeError("analyst benchmark observedLatencyMs must be finite and non-negative");
+	return {
+		value: observedLatencyMs,
+		source: "runner-reported"
+	};
+}
+function registryRunFailure(result) {
+	const failed = result.per_analyst.filter((summary) => summary.status === "failed");
+	if (failed.length === 0) return void 0;
+	return {
+		class: "AnalystRunFailure",
+		message: failed.map((summary) => `${summary.analyst_id}: ${summary.error?.class ?? "Error"}: ${summary.error?.message ?? "analyst failed"}`).join("; ")
+	};
+}
 function benchmarkJobs(cases, runners, repetitions, runnerOrderSeed) {
 	const seededRunners = [...runners].sort((left, right) => stableHash(`${runnerOrderSeed}\u0000${left.id}`) - stableHash(`${runnerOrderSeed}\u0000${right.id}`) || left.id.localeCompare(right.id));
 	let executionIndex = 0;
@@ -174,6 +404,35 @@ function benchmarkJobs(cases, runners, repetitions, runnerOrderSeed) {
 		}));
 	}).flat());
 }
+function validateInitialObservations(observations, jobs) {
+	const expected = new Map(jobs.map((job) => [jobKey(job), job]));
+	const seen = /* @__PURE__ */ new Set();
+	return observations.map((observation) => {
+		const key = observationKey(observation);
+		if (seen.has(key)) throw new TypeError(`duplicate initial analyst benchmark observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}'`);
+		seen.add(key);
+		const job = expected.get(key);
+		if (!job) throw new TypeError(`initial analyst benchmark observation does not match a planned job: '${observation.runnerId}/${observation.caseId}/${observation.repetition}'`);
+		if (observation.executionIndex !== job.executionIndex) throw new TypeError(`initial analyst benchmark observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}' has executionIndex ${observation.executionIndex}; expected ${job.executionIndex}`);
+		if (observation.clusterId !== job.testCase.clusterId || observation.labelState !== job.testCase.labelState) throw new TypeError(`initial analyst benchmark observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}' does not match the current case labels`);
+		if (JSON.stringify(observation.caseTags) !== JSON.stringify(job.testCase.tags ?? []) || JSON.stringify(observation.caseMetadata) !== JSON.stringify(job.testCase.metadata)) throw new TypeError(`initial analyst benchmark observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}' does not match the current case metadata`);
+		const expectedScore = scoreAnalystFindings(job.testCase, observation.error ? [] : observation.findings);
+		if (JSON.stringify(observation.score) !== JSON.stringify(expectedScore)) throw new TypeError(`initial analyst benchmark observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}' has stale or invalid scores`);
+		if (observation.usage) assertValidAnalystUsageReceipt(observation.usage, `initial analyst benchmark observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}' usage`);
+		if (![
+			"benchmark-clock",
+			"runner-reported",
+			"uncaptured"
+		].includes(observation.latencySource) || observation.latencySource === "uncaptured" && observation.latencyMs !== null || observation.latencySource !== "uncaptured" && (observation.latencyMs === null || !Number.isFinite(observation.latencyMs) || observation.latencyMs < 0)) throw new TypeError(`initial analyst benchmark observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}' has invalid latency`);
+		return { ...observation };
+	});
+}
+function jobKey(job) {
+	return `${job.runner.id}\u0000${job.testCase.id}\u0000${job.repetition}`;
+}
+function observationKey(observation) {
+	return `${observation.runnerId}\u0000${observation.caseId}\u0000${observation.repetition}`;
+}
 async function resolveFindingEvidence(testCase, findings, resolver, signal) {
 	const evidence = findings.flatMap((finding) => finding.evidence_refs);
 	const resolved = [];
@@ -206,81 +465,6 @@ async function resolveFindingEvidence(testCase, findings, resolver, signal) {
 		validity: evidence.length === 0 || errors.length > 0 ? null : resolved.length / evidence.length
 	};
 }
-function summarizeRunner(runnerId, observations) {
-	const issueBearing = observations.filter((observation) => observation.score.expectedIssueCount > 0);
-	const expectedIssues = issueBearing.reduce((sum, observation) => sum + observation.score.expectedIssueCount, 0);
-	const matchedIssues = issueBearing.reduce((sum, observation) => sum + observation.score.matchedIssueIds.length, 0);
-	const issueFindings = issueBearing.reduce((sum, observation) => sum + observation.findings.length, 0);
-	const supportedFindings = issueBearing.reduce((sum, observation) => sum + observation.score.supportedFindingIndexes.length, 0);
-	const issueRecall = expectedIssues === 0 ? null : matchedIssues / expectedIssues;
-	const findingPrecision = expectedIssues === 0 ? null : issueFindings === 0 ? 0 : supportedFindings / issueFindings;
-	const critical = observations.map((observation) => observation.score.criticalStepAccuracy).filter((value) => value !== null);
-	const findingsWithEvidence = observations.reduce((sum, observation) => sum + observation.findings.filter((finding) => finding.evidence_refs.length > 0).length, 0);
-	const allFindings = observations.reduce((sum, observation) => sum + observation.findings.length, 0);
-	const citationObservations = observations.filter((observation) => observation.score.citationLabelAgreement !== null);
-	const citationCount = citationObservations.reduce((sum, observation) => sum + observation.findings.reduce((count, finding) => count + finding.evidence_refs.length, 0), 0);
-	const invalidCitationCount = citationObservations.reduce((sum, observation) => sum + observation.score.unlabeledEvidence.length, 0);
-	const clean = observations.filter((observation) => observation.score.expectedIssueCount === 0);
-	const completedClean = clean.filter((observation) => !observation.error);
-	const resolvedCitations = observations.reduce((sum, observation) => sum + (observation.evidenceResolution?.resolved ?? 0), 0);
-	const unresolvedCitations = observations.reduce((sum, observation) => sum + (observation.evidenceResolution?.unresolvedEvidence.length ?? 0), 0);
-	const citationResolutionErrors = observations.reduce((sum, observation) => sum + (observation.evidenceResolution?.errors.length ?? 0), 0);
-	const resolutionAttempts = observations.filter((observation) => observation.findings.some((finding) => finding.evidence_refs.length > 0));
-	const citationResolutionUnknownRuns = resolutionAttempts.filter((observation) => !observation.evidenceResolution || observation.evidenceResolution.errors.length > 0).length;
-	const usages = observations.map((observation) => observation.usage);
-	const knownCostUsd = usages.reduce((sum, usage) => {
-		if (!usage) return sum;
-		return sum + (usage.cost.kind === "uncaptured" ? usage.knownCostUsd ?? 0 : usage.cost.usd);
-	}, 0);
-	return {
-		runnerId,
-		plannedRuns: observations.length,
-		completedRuns: observations.filter((observation) => !observation.error).length,
-		failedRuns: observations.filter((observation) => Boolean(observation.error)).length,
-		issueRecall,
-		findingPrecision,
-		f1: findingPrecision === null || issueRecall === null ? null : harmonicMean(findingPrecision, issueRecall),
-		criticalStepAccuracy: critical.length === 0 ? null : mean(critical),
-		citationCoverage: allFindings === 0 ? null : findingsWithEvidence / allFindings,
-		citationLabelAgreement: citationObservations.length === 0 ? null : citationCount === 0 ? 0 : (citationCount - invalidCitationCount) / citationCount,
-		citationResolution: resolutionAttempts.length === 0 || citationResolutionUnknownRuns > 0 || resolvedCitations + unresolvedCitations === 0 ? null : resolvedCitations / (resolvedCitations + unresolvedCitations),
-		citationResolutionUnknownRuns,
-		unresolvedCitations,
-		citationResolutionErrors,
-		cleanCaseFalsePositiveRate: completedClean.length === 0 ? null : completedClean.filter((observation) => observation.score.cleanFalsePositive).length / completedClean.length,
-		cleanCaseFailureRate: clean.length === 0 ? null : clean.filter((observation) => Boolean(observation.error)).length / clean.length,
-		runAgreement: issueAgreement(observations.filter((observation) => !observation.error)),
-		latencyMs: latencyDistribution(observations.map((observation) => observation.latencyMs)),
-		calls: usages.reduce((sum, usage) => sum + (usage?.calls ?? 0), 0),
-		callsUnknownRuns: usages.filter((usage) => !usage || usage.calls === null).length,
-		inputTokens: usages.reduce((sum, usage) => sum + (usage?.tokens?.input ?? 0), 0),
-		outputTokens: usages.reduce((sum, usage) => sum + (usage?.tokens?.output ?? 0), 0),
-		reasoningTokens: usages.reduce((sum, usage) => sum + (usage?.tokens?.reasoning ?? 0), 0),
-		cachedTokens: usages.reduce((sum, usage) => sum + (usage?.tokens?.cached ?? 0), 0),
-		cacheWriteTokens: usages.reduce((sum, usage) => sum + (usage?.tokens?.cacheWrite ?? 0), 0),
-		tokenUsageUnknownRuns: usages.filter((usage) => !usage?.tokens).length,
-		reasoningTokenUsageUnknownRuns: usages.filter((usage) => usage?.tokens?.reasoning === void 0).length,
-		cachedTokenUsageUnknownRuns: usages.filter((usage) => usage?.tokens?.cached === void 0).length,
-		cacheWriteTokenUsageUnknownRuns: usages.filter((usage) => usage?.tokens?.cacheWrite === void 0).length,
-		knownCostUsd,
-		costUnknownRuns: usages.filter((usage) => !usage || usage.cost.kind === "uncaptured").length
-	};
-}
-function findingMatchesIssue(finding, issue) {
-	if (issue.findingIds && !issue.findingIds.includes(finding.finding_id)) return false;
-	if (issue.areas && !issue.areas.includes(finding.area)) return false;
-	if (issue.subjects && (!finding.subject || !issue.subjects.includes(finding.subject))) return false;
-	if (issue.evidence && !matchesEvidence(finding.evidence_refs, issue.evidence, issue.evidenceMode ?? "any")) return false;
-	return true;
-}
-function matchesEvidence(actual, expected, mode) {
-	if (expected.length === 0) return true;
-	const match = (target) => actual.some((ref) => evidenceMatches(ref, target));
-	return mode === "all" ? expected.every(match) : expected.some(match);
-}
-function evidenceMatches(actual, expected) {
-	return actual.uri === expected.uri && (expected.kind === void 0 || actual.kind === expected.kind);
-}
 function parseTraceSpanUri(uri) {
 	const match = /^trace:\/\/([^/]+)\/span\/([^/]+)$/.exec(uri);
 	if (!match) return null;
@@ -295,17 +479,6 @@ function parseTraceSpanUri(uri) {
 		return null;
 	}
 }
-function validateCase(testCase) {
-	if (!testCase.id.trim()) throw new TypeError("analyst benchmark case id must not be empty");
-	const ids = /* @__PURE__ */ new Set();
-	for (const issue of testCase.expectedIssues) {
-		if (!issue.id.trim()) throw new TypeError(`${testCase.id}: expected issue id must not be empty`);
-		if (ids.has(issue.id)) throw new TypeError(`${testCase.id}: duplicate expected issue id '${issue.id}'`);
-		ids.add(issue.id);
-		if (!issue.findingIds?.length && !issue.areas?.length && !issue.subjects?.length && !issue.evidence?.length) throw new TypeError(`${testCase.id}/${issue.id}: expected issue must identify a finding by id, area, subject, or evidence`);
-	}
-	for (const ref of testCase.labeledEvidence ?? []) if (!ref.uri.trim()) throw new TypeError(`${testCase.id}: labeled evidence URI must not be empty`);
-}
 function validateBenchmarkOptions(options) {
 	if (options.cases.length === 0) throw new TypeError("runAnalystBenchmark requires cases");
 	if (options.runners.length === 0) throw new TypeError("runAnalystBenchmark requires runners");
@@ -316,7 +489,14 @@ function validateBenchmarkOptions(options) {
 	if (!Number.isSafeInteger(options.runnerOrderSeed ?? 0)) throw new RangeError("runAnalystBenchmark runnerOrderSeed must be a safe integer");
 	assertUniqueNonEmpty(options.cases.map((testCase) => testCase.id), "case");
 	assertUniqueNonEmpty(options.runners.map((runner) => runner.id), "runner");
-	for (const testCase of options.cases) validateCase(testCase);
+	for (const testCase of options.cases) validateBenchmarkCase(testCase);
+}
+function validateBenchmarkCase(testCase) {
+	assertValidAnalystScoringCase(testCase);
+	if (!testCase.clusterId.trim()) throw new TypeError(`${testCase.id}: analyst benchmark clusterId must not be empty`);
+	if (testCase.labelState !== "positive" && testCase.labelState !== "trusted-negative" && testCase.labelState !== "unlabeled") throw new TypeError(`${testCase.id}: analyst benchmark labelState is invalid`);
+	if (testCase.labelState === "positive" && testCase.expectedIssues.length === 0) throw new TypeError(`${testCase.id}: positive case requires at least one expected issue`);
+	if (testCase.labelState !== "positive" && testCase.expectedIssues.length > 0) throw new TypeError(`${testCase.id}: ${testCase.labelState} case cannot contain expected issues`);
 }
 function assertUniqueNonEmpty(values, label) {
 	const seen = /* @__PURE__ */ new Set();
@@ -326,26 +506,6 @@ function assertUniqueNonEmpty(values, label) {
 		seen.add(value);
 	}
 }
-function harmonicMean(a, b) {
-	return a + b === 0 ? 0 : 2 * a * b / (a + b);
-}
-function mean(values) {
-	return values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length;
-}
-function latencyDistribution(values) {
-	const sorted = [...values].sort((a, b) => a - b);
-	return {
-		min: sorted[0] ?? 0,
-		mean: mean(sorted),
-		p50: percentile(sorted, .5),
-		p95: percentile(sorted, .95),
-		max: sorted.at(-1) ?? 0
-	};
-}
-function percentile(sorted, quantile) {
-	if (sorted.length === 0) return 0;
-	return sorted[Math.ceil(quantile * sorted.length) - 1] ?? sorted.at(-1) ?? 0;
-}
 function stableHash(value) {
 	let hash = 2166136261;
 	for (let index = 0; index < value.length; index += 1) {
@@ -354,26 +514,6 @@ function stableHash(value) {
 	}
 	return hash >>> 0;
 }
-function issueAgreement(observations) {
-	const byCase = /* @__PURE__ */ new Map();
-	for (const observation of observations) {
-		const rows = byCase.get(observation.caseId) ?? [];
-		rows.push(observation);
-		byCase.set(observation.caseId, rows);
-	}
-	const agreements = [];
-	for (const rows of byCase.values()) for (let left = 0; left < rows.length; left++) for (let right = left + 1; right < rows.length; right++) agreements.push(jaccard(rows[left].score.matchedIssueIds, rows[right].score.matchedIssueIds));
-	return agreements.length === 0 ? null : mean(agreements);
-}
-function jaccard(left, right) {
-	const a = new Set(left);
-	const b = new Set(right);
-	const union = /* @__PURE__ */ new Set([...a, ...b]);
-	if (union.size === 0) return 1;
-	let intersection = 0;
-	for (const value of a) if (b.has(value)) intersection += 1;
-	return intersection / union.size;
-}
 function mergeRegistryUsage(result) {
 	const usages = result.per_analyst.map((summary) => summary.usage);
 	const calls = usages.every((usage) => usage.calls !== null) ? usages.reduce((sum, usage) => sum + (usage.calls ?? 0), 0) : null;
@@ -409,6 +549,6 @@ function mergeRegistryUsage(result) {
 	};
 }
 //#endregion
-export { traceStoreEvidenceResolver as i, runAnalystBenchmark as n, scoreAnalystFindings as r, registryBenchmarkRunner as t };
+export { scoreAnalystFindings as a, summarizeAnalystBenchmarkRunner as i, runAnalystBenchmark as n, traceStoreEvidenceResolver as r, registryBenchmarkRunner as t };
 
-//# sourceMappingURL=benchmark-YDrpumqB.js.map
+//# sourceMappingURL=benchmark-CYtcIF2V.js.map