@tangle-network/agent-eval 0.216.1 → 0.217.1

package/dist/{benchmark-command-By0vh8q7.js → benchmark-command-xQK8nOI6.js}

@@ -1,23 +1,22 @@
-import { l as errorMessage, s as ValidationError, t as AgentEvalError } from "./errors-DVZiqS7K.js";
-import { a as hashCanonical, i as compareCodeUnits, o as jsonDocument, r as canonicalString } from "./canonical-B2RqLofv.js";
-import { i as pairedBootstrap, t as mean$2 } from "./mean-DHoMiudD.js";
+import { c as errorMessage, s as ValidationError, t as AgentEvalError } from "./errors-BcOs5MKW.js";
+import { a as hashCanonical, i as compareCodeUnits, o as jsonDocument, r as canonicalString } from "./canonical-Dq2hOQ8L.js";
+import { i as pairedBootstrap, t as mean$3 } from "./mean-BbA_JCzW.js";
 import { a as resolveModelPricing } from "./metrics-BKaPBtIG.js";
-import { a as CostLedgerPersistenceError, i as CostCeilingReachedError, n as CostAccountingIncompleteError, o as CostReceiptCaptureError, r as CostCallConflictError, s as CostReservationExceededError, t as CostLedger } from "./cost-ledger-BP8V4kUQ.js";
-import { d as withLedgerFileLock, f as writeLedgerFileAtomically } from "./journal-DfRz9kps.js";
-import "./ledger-core-Gq7yp8lP.js";
-import { N as createRunCostLedger, P as fsCampaignStorage, i as startExternalOptimizerModelProxy, r as runWithCleanup, x as AnalystFindingSchema, y as resolveExternalOptimizerProcessLimits } from "./external-optimizer-subprocess-BqW0hHcI.js";
-import { c as redactText } from "./redact-B5xXKqQ3.js";
-import { a as makeFinding } from "./types-IXD2HJzu.js";
-import { t as acquireSingleRunLock } from "./single-run-lock-CV9Eyd_q.js";
-import { M as RAW_FINDING_SCHEMA_PROMPT, N as RawAnalystFindingSchema, P as evidenceRefsFromRawFinding, g as TRACE_ANALYSIS_LIMITS, r as runTraceAnalyst } from "./kind-factory-BobI5Jy3.js";
-import { r as usageReceiptFromCostLedger } from "./usage-receipt-Cx3xNeyB.js";
-import { C as LlmResponseError, S as LlmCallError, n as callLlmJson } from "./llm-client-CyBAoYiX.js";
-import { t as createDspyRlmTraceEngine } from "./dspy-rlm-engine-LRCu61m5.js";
-import { i as otlpTextToTraceAnalysisStore, n as DEFAULT_MAX_TRACE_FILE_BYTES, t as BufferTraceStore } from "./store-otlp-DJ9dtA5x.js";
-import { i as summarizeAnalystBenchmarkRunner, n as runAnalystBenchmark, r as traceStoreEvidenceResolver } from "./benchmark-CXmVVHH2.js";
-import "./external-optimizer-process-BjMGUuds.js";
-import { t as assertEqualDeclarativeTerms } from "./equal-terms-BtdXayCJ.js";
-import { c as primeProtocolSha256, d as runPrimeExchange, f as decodeReplyRows, n as buildPrimePrompt, t as analystUsageReceiptFromPrimeUsage, u as projectPrimeTrajectory } from "./prime-protocol-CHKgA3a6.js";
+import { a as CostLedgerPersistenceError, i as CostCeilingReachedError, n as CostAccountingIncompleteError, o as CostReceiptCaptureError, r as CostCallConflictError, s as CostReservationExceededError, t as CostLedger } from "./cost-ledger-D0352Z4d.js";
+import { l as withLedgerFileLock, u as writeLedgerFileAtomically } from "./journal-CR4QikYp.js";
+import "./ledger-core-DrJhbms9.js";
+import { M as fsCampaignStorage, b as AnalystFindingSchema, i as startExternalOptimizerModelProxy, j as createRunCostLedger, r as runWithCleanup, v as resolveExternalOptimizerProcessLimits } from "./external-optimizer-subprocess-XIHBOSKt.js";
+import { s as redactText } from "./redact-mtkTiJVk.js";
+import { i as makeFinding } from "./types-DqFMNTgK.js";
+import { t as acquireSingleRunLock } from "./single-run-lock-U58yf8N6.js";
+import { a as LlmCallError, n as isRecord, o as LlmResponseError } from "./json-BhdsJaOQ.js";
+import { A as RawAnalystFindingSchema, U as assertValidAnalystUsageReceipt, W as usageReceiptFromCostLedger, j as evidenceRefsFromRawFinding, k as RAW_FINDING_SCHEMA_PROMPT, m as TRACE_ANALYSIS_LIMITS, n as runTraceAnalyst } from "./kind-factory-Gy4fXixd.js";
+import { n as callLlmJson } from "./llm-client-BshCREn6.js";
+import { t as createDspyRlmTraceEngine } from "./dspy-rlm-engine-DhdCr1Dc.js";
+import { i as otlpTextToTraceAnalysisStore, n as DEFAULT_MAX_TRACE_FILE_BYTES, t as BufferTraceStore } from "./store-otlp-BteDAbHK.js";
+import "./external-optimizer-process-KVFGvYX1.js";
+import { t as assertEqualDeclarativeTerms } from "./equal-terms-DbPGcVtI.js";
+import { c as runPrimeExchange, l as decodeReplyRows, n as buildPrimePrompt, o as primeProtocolSha256, s as projectPrimeTrajectory, t as analystUsageReceiptFromPrimeUsage } from "./prime-protocol-BFgGrEMP.js";
 import { t as nodeHttpPrimeBridgeTransport } from "./prime-bridge-transport-Z3cpKGoV.js";
 import { createHash, randomUUID } from "node:crypto";
 import { constants, existsSync, lstatSync, readFileSync } from "node:fs";
@@ -28,6 +27,504 @@ import { arch, platform } from "node:os";
 import { pathToFileURL } from "node:url";
 import { link, lstat, mkdir, open, readFile, readdir, realpath, stat, unlink } from "node:fs/promises";
 import { TextDecoder as TextDecoder$1 } from "node:util";
+import { performance } from "node:perf_hooks";
+import { linearSumAssignment } from "linear-sum-assignment";
+//#region src/analyst/benchmark-scoring.ts
+function scoreAnalystFindings(testCase, findings) {
+	assertValidAnalystScoringCase(testCase);
+	const matchedFindingByIssue = matchFindingsToIssues(testCase.expectedIssues, findings);
+	const matchedIssueIds = testCase.expectedIssues.filter((_, index) => matchedFindingByIssue.has(index)).map((issue) => issue.id);
+	const missedIssueIds = testCase.expectedIssues.filter((_, index) => !matchedFindingByIssue.has(index)).map((issue) => issue.id);
+	const supportedFindingIndexes = new Set(matchedFindingByIssue.values());
+	const unsupportedFindingIndexes = findings.map((_, index) => index).filter((index) => !supportedFindingIndexes.has(index));
+	const expectedIssueCount = testCase.expectedIssues.length;
+	const issueRecall = expectedIssueCount === 0 ? 1 : matchedIssueIds.length / expectedIssueCount;
+	const findingPrecision = findings.length === 0 ? expectedIssueCount === 0 ? 1 : 0 : supportedFindingIndexes.size / findings.length;
+	const f1 = harmonicMeanScore(findingPrecision, issueRecall);
+	const allEvidence = findings.flatMap((finding) => finding.evidence_refs);
+	const criticalIssues = testCase.expectedIssues.filter((issue) => (issue.criticalEvidence?.length ?? 0) > 0);
+	const criticalHits = testCase.expectedIssues.filter((issue) => {
+		if ((issue.criticalEvidence?.length ?? 0) === 0) return false;
+		return matchesEvidence(allEvidence, issue.criticalEvidence ?? [], "any");
+	}).length;
+	const findingsWithEvidence = findings.filter((finding) => finding.evidence_refs.length > 0).length;
+	const unlabeledEvidence = testCase.labeledEvidence ? allEvidence.filter((ref) => !testCase.labeledEvidence.some((expected) => evidenceMatches(ref, expected))) : [];
+	return {
+		expectedIssueCount,
+		matchedIssueIds,
+		missedIssueIds,
+		supportedFindingIndexes: [...supportedFindingIndexes].sort((a, b) => a - b),
+		unsupportedFindingIndexes,
+		unlabeledEvidence,
+		issueRecall,
+		findingPrecision,
+		f1,
+		criticalStepAccuracy: criticalIssues.length === 0 ? null : criticalHits / criticalIssues.length,
+		citationCoverage: findings.length === 0 ? null : findingsWithEvidence / findings.length,
+		citationExcerptCoverage: allEvidence.length === 0 ? null : allEvidence.filter((evidence) => Boolean(evidence.excerpt?.trim())).length / allEvidence.length,
+		citationLabelAgreement: testCase.labeledEvidence === void 0 ? null : allEvidence.length === 0 ? findings.length === 0 ? null : 0 : (allEvidence.length - unlabeledEvidence.length) / allEvidence.length,
+		predictionOnLabelEmptyCase: expectedIssueCount === 0 && findings.length > 0
+	};
+}
+function assertValidAnalystScoringCase(testCase) {
+	if (!testCase.id.trim()) throw new TypeError("analyst benchmark case id must not be empty");
+	const ids = /* @__PURE__ */ new Set();
+	for (const issue of testCase.expectedIssues) {
+		if (!issue.id.trim()) throw new TypeError(`${testCase.id}: expected issue id must not be empty`);
+		if (ids.has(issue.id)) throw new TypeError(`${testCase.id}: duplicate expected issue id '${issue.id}'`);
+		ids.add(issue.id);
+		if (!issue.findingIds?.length && !issue.areas?.length && !issue.subjects?.length && !issue.evidence?.length) throw new TypeError(`${testCase.id}/${issue.id}: expected issue must identify a finding by id, area, subject, or evidence`);
+	}
+	for (const ref of testCase.labeledEvidence ?? []) if (!ref.uri.trim()) throw new TypeError(`${testCase.id}: labeled evidence URI must not be empty`);
+}
+function harmonicMeanScore(a, b) {
+	return a + b === 0 ? 0 : 2 * a * b / (a + b);
+}
+function matchFindingsToIssues(issues, findings) {
+	if (issues.length === 0) return /* @__PURE__ */ new Map();
+	const cardinalityWeight = issues.length + 1;
+	const scores = issues.map((issue) => [...findings.map((finding) => {
+		if (!findingMatchesIssue(finding, issue)) return -1;
+		const criticalHit = (issue.criticalEvidence?.length ?? 0) > 0 && matchesEvidence(finding.evidence_refs, issue.criticalEvidence ?? [], "any");
+		return cardinalityWeight + Number(criticalHit);
+	}), ...Array.from({ length: issues.length }, () => 0)]);
+	const assignment = linearSumAssignment(scores, { maximaze: true }).rowAssignments;
+	const matches = /* @__PURE__ */ new Map();
+	for (const [issueIndex, column] of assignment.entries()) {
+		if (column < 0 || column >= findings.length) continue;
+		if (!findingMatchesIssue(findings[column], issues[issueIndex])) continue;
+		matches.set(issueIndex, column);
+	}
+	return matches;
+}
+function findingMatchesIssue(finding, issue) {
+	if (issue.findingIds && !issue.findingIds.includes(finding.finding_id)) return false;
+	if (issue.areas && !issue.areas.includes(finding.area)) return false;
+	if (issue.subjects && (!finding.subject || !issue.subjects.includes(finding.subject))) return false;
+	if (issue.evidence && !matchesEvidence(finding.evidence_refs, issue.evidence, issue.evidenceMode ?? "any")) return false;
+	return true;
+}
+function matchesEvidence(actual, expected, mode) {
+	if (expected.length === 0) return true;
+	const match = (target) => actual.some((ref) => evidenceMatches(ref, target));
+	return mode === "all" ? expected.every(match) : expected.some(match);
+}
+function evidenceMatches(actual, expected) {
+	return actual.uri === expected.uri && (expected.kind === void 0 || actual.kind === expected.kind);
+}
+//#endregion
+//#region src/analyst/benchmark-summary.ts
+function summarizeAnalystBenchmarkRunner(runnerId, observations) {
+	const issueBearing = observations.filter((observation) => observation.labelState === "positive");
+	const completed = observations.filter((observation) => !observation.error);
+	const expectedIssues = issueBearing.reduce((sum, observation) => sum + observation.score.expectedIssueCount, 0);
+	const matchedIssues = issueBearing.reduce((sum, observation) => sum + observation.score.matchedIssueIds.length, 0);
+	const issueFindings = issueBearing.reduce((sum, observation) => sum + (observation.error ? 0 : observation.findings.length), 0);
+	const supportedFindings = issueBearing.reduce((sum, observation) => sum + observation.score.supportedFindingIndexes.length, 0);
+	const issueRecall = expectedIssues === 0 ? null : matchedIssues / expectedIssues;
+	const findingPrecision = expectedIssues === 0 ? null : issueFindings === 0 ? 0 : supportedFindings / issueFindings;
+	const macroIssueRecall = issueBearing.length === 0 ? null : mean$2(issueBearing.map((observation) => observation.score.issueRecall));
+	const macroFindingPrecision = issueBearing.length === 0 ? null : mean$2(issueBearing.map((observation) => observation.score.findingPrecision));
+	const macroF1 = issueBearing.length === 0 ? null : mean$2(issueBearing.map((observation) => observation.score.f1));
+	const critical = observations.map((observation) => observation.score.criticalStepAccuracy).filter((value) => value !== null);
+	const findingsWithEvidence = completed.reduce((sum, observation) => sum + observation.findings.filter((finding) => finding.evidence_refs.length > 0).length, 0);
+	const allFindings = completed.reduce((sum, observation) => sum + observation.findings.length, 0);
+	const allCitations = completed.flatMap((observation) => observation.findings.flatMap((finding) => finding.evidence_refs));
+	const citationObservations = completed.filter((observation) => observation.score.citationLabelAgreement !== null);
+	const citationCount = citationObservations.reduce((sum, observation) => sum + observation.findings.reduce((count, finding) => count + finding.evidence_refs.length, 0), 0);
+	const invalidCitationCount = citationObservations.reduce((sum, observation) => sum + observation.score.unlabeledEvidence.length, 0);
+	const trustedNegative = observations.filter((observation) => observation.labelState === "trusted-negative");
+	const completedTrustedNegative = trustedNegative.filter((observation) => !observation.error);
+	const unlabeled = observations.filter((observation) => observation.labelState === "unlabeled");
+	const completedUnlabeled = unlabeled.filter((observation) => !observation.error);
+	const resolvedCitations = completed.reduce((sum, observation) => sum + (observation.evidenceResolution?.resolved ?? 0), 0);
+	const unresolvedCitations = completed.reduce((sum, observation) => sum + (observation.evidenceResolution?.unresolvedEvidence.length ?? 0), 0);
+	const citationResolutionErrors = completed.reduce((sum, observation) => sum + (observation.evidenceResolution?.errors.length ?? 0), 0);
+	const resolutionAttempts = completed.filter((observation) => observation.findings.some((finding) => finding.evidence_refs.length > 0));
+	const citationResolutionUnknownRuns = resolutionAttempts.filter((observation) => !observation.evidenceResolution || observation.evidenceResolution.errors.length > 0).length;
+	const usages = observations.map((observation) => observation.usage);
+	const knownCostUsd = stableSum(usages.map((usage) => {
+		if (!usage) return 0;
+		return usage.cost.kind === "uncaptured" ? usage.knownCostUsd ?? 0 : usage.cost.usd;
+	}));
+	const predictionAgreement = repeatedObservationAgreement(observations, predictionSignature);
+	const matchedLabelAgreement = repeatedObservationAgreement(observations.filter((observation) => observation.labelState === "positive"), matchedLabelSignature);
+	return {
+		runnerId,
+		plannedRuns: observations.length,
+		completedRuns: observations.filter((observation) => !observation.error).length,
+		failedRuns: observations.filter((observation) => Boolean(observation.error)).length,
+		issueBearingRuns: issueBearing.length,
+		trustedNegativeRuns: trustedNegative.length,
+		unlabeledRuns: unlabeled.length,
+		issueRecall,
+		findingPrecision,
+		f1: findingPrecision === null || issueRecall === null ? null : harmonicMeanScore(findingPrecision, issueRecall),
+		macroIssueRecall,
+		macroFindingPrecision,
+		macroF1,
+		criticalStepAccuracy: critical.length === 0 ? null : mean$2(critical),
+		citationCoverage: allFindings === 0 ? null : findingsWithEvidence / allFindings,
+		citationExcerptCoverage: allCitations.length === 0 ? null : allCitations.filter((evidence) => Boolean(evidence.excerpt?.trim())).length / allCitations.length,
+		citationLabelAgreement: citationObservations.length === 0 ? null : citationCount === 0 ? 0 : (citationCount - invalidCitationCount) / citationCount,
+		citationResolution: resolutionAttempts.length === 0 || citationResolutionUnknownRuns > 0 || resolvedCitations + unresolvedCitations === 0 ? null : resolvedCitations / (resolvedCitations + unresolvedCitations),
+		citationResolutionUnknownRuns,
+		unresolvedCitations,
+		citationResolutionErrors,
+		trustedNegativeFalsePositiveRate: completedTrustedNegative.length === 0 ? null : completedTrustedNegative.filter((observation) => observation.score.predictionOnLabelEmptyCase).length / completedTrustedNegative.length,
+		trustedNegativeFailureRate: trustedNegative.length === 0 ? null : trustedNegative.filter((observation) => Boolean(observation.error)).length / trustedNegative.length,
+		unlabeledPredictionRate: completedUnlabeled.length === 0 ? null : completedUnlabeled.filter((observation) => observation.findings.length > 0).length / completedUnlabeled.length,
+		unlabeledFailureRate: unlabeled.length === 0 ? null : unlabeled.filter((observation) => Boolean(observation.error)).length / unlabeled.length,
+		predictionAgreement: predictionAgreement.value,
+		predictionAgreementCases: predictionAgreement.cases,
+		matchedLabelAgreement: matchedLabelAgreement.value,
+		matchedLabelAgreementCases: matchedLabelAgreement.cases,
+		latencyMs: latencyDistribution(observations.map((observation) => observation.latencyMs).filter((value) => value !== null)),
+		benchmarkClockLatencyRuns: observations.filter((observation) => observation.latencySource === "benchmark-clock").length,
+		runnerReportedLatencyRuns: observations.filter((observation) => observation.latencySource === "runner-reported").length,
+		latencyUnknownRuns: observations.filter((observation) => observation.latencySource === "uncaptured").length,
+		calls: usages.reduce((sum, usage) => sum + (usage?.calls ?? 0), 0),
+		callsUnknownRuns: usages.filter((usage) => !usage || usage.calls === null).length,
+		inputTokens: usages.reduce((sum, usage) => sum + (usage?.tokens?.input ?? 0), 0),
+		outputTokens: usages.reduce((sum, usage) => sum + (usage?.tokens?.output ?? 0), 0),
+		reasoningTokens: usages.reduce((sum, usage) => sum + (usage?.tokens?.reasoning ?? 0), 0),
+		cachedTokens: usages.reduce((sum, usage) => sum + (usage?.tokens?.cached ?? 0), 0),
+		cacheWriteTokens: usages.reduce((sum, usage) => sum + (usage?.tokens?.cacheWrite ?? 0), 0),
+		tokenUsageUnknownRuns: usages.filter((usage) => !usage?.tokens).length,
+		reasoningTokenUsageUnknownRuns: usages.filter((usage) => usage?.tokens?.reasoning === void 0).length,
+		cachedTokenUsageUnknownRuns: usages.filter((usage) => usage?.tokens?.cached === void 0).length,
+		cacheWriteTokenUsageUnknownRuns: usages.filter((usage) => usage?.tokens?.cacheWrite === void 0).length,
+		knownCostUsd,
+		costUnknownRuns: usages.filter((usage) => !usage || usage.cost.kind === "uncaptured").length
+	};
+}
+function mean$2(values) {
+	return values.length === 0 ? 0 : stableSum(values) / values.length;
+}
+function stableSum(values) {
+	const ordered = [...values].sort((left, right) => Math.abs(left) - Math.abs(right) || left - right);
+	let sum = 0;
+	let correction = 0;
+	for (const value of ordered) {
+		const next = sum + value;
+		correction += Math.abs(sum) >= Math.abs(value) ? sum - next + value : value - next + sum;
+		sum = next;
+	}
+	return sum + correction;
+}
+function latencyDistribution(values) {
+	if (values.length === 0) return null;
+	const sorted = [...values].sort((a, b) => a - b);
+	return {
+		min: sorted[0],
+		mean: mean$2(sorted),
+		p50: percentile(sorted, .5),
+		p95: percentile(sorted, .95),
+		max: sorted.at(-1)
+	};
+}
+function percentile(sorted, quantile) {
+	if (sorted.length === 0) return 0;
+	return sorted[Math.ceil(quantile * sorted.length) - 1] ?? sorted.at(-1) ?? 0;
+}
+function repeatedObservationAgreement(observations, signature) {
+	const byCase = /* @__PURE__ */ new Map();
+	for (const observation of observations) {
+		const rows = byCase.get(observation.caseId) ?? [];
+		rows.push(observation);
+		byCase.set(observation.caseId, rows);
+	}
+	const caseAgreements = [];
+	for (const rows of byCase.values()) {
+		const agreements = [];
+		for (let left = 0; left < rows.length; left++) for (let right = left + 1; right < rows.length; right++) agreements.push(jaccard(signature(rows[left]), signature(rows[right])));
+		if (agreements.length > 0) caseAgreements.push(mean$2(agreements));
+	}
+	return {
+		value: caseAgreements.length === 0 ? null : mean$2(caseAgreements),
+		cases: caseAgreements.length
+	};
+}
+function matchedLabelSignature(observation) {
+	if (observation.error) return [`error:${observation.error.class}`];
+	return observation.score.matchedIssueIds;
+}
+function predictionSignature(observation) {
+	if (observation.error) return [`error:${observation.error.class}`];
+	return observation.findings.map((finding) => JSON.stringify([finding.finding_id, finding.evidence_refs.map((evidence) => [
+		evidence.kind,
+		evidence.uri,
+		evidence.excerpt ?? null
+	]).sort((left, right) => compareCodeUnits(JSON.stringify(left), JSON.stringify(right)))])).sort();
+}
+function jaccard(left, right) {
+	const a = new Set(left);
+	const b = new Set(right);
+	const union = /* @__PURE__ */ new Set([...a, ...b]);
+	if (union.size === 0) return 1;
+	let intersection = 0;
+	for (const value of a) if (b.has(value)) intersection += 1;
+	return intersection / union.size;
+}
+//#endregion
+//#region src/analyst/benchmark.ts
+/**
+* Resolve canonical `trace://<trace>/span/<span>` evidence against a trace store.
+* Other evidence kinds and URI schemes require a caller-supplied resolver.
+*/
+function traceStoreEvidenceResolver(getStore) {
+	return async ({ caseInput, evidence, signal }) => {
+		if (evidence.kind !== "span") return false;
+		const location = parseTraceSpanUri(evidence.uri);
+		if (!location) return false;
+		const result = await getStore(caseInput).viewSpans({
+			trace_id: location.traceId,
+			span_ids: [location.spanId]
+		}, signal ? { signal } : void 0);
+		return result.trace_id === location.traceId && result.missing_span_ids.length === 0 && result.spans.some((span) => span.span_id === location.spanId);
+	};
+}
+async function runAnalystBenchmark(options) {
+	validateBenchmarkOptions(options);
+	const startedAt = (/* @__PURE__ */ new Date()).toISOString();
+	const repetitions = options.repetitions ?? 1;
+	const runnerOrderSeed = options.runnerOrderSeed ?? 0;
+	const allJobs = benchmarkJobs(options.cases, options.runners, repetitions, runnerOrderSeed);
+	const maxConcurrency = Math.min(options.maxConcurrency ?? 1, allJobs.length);
+	const initialObservations = validateInitialObservations(options.initialObservations ?? [], allJobs);
+	const completed = new Set(initialObservations.map(observationKey$1));
+	const jobs = allJobs.filter((job) => !completed.has(jobKey(job)));
+	const observations = [...initialObservations];
+	let cursor = 0;
+	const worker = async () => {
+		while (cursor < jobs.length) {
+			options.signal?.throwIfAborted();
+			const job = jobs[cursor++];
+			const observation = await runBenchmarkJob(job, options.signal, options.resolveEvidence);
+			options.signal?.throwIfAborted();
+			await options.onObservation?.(observation);
+			options.signal?.throwIfAborted();
+			observations.push(observation);
+		}
+	};
+	await Promise.all(Array.from({ length: Math.min(maxConcurrency, jobs.length) }, worker));
+	options.signal?.throwIfAborted();
+	const runnerOrder = new Map(options.runners.map((runner, index) => [runner.id, index]));
+	const caseOrder = new Map(options.cases.map((testCase, index) => [testCase.id, index]));
+	observations.sort((a, b) => (runnerOrder.get(a.runnerId) ?? 0) - (runnerOrder.get(b.runnerId) ?? 0) || (caseOrder.get(a.caseId) ?? 0) - (caseOrder.get(b.caseId) ?? 0) || a.repetition - b.repetition);
+	return {
+		provenance: {
+			...options.benchmark,
+			startedAt,
+			endedAt: (/* @__PURE__ */ new Date()).toISOString(),
+			caseCount: options.cases.length,
+			runnerIds: options.runners.map((runner) => runner.id),
+			repetitions,
+			maxConcurrency,
+			runnerOrderSeed
+		},
+		observations,
+		summaries: options.runners.map((runner) => summarizeAnalystBenchmarkRunner(runner.id, observations.filter((observation) => observation.runnerId === runner.id)))
+	};
+}
+async function runBenchmarkJob(job, signal, resolveEvidence) {
+	const started = performance.now();
+	try {
+		const output = await job.runner.analyze(job.testCase.input, {
+			caseId: job.testCase.id,
+			repetition: job.repetition,
+			signal
+		});
+		if (output.usage) assertValidAnalystUsageReceipt(output.usage, "analyst benchmark usage");
+		const benchmarkLatencyMs = performance.now() - started;
+		const latency = resolveBenchmarkLatency(output.observedLatencyMs, benchmarkLatencyMs);
+		const scoredFindings = output.error ? [] : output.findings;
+		return {
+			runnerId: job.runner.id,
+			caseId: job.testCase.id,
+			clusterId: job.testCase.clusterId,
+			labelState: job.testCase.labelState,
+			repetition: job.repetition,
+			executionIndex: job.executionIndex,
+			latencyMs: latency.value,
+			latencySource: latency.source,
+			findings: output.findings,
+			score: scoreAnalystFindings(job.testCase, scoredFindings),
+			evidenceResolution: resolveEvidence ? await resolveFindingEvidence(job.testCase, output.findings, resolveEvidence, signal) : void 0,
+			caseTags: [...job.testCase.tags ?? []],
+			caseMetadata: job.testCase.metadata,
+			usage: output.usage,
+			runnerMetadata: output.metadata,
+			...output.error ? { error: output.error } : {}
+		};
+	} catch (error) {
+		if (signal?.aborted) throw error;
+		const findings = [];
+		return {
+			runnerId: job.runner.id,
+			caseId: job.testCase.id,
+			clusterId: job.testCase.clusterId,
+			labelState: job.testCase.labelState,
+			repetition: job.repetition,
+			executionIndex: job.executionIndex,
+			latencyMs: performance.now() - started,
+			latencySource: "benchmark-clock",
+			findings,
+			score: scoreAnalystFindings(job.testCase, findings),
+			caseTags: [...job.testCase.tags ?? []],
+			caseMetadata: job.testCase.metadata,
+			error: {
+				class: error instanceof Error ? error.constructor.name : "Error",
+				message: error instanceof Error ? error.message : String(error)
+			}
+		};
+	}
+}
+function resolveBenchmarkLatency(observedLatencyMs, fallbackMs) {
+	if (observedLatencyMs === void 0) return {
+		value: fallbackMs,
+		source: "benchmark-clock"
+	};
+	if (observedLatencyMs === null) return {
+		value: null,
+		source: "uncaptured"
+	};
+	if (!Number.isFinite(observedLatencyMs) || observedLatencyMs < 0) throw new RangeError("analyst benchmark observedLatencyMs must be finite and non-negative");
+	return {
+		value: observedLatencyMs,
+		source: "runner-reported"
+	};
+}
+function benchmarkJobs(cases, runners, repetitions, runnerOrderSeed) {
+	const seededRunners = [...runners].sort((left, right) => stableHash(`${runnerOrderSeed}\u0000${left.id}`) - stableHash(`${runnerOrderSeed}\u0000${right.id}`) || left.id.localeCompare(right.id));
+	let executionIndex = 0;
+	return cases.flatMap((testCase, caseIndex) => Array.from({ length: repetitions }, (_, repetition) => {
+		const rotation = (caseIndex * repetitions + repetition) % seededRunners.length;
+		return [...seededRunners.slice(rotation), ...seededRunners.slice(0, rotation)].map((runner) => ({
+			runner,
+			testCase,
+			repetition,
+			executionIndex: executionIndex++
+		}));
+	}).flat());
+}
+function validateInitialObservations(observations, jobs) {
+	const expected = new Map(jobs.map((job) => [jobKey(job), job]));
+	const seen = /* @__PURE__ */ new Set();
+	return observations.map((observation) => {
+		const key = observationKey$1(observation);
+		if (seen.has(key)) throw new TypeError(`duplicate initial analyst benchmark observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}'`);
+		seen.add(key);
+		const job = expected.get(key);
+		if (!job) throw new TypeError(`initial analyst benchmark observation does not match a planned job: '${observation.runnerId}/${observation.caseId}/${observation.repetition}'`);
+		if (observation.executionIndex !== job.executionIndex) throw new TypeError(`initial analyst benchmark observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}' has executionIndex ${observation.executionIndex}; expected ${job.executionIndex}`);
+		if (observation.clusterId !== job.testCase.clusterId || observation.labelState !== job.testCase.labelState) throw new TypeError(`initial analyst benchmark observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}' does not match the current case labels`);
+		if (JSON.stringify(observation.caseTags) !== JSON.stringify(job.testCase.tags ?? []) || JSON.stringify(observation.caseMetadata) !== JSON.stringify(job.testCase.metadata)) throw new TypeError(`initial analyst benchmark observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}' does not match the current case metadata`);
+		const expectedScore = scoreAnalystFindings(job.testCase, observation.error ? [] : observation.findings);
+		if (JSON.stringify(observation.score) !== JSON.stringify(expectedScore)) throw new TypeError(`initial analyst benchmark observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}' has stale or invalid scores`);
+		if (observation.usage) assertValidAnalystUsageReceipt(observation.usage, `initial analyst benchmark observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}' usage`);
+		if (![
+			"benchmark-clock",
+			"runner-reported",
+			"uncaptured"
+		].includes(observation.latencySource) || observation.latencySource === "uncaptured" && observation.latencyMs !== null || observation.latencySource !== "uncaptured" && (observation.latencyMs === null || !Number.isFinite(observation.latencyMs) || observation.latencyMs < 0)) throw new TypeError(`initial analyst benchmark observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}' has invalid latency`);
+		return { ...observation };
+	});
+}
+function jobKey(job) {
+	return `${job.runner.id}\u0000${job.testCase.id}\u0000${job.repetition}`;
+}
+function observationKey$1(observation) {
+	return `${observation.runnerId}\u0000${observation.caseId}\u0000${observation.repetition}`;
+}
+async function resolveFindingEvidence(testCase, findings, resolver, signal) {
+	const evidence = findings.flatMap((finding) => finding.evidence_refs);
+	const resolved = [];
+	const unresolvedEvidence = [];
+	const errors = [];
+	for (const ref of evidence) {
+		signal?.throwIfAborted();
+		try {
+			if (await resolver({
+				caseId: testCase.id,
+				caseInput: testCase.input,
+				evidence: ref,
+				signal
+			})) resolved.push(ref);
+			else unresolvedEvidence.push(ref);
+		} catch (error) {
+			if (signal?.aborted) throw error;
+			errors.push({
+				evidence: ref,
+				class: error instanceof Error ? error.constructor.name : "Error",
+				message: error instanceof Error ? error.message : String(error)
+			});
+		}
+	}
+	return {
+		checked: evidence.length,
+		resolved: resolved.length,
+		unresolvedEvidence,
+		errors,
+		validity: evidence.length === 0 || errors.length > 0 ? null : resolved.length / evidence.length
+	};
+}
+function parseTraceSpanUri(uri) {
+	const match = /^trace:\/\/([^/]+)\/span\/([^/]+)$/.exec(uri);
+	if (!match) return null;
+	try {
+		const traceId = decodeURIComponent(match[1]);
+		const spanId = decodeURIComponent(match[2]);
+		return traceId && spanId ? {
+			traceId,
+			spanId
+		} : null;
+	} catch {
+		return null;
+	}
+}
+function validateBenchmarkOptions(options) {
+	if (options.cases.length === 0) throw new TypeError("runAnalystBenchmark requires cases");
+	if (options.runners.length === 0) throw new TypeError("runAnalystBenchmark requires runners");
+	const repetitions = options.repetitions ?? 1;
+	const maxConcurrency = options.maxConcurrency ?? 1;
+	if (!Number.isSafeInteger(repetitions) || repetitions < 1) throw new RangeError("runAnalystBenchmark repetitions must be a positive safe integer");
+	if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1) throw new RangeError("runAnalystBenchmark maxConcurrency must be a positive safe integer");
+	if (!Number.isSafeInteger(options.runnerOrderSeed ?? 0)) throw new RangeError("runAnalystBenchmark runnerOrderSeed must be a safe integer");
+	assertUniqueNonEmpty(options.cases.map((testCase) => testCase.id), "case");
+	assertUniqueNonEmpty(options.runners.map((runner) => runner.id), "runner");
+	for (const testCase of options.cases) validateBenchmarkCase(testCase);
+}
+function validateBenchmarkCase(testCase) {
+	assertValidAnalystScoringCase(testCase);
+	if (!testCase.clusterId.trim()) throw new TypeError(`${testCase.id}: analyst benchmark clusterId must not be empty`);
+	if (testCase.labelState !== "positive" && testCase.labelState !== "trusted-negative" && testCase.labelState !== "unlabeled") throw new TypeError(`${testCase.id}: analyst benchmark labelState is invalid`);
+	if (testCase.labelState === "positive" && testCase.expectedIssues.length === 0) throw new TypeError(`${testCase.id}: positive case requires at least one expected issue`);
+	if (testCase.labelState !== "positive" && testCase.expectedIssues.length > 0) throw new TypeError(`${testCase.id}: ${testCase.labelState} case cannot contain expected issues`);
+}
+function assertUniqueNonEmpty(values, label) {
+	const seen = /* @__PURE__ */ new Set();
+	for (const value of values) {
+		if (!value.trim()) throw new TypeError(`analyst benchmark ${label} id must not be empty`);
+		if (seen.has(value)) throw new TypeError(`duplicate analyst benchmark ${label} id '${value}'`);
+		seen.add(value);
+	}
+}
+/** 32-bit FNV-1a over raw UTF-16 code units, left SIGNED so subtracting two
+*  results is a valid comparator.
+*
+*  Frozen: it fixes the runner order a benchmark reports. Adding the `>>> 0`
+*  the other variants carry would change that order, which is why this one
+*  does not have it. */
+function stableHash(value) {
+	let hash = 2166136261;
+	for (let index = 0; index < value.length; index += 1) {
+		hash ^= value.charCodeAt(index);
+		hash = Math.imul(hash, 16777619);
+	}
+	return hash >>> 0;
+}
+//#endregion
 //#region src/analyst/benchmark-dataset-utils.ts
 function normalizeBenchmarkLabel(value) {
 	const normalized = nonEmpty$1(value, "benchmark label").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "");
@@ -47,9 +544,6 @@ function assertStepWithinRange(step, stepCount, field) {
 function defaultStepUri(trajectoryId, step) {
 	return `trace://${encodeURIComponent(trajectoryId)}/span/step-${step}`;
 }
-function isRecord$2(value) {
-	return typeof value === "object" && value !== null && !Array.isArray(value);
-}
 function positiveStep(value, field) {
 	if (!Number.isSafeInteger(value) || value < 1) throw new RangeError(`${field} must be a positive safe integer`);
 	return value;
@@ -203,7 +697,7 @@ function parseAgentRxPredictions(output, trajectoryId) {
 	let failures;
 	let report;
 	if (Array.isArray(output)) failures = output;
-	else if (isRecord$2(output)) {
+	else if (isRecord(output)) {
 		assertMatchingAgentRxTaskId(output.task_id, trajectoryId, "report.task_id");
 		if (!Object.hasOwn(output, "failures")) throw new TypeError(`AgentRx prediction '${trajectoryId}' report must contain failures`);
 		failures = output.failures;
@@ -219,11 +713,11 @@ function parseAgentRxPredictions(output, trajectoryId) {
 	} else throw new TypeError(`AgentRx prediction '${trajectoryId}' must be a report or failures array`);
 	if (!Array.isArray(failures)) throw new TypeError(`AgentRx prediction '${trajectoryId}' failures must be an array`);
 	if (failures.length === 0) throw new TypeError(`AgentRx prediction '${trajectoryId}' failures must contain a judge prediction`);
-	if (isRecord$2(output) && output.num_judges !== void 0 && output.num_judges !== failures.length) throw new TypeError(`AgentRx prediction '${trajectoryId}' declares ${output.num_judges} judges but contains ${failures.length} failures`);
+	if (isRecord(output) && output.num_judges !== void 0 && output.num_judges !== failures.length) throw new TypeError(`AgentRx prediction '${trajectoryId}' declares ${output.num_judges} judges but contains ${failures.length} failures`);
 	return {
 		predictions: failures.map((value, index) => {
 			const field = `AgentRx prediction '${trajectoryId}' failures[${index}]`;
-			if (!isRecord$2(value)) throw new TypeError(`${field} must be an object`);
+			if (!isRecord(value)) throw new TypeError(`${field} must be an object`);
 			assertMatchingAgentRxTaskId(value.task_id, trajectoryId, `${field}.task_id`);
 			const failureCase = parseAgentRxFailureCase(value.failure_case, `${field}.failure_case`);
 			if (!Number.isSafeInteger(value.step_number)) throw new TypeError(`${field}.step_number must be a safe integer`);
@@ -437,7 +931,7 @@ function parseCodeTracerPredictionLabels(value, trajectoryId) {
 			flat.push(toCodeTracerPredictionLabel(item, codeTracerStageId(item, index + 1, trajectoryId), trajectoryId));
 			continue;
 		}
-		if (!isRecord$2(item) || !Array.isArray(item.labels)) throw new TypeError(`CodeTracer prediction '${trajectoryId}' contains an unsupported label row`);
+		if (!isRecord(item) || !Array.isArray(item.labels)) throw new TypeError(`CodeTracer prediction '${trajectoryId}' contains an unsupported label row`);
 		const stageId = codeTracerStageId(item, index + 1, trajectoryId);
 		for (const label of item.labels) {
 			if (!isCodeTracerStepLabel(label)) throw new TypeError(`CodeTracer prediction '${trajectoryId}' contains an invalid step label`);
@@ -447,10 +941,10 @@ function parseCodeTracerPredictionLabels(value, trajectoryId) {
 	return flat;
 }
 function isCodeTraceStageAnnotation(value) {
-	return isRecord$2(value) && Number.isSafeInteger(value.stage_id) && value.stage_id > 0 && optionalStepArray(value.incorrect_step_ids) && optionalStepArray(value.unuseful_step_ids) && (value.reasoning === void 0 || typeof value.reasoning === "string");
+	return isRecord(value) && Number.isSafeInteger(value.stage_id) && value.stage_id > 0 && optionalStepArray(value.incorrect_step_ids) && optionalStepArray(value.unuseful_step_ids) && (value.reasoning === void 0 || typeof value.reasoning === "string");
 }
 function isCodeTracerStepLabel(value) {
-	return isRecord$2(value) && Number.isSafeInteger(value.step_id) && value.step_id > 0 && (value.stage === void 0 || typeof value.stage === "string" || typeof value.stage === "number") && (value.stage_name === void 0 || typeof value.stage_name === "string" || typeof value.stage_name === "number") && (value.stage_id === void 0 || typeof value.stage_id === "string" || typeof value.stage_id === "number") && (value.label === "incorrect" || value.label === "unuseful") && (value.rationale === void 0 || typeof value.rationale === "string") && (value.reason === void 0 || typeof value.reason === "string") && (value.note === void 0 || typeof value.note === "string") && (value.comment === void 0 || typeof value.comment === "string");
+	return isRecord(value) && Number.isSafeInteger(value.step_id) && value.step_id > 0 && (value.stage === void 0 || typeof value.stage === "string" || typeof value.stage === "number") && (value.stage_name === void 0 || typeof value.stage_name === "string" || typeof value.stage_name === "number") && (value.stage_id === void 0 || typeof value.stage_id === "string" || typeof value.stage_id === "number") && (value.label === "incorrect" || value.label === "unuseful") && (value.rationale === void 0 || typeof value.rationale === "string") && (value.reason === void 0 || typeof value.reason === "string") && (value.note === void 0 || typeof value.note === "string") && (value.comment === void 0 || typeof value.comment === "string");
 }
 function toCodeTracerPredictionLabel(label, stageId, trajectoryId) {
 	if (!isCodeTracerStepLabel(label)) throw new TypeError(`CodeTracer prediction '${trajectoryId}' contains an invalid step label`);
@@ -728,8 +1222,8 @@ function compareMetric(options) {
 		if (caseBefore.length === 0 || !clusterId) continue;
 		pairedCases.push({
 			clusterId,
-			baseline: mean$2(caseBefore),
-			candidate: mean$2(caseAfter)
+			baseline: mean$3(caseBefore),
+			candidate: mean$3(caseAfter)
 		});
 	}
 	const byCluster = /* @__PURE__ */ new Map();
@@ -738,8 +1232,8 @@ function compareMetric(options) {
 		rows.push(pairedCase);
 		byCluster.set(pairedCase.clusterId, rows);
 	}
-	const before = [...byCluster.values()].map((rows) => mean$2(rows.map((row) => row.baseline)));
-	const after = [...byCluster.values()].map((rows) => mean$2(rows.map((row) => row.candidate)));
+	const before = [...byCluster.values()].map((rows) => mean$3(rows.map((row) => row.baseline)));
+	const after = [...byCluster.values()].map((rows) => mean$3(rows.map((row) => row.candidate)));
 	const interval = before.length === 0 ? null : pairedBootstrap(before, after, {
 		confidence: options.confidence,
 		resamples: options.resamples,
@@ -762,8 +1256,8 @@ function compareMetric(options) {
 		candidateMissingObservations,
 		asymmetricMissingObservations,
 		survivorOnly,
-		baselineMean: before.length === 0 ? null : mean$2(before),
-		candidateMean: after.length === 0 ? null : mean$2(after),
+		baselineMean: before.length === 0 ? null : mean$3(before),
+		candidateMean: after.length === 0 ? null : mean$3(after),
 		meanDelta: interval?.mean ?? null,
 		intervalLow: interval?.low ?? null,
 		intervalHigh: interval?.high ?? null,
@@ -1347,7 +1841,7 @@ const ANALYST_BENCHMARK_COST_LEDGER_FILE = "cost-ledger.jsonl";
 function observationKey(observation) {
 	return `${observation.runnerId}\u0000${observation.caseId}\u0000${observation.repetition}`;
 }
-function parseJson(text, source) {
+function parseJsonText(text, source) {
 	try {
 		return JSON.parse(text);
 	} catch {
@@ -1371,22 +1865,13 @@ function digestCanonical(value) {
 }
 /** RFC 8785 canonical JSON of the value's JSON document form — the byte form
 *  the receipt digests and compares against. */
-function canonicalJson(value) {
+function canonicalJsonDocument(value) {
 	return canonicalString(jsonDocument(value));
 }
-function isRecord$1(value) {
-	return typeof value === "object" && value !== null && !Array.isArray(value);
-}
 function isSha256(value) {
 	return typeof value === "string" && /^[a-f0-9]{64}$/.test(value);
 }
-//#endregion
-//#region src/analyst/benchmark-implementation.ts
-const ANALYST_BENCHMARK_IMPLEMENTATION_DIGEST_ALGORITHM = "sha256-canonical-source-manifest";
-/** The dependency files' manifest digest with the package's own version
-* fields blanked, so a release that only bumps the version keeps the pin. */
-const ANALYST_BENCHMARK_DEPENDENCY_LOCK_DIGEST_ALGORITHM = "sha256-canonical-file-manifest-without-own-version";
-const ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES = Object.freeze([
+Object.freeze([
 	"clients/python/pyproject.toml",
 	"clients/python/uv.lock",
 	"package.json",
@@ -1542,13 +2027,7 @@ const ANALYST_BENCHMARK_IMPLEMENTATION_FILES = Object.freeze([
 	"src/trace/redaction/detectors.ts",
 	"src/verdict-cache.ts"
 ]);
-const ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 = "c985d8e45f839a3fd38570ba30c0a874cd2af0aa577b3f190ec5e1785cbf5d3b";
-function analystBenchmarkImplementationDigest() {
-	return ANALYST_BENCHMARK_IMPLEMENTATION_SHA256;
-}
-function analystBenchmarkDependencyLockDigest() {
-	return ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256;
-}
+const ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 = "705fdcf5f1226d15fde853ffe5c8f8771fea0dfb0f3e66671748cff8aa70e8b5";
 //#endregion
 //#region src/analyst/benchmark-evidence-validation.ts
 const MIN_ACTION_EXCERPT_CHARACTERS = 12;
@@ -2356,17 +2835,6 @@ function isMissing(error) {
 function isNodeError$1(error, code) {
 	return error instanceof Error && "code" in error && error.code === code;
 }
-//#endregion
-//#region src/analyst/benchmark-public-prompt.ts
-/** Widest contiguous failure block a model may report. The published corpus's
-* widest labeled block is 8 steps and its widest stage span is 9, so this bound
-* never binds honest enumeration; it caps how far one over-wide block can push
-* unlabeled steps into the precision denominator. */
-const MAX_INCORRECT_BLOCK_STEPS = 12;
-/** Most blocks a model may report for one trajectory. The published corpus's
-* densest case carries 4 disjoint labeled blocks. Together with the per-block
-* cap this bounds one case at 192 predicted steps without a second ceiling. */
-const MAX_INCORRECT_BLOCKS = 16;
 const TRACE_PROJECTION_ATTRIBUTE_BYTE_CAPS = [
 	4096,
 	2048,
@@ -2738,8 +3206,8 @@ async function readAndValidateResumeFiles(paths, currentIdentity, currentIdentit
 	if (!await regularFileExists(paths.initializationComplete)) return initializeRunFiles(paths, currentIdentity, currentIdentitySha256, currentLocalIdentitySha256, localReceiptInput);
 	const manifest = await readAndValidateManifest(paths.manifest, currentIdentity, currentIdentitySha256, currentLocalIdentitySha256);
 	const localReceiptContent = await readRegularFile(paths.localReceipt, "benchmark local run receipt");
-	const value = parseJson(localReceiptContent, paths.localReceipt);
-	if (!isRecord$1(value)) throw new TypeError(`benchmark local run receipt must be an object: ${paths.localReceipt}`);
+	const value = parseJsonText(localReceiptContent, paths.localReceipt);
+	if (!isRecord(value)) throw new TypeError(`benchmark local run receipt must be an object: ${paths.localReceipt}`);
 	assertExactKeys(value, [
 		"kind",
 		"runIdentitySha256",
@@ -2749,13 +3217,13 @@ async function readAndValidateResumeFiles(paths, currentIdentity, currentIdentit
 		"environment",
 		"files"
 	], "benchmark local run receipt");
-	if (value.kind !== "agent-eval/analyst-benchmark-local-run" || value.runIdentitySha256 !== currentIdentitySha256 || value.localIdentitySha256 !== currentLocalIdentitySha256 || !isRecord$1(value.local)) throw new Error("benchmark local run receipt does not match the requested resume");
+	if (value.kind !== "agent-eval/analyst-benchmark-local-run" || value.runIdentitySha256 !== currentIdentitySha256 || value.localIdentitySha256 !== currentLocalIdentitySha256 || !isRecord(value.local)) throw new Error("benchmark local run receipt does not match the requested resume");
 	const expectedLocalReceipt = {
 		...localReceiptInput,
 		runIdentitySha256: currentIdentitySha256,
 		localIdentitySha256: currentLocalIdentitySha256
 	};
-	if (digestCanonical(value.local) !== currentLocalIdentitySha256 || canonicalJson(value.local) !== canonicalJson(localReceiptInput.local)) throw new Error("benchmark local paths or model-owner module do not match the requested resume");
+	if (digestCanonical(value.local) !== currentLocalIdentitySha256 || canonicalJsonDocument(value.local) !== canonicalJsonDocument(localReceiptInput.local)) throw new Error("benchmark local paths or model-owner module do not match the requested resume");
 	if (localReceiptContent !== `${JSON.stringify(expectedLocalReceipt, null, 2)}\n`) throw new Error(`benchmark local run receipt does not exactly match: ${paths.localReceipt}`);
 	if (await readRegularFile(paths.initializationComplete, "benchmark initialization marker") !== renderInitializationComplete(manifest)) throw new Error(`benchmark initialization marker does not match the run manifest: ${paths.initializationComplete}`);
 	return manifest;
@@ -2767,8 +3235,8 @@ async function readAndValidateManifest(path, currentIdentity, currentIdentitySha
 	return manifest;
 }
 async function readAndValidateManifestContent(path, content, currentIdentity, currentIdentitySha256, currentLocalIdentitySha256) {
-	const value = parseJson(content, path);
-	if (!isRecord$1(value)) throw new TypeError(`benchmark run manifest must be an object: ${path}`);
+	const value = parseJsonText(content, path);
+	if (!isRecord(value)) throw new TypeError(`benchmark run manifest must be an object: ${path}`);
 	assertExactKeys(value, [
 		"kind",
 		"createdAt",
@@ -2778,9 +3246,9 @@ async function readAndValidateManifestContent(path, content, currentIdentity, cu
 	], "benchmark run manifest");
 	if (value.kind !== "agent-eval/analyst-benchmark-run") throw new TypeError(`unsupported benchmark run manifest: ${path}`);
 	if (typeof value.createdAt !== "string" || !Number.isFinite(Date.parse(value.createdAt))) throw new TypeError(`benchmark run manifest has an invalid createdAt: ${path}`);
-	if (!isSha256(value.identitySha256) || !isSha256(value.localIdentitySha256) || !isRecord$1(value.identity)) throw new TypeError(`benchmark run manifest has an invalid identity: ${path}`);
+	if (!isSha256(value.identitySha256) || !isSha256(value.localIdentitySha256) || !isRecord(value.identity)) throw new TypeError(`benchmark run manifest has an invalid identity: ${path}`);
 	if (digestCanonical(value.identity) !== value.identitySha256) throw new Error(`benchmark run manifest identity digest does not match its contents: ${path}`);
-	if (currentIdentitySha256 !== value.identitySha256 || currentLocalIdentitySha256 !== value.localIdentitySha256 || canonicalJson(currentIdentity) !== canonicalJson(value.identity)) throw new Error(`benchmark resume configuration or inputs do not match ${ANALYST_BENCHMARK_MANIFEST_FILE}`);
+	if (currentIdentitySha256 !== value.identitySha256 || currentLocalIdentitySha256 !== value.localIdentitySha256 || canonicalJsonDocument(currentIdentity) !== canonicalJsonDocument(value.identity)) throw new Error(`benchmark resume configuration or inputs do not match ${ANALYST_BENCHMARK_MANIFEST_FILE}`);
 	return {
 		kind: "agent-eval/analyst-benchmark-run",
 		createdAt: value.createdAt,
@@ -2844,8 +3312,8 @@ async function readProgress(path, runIdentitySha256, caseIds, repetitions, analy
 	const plannedObservationCount = caseIds.length * 2 * repetitions;
 	for (const [index, line] of rawLines.entries()) {
 		if (!line.trim()) throw new Error(`benchmark observation log contains an empty row at line ${index + 1}`);
-		const parsed = parseJson(line, `${path}:${index + 1}`);
-		if (!isRecord$1(parsed)) throw new TypeError(`benchmark observation row ${index + 1} must be an object`);
+		const parsed = parseJsonText(line, `${path}:${index + 1}`);
+		if (!isRecord(parsed)) throw new TypeError(`benchmark observation row ${index + 1} must be an object`);
 		assertExactKeys(parsed, [
 			"sequence",
 			"runIdentitySha256",
@@ -3045,7 +3513,7 @@ function escapeCell$1(value) {
 //#endregion
 //#region src/analyst/benchmark-command-result.ts
 async function readAnalystBenchmarkArtifact(path) {
-	const value = parseJson(await readRegularFile(path, "analyst benchmark result"), path);
+	const value = parseJsonText(await readRegularFile(path, "analyst benchmark result"), path);
 	assertAnalystBenchmarkArtifact(value, "analyst benchmark result");
 	return value;
 }
@@ -3103,33 +3571,33 @@ function assertCompletedArtifactMatchesRun(artifact, manifest, observations, pre
 			dependencyLockSha256: config.dependencyLockSha256
 		}
 	};
-	if (canonicalJson(artifact.inputs) !== canonicalJson(expectedInputs)) throw new Error("completed benchmark result inputs do not match the run manifest");
+	if (canonicalJsonDocument(artifact.inputs) !== canonicalJsonDocument(expectedInputs)) throw new Error("completed benchmark result inputs do not match the run manifest");
 	const provenance = artifact.result.provenance;
 	const expectedDatasetId = config.dataset === "agentrx" ? "microsoft/AgentRx" : "NJU-LINK/CodeTraceBench";
 	const expectedOutputAdapter = config.dataset === "agentrx" ? "agentrx-taxonomy-and-root-step" : "codetracebench-incorrect-block";
-	if (provenance.id !== `${config.dataset}-real-model-analyst` || provenance.startedAt !== manifest.createdAt || !Number.isFinite(Date.parse(provenance.endedAt)) || Date.parse(provenance.endedAt) < Date.parse(provenance.startedAt) || canonicalJson(provenance.dataset) !== canonicalJson({
+	if (provenance.id !== `${config.dataset}-real-model-analyst` || provenance.startedAt !== manifest.createdAt || !Number.isFinite(Date.parse(provenance.endedAt)) || Date.parse(provenance.endedAt) < Date.parse(provenance.startedAt) || canonicalJsonDocument(provenance.dataset) !== canonicalJsonDocument({
 		id: expectedDatasetId,
 		revision: config.datasetRevision,
 		split: config.datasetSplit
-	}) || provenance.caseCount !== inputs.selectedCaseIds.length || canonicalJson(provenance.runnerIds) !== canonicalJson(config.runnerIds) || provenance.repetitions !== config.repetitions || provenance.maxConcurrency !== Math.min(config.concurrency, expectedCount) || provenance.runnerOrderSeed !== config.seed || provenance.metadata?.model !== config.model.id || provenance.metadata?.modelOwnerCallRef !== config.model.ownerCallRef || provenance.metadata?.rlmSamples !== config.rlmSamples || provenance.metadata?.outputAdapter !== expectedOutputAdapter || provenance.metadata?.caseSelection !== prepared.selection.method || provenance.metadata?.caseSelectionSeed !== config.seed || provenance.metadata?.selectionStratified !== prepared.selection.stratified || provenance.metadata?.protocolSha256 !== config.analystProtocolSha256 || provenance.metadata?.implementationSha256 !== config.implementationSha256 || provenance.metadata?.dependencyLockSha256 !== config.dependencyLockSha256 || provenance.metadata?.populationRepresentativenessProven !== false) throw new Error("completed benchmark result provenance does not match the run manifest");
+	}) || provenance.caseCount !== inputs.selectedCaseIds.length || canonicalJsonDocument(provenance.runnerIds) !== canonicalJsonDocument(config.runnerIds) || provenance.repetitions !== config.repetitions || provenance.maxConcurrency !== Math.min(config.concurrency, expectedCount) || provenance.runnerOrderSeed !== config.seed || provenance.metadata?.model !== config.model.id || provenance.metadata?.modelOwnerCallRef !== config.model.ownerCallRef || provenance.metadata?.rlmSamples !== config.rlmSamples || provenance.metadata?.outputAdapter !== expectedOutputAdapter || provenance.metadata?.caseSelection !== prepared.selection.method || provenance.metadata?.caseSelectionSeed !== config.seed || provenance.metadata?.selectionStratified !== prepared.selection.stratified || provenance.metadata?.protocolSha256 !== config.analystProtocolSha256 || provenance.metadata?.implementationSha256 !== config.implementationSha256 || provenance.metadata?.dependencyLockSha256 !== config.dependencyLockSha256 || provenance.metadata?.populationRepresentativenessProven !== false) throw new Error("completed benchmark result provenance does not match the run manifest");
 	const expectedSummaries = config.runnerIds.map((runnerId) => summarizeAnalystBenchmarkRunner(runnerId, observations.filter((observation) => observation.runnerId === runnerId)));
-	if (canonicalJson(artifact.result.summaries) !== canonicalJson(expectedSummaries)) throw new Error("completed benchmark summaries do not match durable observations");
+	if (canonicalJsonDocument(artifact.result.summaries) !== canonicalJsonDocument(expectedSummaries)) throw new Error("completed benchmark summaries do not match durable observations");
 	const expectedComparisons = [compareAnalystRunners(artifact.result, {
 		baselineRunnerId: "empty",
 		candidateRunnerId: config.runnerIds[1],
 		seed: config.seed
 	})];
-	if (canonicalJson(artifact.comparisons) !== canonicalJson(expectedComparisons)) throw new Error("completed benchmark comparisons do not match durable observations");
+	if (canonicalJsonDocument(artifact.comparisons) !== canonicalJsonDocument(expectedComparisons)) throw new Error("completed benchmark comparisons do not match durable observations");
 	if (config.dataset === "codetracebench") {
-		if (canonicalJson(artifact.codeTraceCalibration) !== canonicalJson(summarizeCodeTraceCalibration(artifact.result)) || artifact.agentRxCalibration !== void 0) throw new Error("completed CodeTraceBench calibration does not match durable observations");
-	} else if (canonicalJson(artifact.agentRxCalibration) !== canonicalJson(summarizeAgentRxCalibration(artifact.result, "f228165bfec60a801fd5fedd9d8ffe0f9de0c69d")) || artifact.codeTraceCalibration !== void 0) throw new Error("completed AgentRx calibration does not match durable observations");
+		if (canonicalJsonDocument(artifact.codeTraceCalibration) !== canonicalJsonDocument(summarizeCodeTraceCalibration(artifact.result)) || artifact.agentRxCalibration !== void 0) throw new Error("completed CodeTraceBench calibration does not match durable observations");
+	} else if (canonicalJsonDocument(artifact.agentRxCalibration) !== canonicalJsonDocument(summarizeAgentRxCalibration(artifact.result, "f228165bfec60a801fd5fedd9d8ffe0f9de0c69d")) || artifact.codeTraceCalibration !== void 0) throw new Error("completed AgentRx calibration does not match durable observations");
 }
 function assertSameObservations(expected, actual) {
 	if (expected.length !== actual.length) throw new Error(`benchmark result has ${expected.length} observations but the durable log has ${actual.length}`);
-	const expectedByKey = new Map(expected.map((observation) => [observationKey(observation), canonicalJson(observation)]));
+	const expectedByKey = new Map(expected.map((observation) => [observationKey(observation), canonicalJsonDocument(observation)]));
 	for (const observation of actual) {
 		const key = observationKey(observation);
-		if (expectedByKey.get(key) !== canonicalJson(observation)) throw new Error(`benchmark result does not match durable observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}'`);
+		if (expectedByKey.get(key) !== canonicalJsonDocument(observation)) throw new Error(`benchmark result does not match durable observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}'`);
 		expectedByKey.delete(key);
 	}
 	if (expectedByKey.size > 0) throw new Error("benchmark result is missing durable observations");
@@ -3475,9 +3943,6 @@ function safeInteger(value, field) {
 	if (!Number.isSafeInteger(value)) throw new RangeError(`${field} must be a safe integer`);
 	return value;
 }
-function isRecord(value) {
-	return typeof value === "object" && value !== null && !Array.isArray(value);
-}
 //#endregion
 //#region src/analyst/benchmark-response-cache.ts
 const SHA256 = /^[a-f0-9]{64}$/;
@@ -6324,6 +6789,6 @@ function shellQuote(value) {
 	return /^[a-zA-Z0-9_./:@%+=,-]+$/.test(value) ? value : `'${value.replaceAll("'", "'\\''")}'`;
 }
 //#endregion
-export { ANALYST_BENCHMARK_MANIFEST_FILE as $, readAnalystInstructionsOverride as A, parseVerificationOutcome as B, emptyPublicBenchmarkRunner as C, summarizeCodeTraceCalibration as D, renderCodeTraceCalibrationMarkdown as E, publicBenchmarkRlmInstructions as F, ANALYST_BENCHMARK_EVIDENCE_IMPLEMENTATION_SHA256 as G, ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES as H, publicBenchmarkSystemPrompt as I, ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 as J, ANALYST_BENCHMARK_IMPLEMENTATION_DIGEST_ALGORITHM as K, DEFAULT_MAX_VERIFICATION_ARTIFACT_BYTES as L, MAX_INCORRECT_BLOCKS as M, MAX_INCORRECT_BLOCK_STEPS as N, analystInstructionsOverrideFromText as O, publicBenchmarkProtocolSha256 as P, ANALYST_BENCHMARK_LOCAL_RECEIPT_FILE as Q, appendVerificationArtifactsToOtlp as R, adaptPublicBenchmarkFindings as S, readAnalystBenchmarkArtifact as T, ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 as U, ANALYST_BENCHMARK_DEPENDENCY_LOCK_DIGEST_ALGORITHM as V, ANALYST_BENCHMARK_EVIDENCE_DEPENDENCY_LOCK_SHA256 as W, analystBenchmarkImplementationDigest as X, analystBenchmarkDependencyLockDigest as Y, ANALYST_BENCHMARK_COST_LEDGER_FILE as Z, publicDirectAnalystDefinition as _, primeCodeTraceAnalystDefinition as a, codeTraceBenchCase as at, analystDefinitionAsymmetries as b, preparePublicAnalystBenchmark as c, agentRxPredictionsToFindings as ct, selectPublicBenchmarkRows as d, normalizeBenchmarkLabel as dt, ANALYST_BENCHMARK_OBSERVATIONS_FILE as et, createPublicBenchmarkRlmRunner as f, createPublicBenchmarkDirectRunner as g, runReplVariableAnalystDefinition as h, primeAnalystProtocolSha256 as i, summarizeAgentRxCalibration as it, CODE_TRACE_BENCH_ANALYST_PROMPT as j, effectiveAnalystProtocolSha256 as k, publicBenchmarkDistributions as l, normalizeAgentRxCategory as lt, rlmEngineLimits as m, renderAnalystBenchmarkMarkdown as n, AGENT_RX_UPSTREAM_REVISION as nt, runInlineAnalystDefinition as o, codeTracerPredictionsToFindings as ot, publicRlmAnalystDefinition as p, ANALYST_BENCHMARK_IMPLEMENTATION_FILES as q, createPrimeBenchmarkRunner as r, renderAgentRxCalibrationMarkdown as rt, loadPublicBenchmarkRows as s, agentRxBenchmarkCase as st, runAnalystBenchmarkCommand as t, compareAnalystRunners as tt, publicBenchmarkSelectionReport as u, roundAgentRxStep as ut, runChunkedAnalystDefinition as v, expandCodeTraceFailureBlocks as w, analystDefinitionProtocolSha256 as x, AnalystExpressivenessError as y, loadCodeTraceVerificationArtifacts as z };
+export { agentRxBenchmarkCase as A, ANALYST_BENCHMARK_COST_LEDGER_FILE as C, AGENT_RX_UPSTREAM_REVISION as D, ANALYST_BENCHMARK_OBSERVATIONS_FILE as E, normalizeAgentRxCategory as M, runAnalystBenchmark as N, codeTraceBenchCase as O, traceStoreEvidenceResolver as P, ANALYST_BENCHMARK_IMPLEMENTATION_FILES as S, ANALYST_BENCHMARK_MANIFEST_FILE as T, readAnalystBenchmarkArtifact as _, runInlineAnalystDefinition as a, ANALYST_BENCHMARK_EVIDENCE_DEPENDENCY_LOCK_SHA256 as b, publicRlmAnalystDefinition as c, publicDirectAnalystDefinition as d, runChunkedAnalystDefinition as f, expandCodeTraceFailureBlocks as g, analystDefinitionProtocolSha256 as h, primeCodeTraceAnalystDefinition as i, agentRxPredictionsToFindings as j, codeTracerPredictionsToFindings as k, runReplVariableAnalystDefinition as l, analystDefinitionAsymmetries as m, createPrimeBenchmarkRunner as n, loadPublicBenchmarkRows as o, AnalystExpressivenessError as p, primeAnalystProtocolSha256 as r, createPublicBenchmarkRlmRunner as s, runAnalystBenchmarkCommand as t, createPublicBenchmarkDirectRunner as u, CODE_TRACE_BENCH_ANALYST_PROMPT as v, ANALYST_BENCHMARK_LOCAL_RECEIPT_FILE as w, ANALYST_BENCHMARK_EVIDENCE_IMPLEMENTATION_SHA256 as x, publicBenchmarkProtocolSha256 as y };
 
-//# sourceMappingURL=benchmark-command-By0vh8q7.js.map
+//# sourceMappingURL=benchmark-command-xQK8nOI6.js.map