@tangle-network/agent-eval 0.137.0 → 0.138.0

package/dist/benchmark-command-CMqVqReF.js

@@ -0,0 +1,4332 @@
+import { c as ValidationError, t as AgentEvalError } from "./errors-D-LKuDhb.js";
+import { a as CostLedgerPersistenceError, i as CostLedger, n as CostCallConflictError, o as CostReceiptCaptureError, r as CostCeilingReachedError, s as CostReservationExceededError, t as CostAccountingIncompleteError } from "./cost-ledger-CHDLA0Ss.js";
+import { c as callLlmJson, f as maximumChargeForLlmRequest, l as costReceiptFromLlm, r as LlmResponseError, t as LlmCallError, u as costReceiptFromLlmError } from "./llm-client-Cj3c7PEm.js";
+import { d as TRACE_ANALYSIS_LIMITS, i as otlpTextToTraceAnalysisStore, r as createOtlpBufferTraceStore, t as DEFAULT_MAX_TRACE_FILE_BYTES } from "./store-otlp-BenKynPE.js";
+import { d as usageReceiptFromCostLedger, m as makeFinding, n as createRunCostLedger, r as fsCampaignStorage, t as acquireSingleRunLock } from "./single-run-lock-D_bS5xhj.js";
+import { _ as canonicalString, c as writeLedgerFileAtomically, s as withLedgerFileLock, v as hashCanonical } from "./ledger-core-C0Yx1I14.js";
+import { E as pairedBootstrap } from "./statistics-ByxzSiOM.js";
+import { i as summarizeAnalystBenchmarkRunner, n as runAnalystBenchmark, r as traceStoreEvidenceResolver } from "./benchmark-D8dkki-J.js";
+import { z } from "zod";
+import { constants, existsSync, lstatSync, readFileSync } from "node:fs";
+import * as nodePath from "node:path";
+import { dirname, isAbsolute, relative, resolve, sep } from "node:path";
+import { createHash, randomUUID } from "node:crypto";
+import { link, lstat, mkdir, open, readFile, readdir, realpath, stat, unlink } from "node:fs/promises";
+import { arch, platform } from "node:os";
+import { TextDecoder as TextDecoder$1 } from "node:util";
+//#region src/analyst/benchmark-dataset-utils.ts
+function normalizeBenchmarkLabel(value) {
+	const normalized = nonEmpty$1(value, "benchmark label").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "");
+	if (!normalized) throw new TypeError("benchmark label must contain letters or digits");
+	return normalized;
+}
+function predictionConfidence(value) {
+	const confidence = value ?? .5;
+	if (!Number.isFinite(confidence) || confidence < 0 || confidence > 1) throw new RangeError("upstream prediction confidence must be between 0 and 1");
+	return confidence;
+}
+function assertStepWithinRange(step, stepCount, field) {
+	if (stepCount === void 0) return;
+	const count = positiveStep(stepCount, `${field} stepCount`);
+	if (step > count) throw new RangeError(`${field} step ${step} exceeds stepCount ${count}`);
+}
+function defaultStepUri(trajectoryId, step) {
+	return `trace://${encodeURIComponent(trajectoryId)}/span/step-${step}`;
+}
+function isRecord$2(value) {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function positiveStep(value, field) {
+	if (!Number.isSafeInteger(value) || value < 1) throw new RangeError(`${field} must be a positive safe integer`);
+	return value;
+}
+function externalId(value, field) {
+	if (typeof value !== "string" && typeof value !== "number") throw new TypeError(`${field} must be a string or number`);
+	if (typeof value === "number" && !Number.isSafeInteger(value)) throw new TypeError(`${field} must be a safe integer when numeric`);
+	return nonEmpty$1(String(value), field);
+}
+function nonEmpty$1(value, field) {
+	if (!value.trim()) throw new TypeError(`${field} must not be empty`);
+	return value;
+}
+//#endregion
+//#region src/analyst/benchmark-dataset-agentrx.ts
+function agentRxBenchmarkCase(row, input, options = {}) {
+	const trajectoryId = externalId(row.trajectory_id, "AgentRx trajectory_id");
+	if (!Array.isArray(row.failures) || row.failures.length === 0) throw new TypeError(`AgentRx trajectory '${trajectoryId}' must contain failures`);
+	if (row.num_failures !== void 0 && row.num_failures !== row.failures.length) throw new TypeError(`AgentRx trajectory '${trajectoryId}' declares ${row.num_failures} failures but contains ${row.failures.length}`);
+	const rootCauseId = externalId(row.root_cause_failure_id ?? row.root_cause?.failure_id, `AgentRx trajectory '${trajectoryId}' root cause failure id`);
+	const failureIds = /* @__PURE__ */ new Set();
+	const failureMetadata = [];
+	const evidenceKind = options.evidenceKind ?? "span";
+	const uri = options.stepUri ?? defaultStepUri;
+	const allIssues = row.failures.map((failure) => {
+		const failureId = externalId(failure.failure_id, `AgentRx trajectory '${trajectoryId}' failure id`);
+		if (failureIds.has(failureId)) throw new TypeError(`AgentRx trajectory '${trajectoryId}' repeats failure id '${failureId}'`);
+		failureIds.add(failureId);
+		const step = positiveStep(failure.step_number, `AgentRx trajectory '${trajectoryId}'`);
+		const evidence = [{
+			kind: evidenceKind,
+			uri: uri(trajectoryId, step)
+		}];
+		if (typeof failure.failure_category !== "string") throw new TypeError(`AgentRx trajectory '${trajectoryId}' failure '${failureId}' category must be a string`);
+		const category = normalizeAgentRxCategory(failure.failure_category);
+		if (!AGENT_RX_TAXONOMY_BY_LABEL.has(category)) throw new RangeError(`AgentRx trajectory '${trajectoryId}' failure '${failureId}' category '${failure.failure_category}' is outside the AgentRx taxonomy`);
+		failureMetadata.push({
+			id: failureId,
+			step,
+			category
+		});
+		return {
+			id: failureId,
+			areas: [category],
+			...failureId === rootCauseId && (options.target ?? "root-cause") === "root-cause" ? {} : { evidence },
+			criticalEvidence: failureId === rootCauseId ? evidence : void 0
+		};
+	});
+	if (!failureIds.has(rootCauseId)) throw new TypeError(`AgentRx trajectory '${trajectoryId}' root cause '${rootCauseId}' is not in failures`);
+	if (options.stepCount !== void 0 && row.failures.some((failure) => failure.step_number > options.stepCount)) throw new RangeError(`AgentRx trajectory '${trajectoryId}' contains a failure beyond stepCount ${options.stepCount}`);
+	const expectedIssues = (options.target ?? "root-cause") === "root-cause" ? allIssues.filter((issue) => issue.id === rootCauseId) : allIssues;
+	const rootCause = failureMetadata.find((failure) => failure.id === rootCauseId);
+	const orderedFailures = [...failureMetadata].sort((left, right) => left.step - right.step || left.id.localeCompare(right.id));
+	return {
+		id: `agentrx:${trajectoryId}`,
+		clusterId: `agentrx:${trajectoryId}`,
+		labelState: "positive",
+		input,
+		expectedIssues,
+		labeledEvidence: expectedIssues.flatMap((issue) => issue.evidence ?? issue.criticalEvidence ?? []),
+		tags: ["agentrx"],
+		metadata: {
+			benchmark: "AgentRx",
+			trajectoryId,
+			failureSummary: row.failure_summary,
+			rootCauseReason: row.root_cause_reason ?? row.root_cause?.reason_for_root_cause,
+			annotatedFailures: row.failures.length,
+			target: options.target ?? "root-cause",
+			rootCauseStep: rootCause.step,
+			rootCauseCategory: rootCause.category,
+			allFailureCategories: [...new Set(failureMetadata.map((failure) => failure.category))].sort(),
+			earliestFailureCategory: orderedFailures[0].category,
+			terminalFailureCategory: orderedFailures.at(-1).category,
+			...options.stepCount === void 0 ? {} : { trajectoryLength: options.stepCount }
+		}
+	};
+}
+/** Translate AgentRx `Report.to_dict()` output or its `failures` array into findings. */
+function agentRxPredictionsToFindings(trajectoryIdValue, output, options = {}) {
+	const trajectoryId = externalId(trajectoryIdValue, "AgentRx prediction trajectory id");
+	const parsed = parseAgentRxPredictions(output, trajectoryId);
+	for (const prediction of parsed.predictions) {
+		assertStepWithinRange(prediction.step_number, parsed.report?.trajectory_length, `AgentRx prediction '${trajectoryId}' report`);
+		assertStepWithinRange(prediction.step_number, options.stepCount, `AgentRx prediction '${trajectoryId}'`);
+	}
+	const consensus = agentRxConsensus(parsed, trajectoryId);
+	if (consensus.failureCase === 0) return [];
+	const confidence = predictionConfidence(options.confidence);
+	const uri = options.stepUri ?? defaultStepUri;
+	assertStepWithinRange(consensus.step, options.stepCount, `AgentRx prediction '${trajectoryId}'`);
+	const area = AGENT_RX_TAXONOMY.get(consensus.failureCase);
+	return [makeFinding({
+		analyst_id: options.analystId ?? "agentrx",
+		produced_at: options.producedAt,
+		area,
+		subject: "root-cause",
+		claim: `AgentRx classified step ${consensus.step} as ${area}.`,
+		id_basis: `${area}:${consensus.step}`,
+		rationale: consensus.representative.description,
+		severity: "high",
+		confidence,
+		evidence_refs: [{
+			kind: options.evidenceKind ?? "span",
+			uri: uri(trajectoryId, consensus.step)
+		}],
+		metadata: {
+			upstream: "AgentRx",
+			failure_case: consensus.failureCase,
+			step: consensus.step,
+			step_mean: consensus.stepMean,
+			judge_votes: parsed.predictions.length,
+			consensus_votes: consensus.votes,
+			category_agreement: consensus.votes / parsed.predictions.length,
+			...consensus.representative.checklist_reasoning === void 0 || consensus.representative.checklist_reasoning === null ? {} : { checklist_reasoning: consensus.representative.checklist_reasoning }
+		}
+	})];
+}
+const AGENT_RX_TAXONOMY = /* @__PURE__ */ new Map([
+	[1, "instruction-plan-adherence-failure"],
+	[2, "invention-of-new-information"],
+	[3, "invalid-invocation"],
+	[4, "misinterpretation-of-tool-output-handoff-failure"],
+	[5, "intent-plan-misalignment"],
+	[6, "underspecified-user-intent"],
+	[7, "intent-not-supported"],
+	[8, "guardrails-triggered"],
+	[9, "system-failure"],
+	[10, "inconclusive"]
+]);
+const AGENT_RX_CATEGORY_ALIASES = /* @__PURE__ */ new Map([["instruction-adherence-failure", "instruction-plan-adherence-failure"], ["misinterpretation-of-tool-output", "misinterpretation-of-tool-output-handoff-failure"]]);
+const AGENT_RX_TAXONOMY_BY_LABEL = new Map([...AGENT_RX_TAXONOMY].map(([failureCase, label]) => [label, failureCase]));
+function normalizeAgentRxCategory(value) {
+	const normalized = normalizeBenchmarkLabel(value);
+	return AGENT_RX_CATEGORY_ALIASES.get(normalized) ?? normalized;
+}
+function parseAgentRxFailureCase(value, field) {
+	if (typeof value !== "number" && typeof value !== "string") throw new TypeError(`${field} must be a taxonomy number or label`);
+	if (typeof value === "string" && !/^\d+$/.test(value.trim())) {
+		const normalized = normalizeAgentRxCategory(value);
+		const failureCase = AGENT_RX_TAXONOMY_BY_LABEL.get(normalized);
+		if (failureCase === void 0) throw new RangeError(`${field} '${value}' is not an AgentRx taxonomy label`);
+		return failureCase;
+	}
+	const numeric = typeof value === "number" ? value : Number(value);
+	if (!Number.isSafeInteger(numeric)) throw new TypeError(`${field} must be a taxonomy number or label`);
+	if (numeric < 0 || numeric > 10) throw new RangeError(`${field} ${numeric} is outside 0-10`);
+	return numeric;
+}
+function parseAgentRxPredictions(output, trajectoryId) {
+	let failures;
+	let report;
+	if (Array.isArray(output)) failures = output;
+	else if (isRecord$2(output)) {
+		assertMatchingAgentRxTaskId(output.task_id, trajectoryId, "report.task_id");
+		if (!Object.hasOwn(output, "failures")) throw new TypeError(`AgentRx prediction '${trajectoryId}' report must contain failures`);
+		failures = output.failures;
+		if (output.num_judges !== void 0) {
+			if (!Number.isSafeInteger(output.num_judges) || output.num_judges < 0) throw new RangeError(`AgentRx prediction '${trajectoryId}' report.num_judges must be a non-negative safe integer`);
+		}
+		if (output.trajectory_length !== void 0) positiveStep(output.trajectory_length, `AgentRx prediction '${trajectoryId}' report.trajectory_length`);
+		if (output.step_mean !== void 0) {
+			if (typeof output.step_mean !== "number" || !Number.isFinite(output.step_mean)) throw new TypeError(`AgentRx prediction '${trajectoryId}' report.step_mean must be finite`);
+		}
+		if (output.modes !== void 0 && !Array.isArray(output.modes)) throw new TypeError(`AgentRx prediction '${trajectoryId}' report.modes must be an array`);
+		report = output;
+	} else throw new TypeError(`AgentRx prediction '${trajectoryId}' must be a report or failures array`);
+	if (!Array.isArray(failures)) throw new TypeError(`AgentRx prediction '${trajectoryId}' failures must be an array`);
+	if (failures.length === 0) throw new TypeError(`AgentRx prediction '${trajectoryId}' failures must contain a judge prediction`);
+	if (isRecord$2(output) && output.num_judges !== void 0 && output.num_judges !== failures.length) throw new TypeError(`AgentRx prediction '${trajectoryId}' declares ${output.num_judges} judges but contains ${failures.length} failures`);
+	return {
+		predictions: failures.map((value, index) => {
+			const field = `AgentRx prediction '${trajectoryId}' failures[${index}]`;
+			if (!isRecord$2(value)) throw new TypeError(`${field} must be an object`);
+			assertMatchingAgentRxTaskId(value.task_id, trajectoryId, `${field}.task_id`);
+			const failureCase = parseAgentRxFailureCase(value.failure_case, `${field}.failure_case`);
+			if (!Number.isSafeInteger(value.step_number)) throw new TypeError(`${field}.step_number must be a safe integer`);
+			const stepNumber = value.step_number;
+			if (failureCase === 0 ? stepNumber !== 0 : stepNumber < 1) throw new RangeError(failureCase === 0 ? `${field}.step_number must be 0 when failure_case is 0` : `${field}.step_number must be positive when failure_case is 1-10`);
+			if (value.description !== void 0 && typeof value.description !== "string") throw new TypeError(`${field}.description must be a string`);
+			if (value.checklist_reasoning !== void 0 && value.checklist_reasoning !== null && typeof value.checklist_reasoning !== "string") throw new TypeError(`${field}.checklist_reasoning must be a string or null`);
+			return {
+				...value.task_id === void 0 ? {} : { task_id: value.task_id },
+				failure_case: failureCase,
+				step_number: stepNumber,
+				...value.description === void 0 ? {} : { description: value.description },
+				...value.checklist_reasoning === void 0 ? {} : { checklist_reasoning: value.checklist_reasoning }
+			};
+		}),
+		report
+	};
+}
+function agentRxConsensus(parsed, trajectoryId) {
+	const counts = /* @__PURE__ */ new Map();
+	for (const prediction of parsed.predictions) counts.set(prediction.failure_case, (counts.get(prediction.failure_case) ?? 0) + 1);
+	const maxVotes = Math.max(...counts.values());
+	let failureCase = [...counts].find(([, count]) => count === maxVotes)[0];
+	if (parsed.report?.most_common_failure !== void 0) {
+		const declared = parseAgentRxFailureCase(parsed.report.most_common_failure, `AgentRx prediction '${trajectoryId}' report.most_common_failure`);
+		if ((counts.get(declared) ?? 0) !== maxVotes) throw new TypeError(`AgentRx prediction '${trajectoryId}' report.most_common_failure disagrees with failures`);
+		failureCase = declared;
+	}
+	if (parsed.report?.modes !== void 0) {
+		const declaredModes = parsed.report.modes.map((value, index) => parseAgentRxFailureCase(value, `AgentRx prediction '${trajectoryId}' report.modes[${index}]`));
+		if (new Set(declaredModes).size !== declaredModes.length) throw new TypeError(`AgentRx prediction '${trajectoryId}' report.modes contains duplicates`);
+		const expectedModes = [...counts].filter(([, count]) => count === maxVotes).map(([value]) => value).sort((left, right) => left - right);
+		if ([...new Set(declaredModes)].sort((left, right) => left - right).join(",") !== expectedModes.join(",")) throw new TypeError(`AgentRx prediction '${trajectoryId}' report.modes disagrees with failures`);
+	}
+	const computedStepMean = parsed.predictions.reduce((sum, prediction) => sum + prediction.step_number, 0) / parsed.predictions.length;
+	if (parsed.report?.step_mean !== void 0 && Math.abs(parsed.report.step_mean - computedStepMean) > 1e-12) throw new TypeError(`AgentRx prediction '${trajectoryId}' report.step_mean disagrees with failures`);
+	const stepMean = parsed.report?.step_mean ?? computedStepMean;
+	const step = failureCase === 0 ? 0 : positiveStep(roundAgentRxStep(stepMean), `AgentRx prediction '${trajectoryId}' consensus step`);
+	const representative = parsed.predictions.filter((prediction) => prediction.failure_case === failureCase).sort((left, right) => Math.abs(left.step_number - step) - Math.abs(right.step_number - step))[0] ?? parsed.predictions[0];
+	return {
+		failureCase,
+		step,
+		stepMean,
+		votes: counts.get(failureCase),
+		representative
+	};
+}
+/** Match Python's round() behavior used by AgentRx for consensus steps. */
+function roundAgentRxStep(value) {
+	if (!Number.isFinite(value)) throw new TypeError("AgentRx step mean must be finite");
+	const lower = Math.floor(value);
+	const fraction = value - lower;
+	if (Math.abs(fraction - .5) <= Number.EPSILON * Math.max(1, Math.abs(value))) return lower % 2 === 0 ? lower : lower + 1;
+	return Math.round(value);
+}
+function assertMatchingAgentRxTaskId(value, trajectoryId, field) {
+	if (value === void 0) return;
+	const taskId = externalId(value, `AgentRx prediction '${trajectoryId}' ${field}`);
+	if (taskId !== trajectoryId) throw new TypeError(`AgentRx prediction '${trajectoryId}' ${field} '${taskId}' does not match trajectory id`);
+}
+//#endregion
+//#region src/analyst/benchmark-dataset-codetrace.ts
+function codeTraceBenchCase(row, input, options = {}) {
+	const trajectoryId = requiredCodeTraceString(row.traj_id, "CodeTraceBench traj_id");
+	const taskName = requiredCodeTraceString(row.task_name, `CodeTraceBench '${trajectoryId}' task_name`);
+	const agent = requiredCodeTraceString(row.agent, `CodeTraceBench '${trajectoryId}' agent`);
+	const model = requiredCodeTraceString(row.model, `CodeTraceBench '${trajectoryId}' model`);
+	const difficulty = optionalCodeTraceString(row.difficulty, `CodeTraceBench '${trajectoryId}' difficulty`);
+	const category = optionalCodeTraceString(row.category, `CodeTraceBench '${trajectoryId}' category`);
+	const sourceRelpath = optionalSourceRelativePath(row.source_relpath, trajectoryId);
+	const solved = codeTraceSolved(row.solved, trajectoryId);
+	const tags = parseTags(row.tags, trajectoryId);
+	const stepCount = positiveStep(row.step_count, `CodeTraceBench '${trajectoryId}' step_count`);
+	const stages = parseCodeTraceStages(row.incorrect_stages, trajectoryId);
+	const evidenceKind = options.evidenceKind ?? "span";
+	const uri = options.stepUri ?? defaultStepUri;
+	const labelSet = codeTraceLabelSet(options.labelSet);
+	const labels = /* @__PURE__ */ new Set();
+	const expectedIssues = stages.flatMap((stage) => {
+		const incorrect = stepIssues("incorrect", stage.incorrect_step_ids ?? []);
+		const unuseful = stepIssues("unuseful", stage.unuseful_step_ids ?? []);
+		return labelSet === "incorrect-only" ? incorrect : [...incorrect, ...unuseful];
+	});
+	const labelState = expectedIssues.length > 0 ? "positive" : solved === true ? "trusted-negative" : "unlabeled";
+	return {
+		id: `codetrace:${trajectoryId}`,
+		clusterId: `codetrace-task:${taskName}`,
+		labelState,
+		input,
+		expectedIssues,
+		...labelState === "unlabeled" ? {} : { labeledEvidence: expectedIssues.flatMap((issue) => issue.evidence ?? []) },
+		tags: [
+			"codetracebench",
+			agent,
+			model,
+			...difficulty === void 0 ? [] : [difficulty],
+			...category === void 0 ? [] : [category],
+			...tags
+		],
+		metadata: {
+			benchmark: "CodeTraceBench",
+			trajectoryId,
+			taskName,
+			agent,
+			model,
+			solved,
+			stepCount,
+			labelSet,
+			...sourceRelpath === void 0 ? {} : { sourceRelpath },
+			...difficulty === void 0 ? {} : { difficulty },
+			...category === void 0 ? {} : { category }
+		}
+	};
+	function stepIssues(label, steps) {
+		return steps.map((rawStep) => {
+			const step = positiveStep(rawStep, `CodeTraceBench '${trajectoryId}' ${label} step`);
+			if (step > stepCount) throw new RangeError(`CodeTraceBench '${trajectoryId}' ${label} step ${step} exceeds step_count ${stepCount}`);
+			const id = `${label}:${step}`;
+			if (labels.has(id)) throw new TypeError(`CodeTraceBench '${trajectoryId}' repeats label '${id}'`);
+			labels.add(id);
+			return {
+				id,
+				areas: [label],
+				evidence: [{
+					kind: evidenceKind,
+					uri: uri(trajectoryId, step)
+				}]
+			};
+		});
+	}
+}
+/** Translate CodeTracer's `codetracer_labels.json` into shared findings. */
+function codeTracerPredictionsToFindings(trajectoryIdValue, predictions, options = {}) {
+	const trajectoryId = nonEmpty$1(trajectoryIdValue, "CodeTracer prediction trajectory id");
+	const labels = parseCodeTracerPredictionLabels(predictions, trajectoryId);
+	const confidence = predictionConfidence(options.confidence);
+	const uri = options.stepUri ?? defaultStepUri;
+	const labelSet = codeTraceLabelSet(options.labelSet);
+	const seen = /* @__PURE__ */ new Set();
+	const findings = [];
+	for (const label of labels) {
+		const step = positiveStep(label.step, `CodeTracer prediction '${trajectoryId}' ${label.area} step`);
+		assertStepWithinRange(step, options.stepCount, `CodeTracer prediction '${trajectoryId}'`);
+		const key = `${label.area}:${step}`;
+		if (seen.has(key)) throw new TypeError(`CodeTracer prediction '${trajectoryId}' repeats label '${key}'`);
+		seen.add(key);
+		if (label.area === "unuseful" && labelSet === "incorrect-only") continue;
+		findings.push(makeFinding({
+			analyst_id: options.analystId ?? "codetracer",
+			produced_at: options.producedAt,
+			area: label.area,
+			subject: `step-${step}`,
+			claim: `CodeTracer labeled step ${step} as ${label.area}.`,
+			id_basis: key,
+			rationale: label.reasoning,
+			severity: "medium",
+			confidence,
+			evidence_refs: [{
+				kind: options.evidenceKind ?? "span",
+				uri: uri(trajectoryId, step)
+			}],
+			metadata: {
+				upstream: "CodeTracer",
+				stage_id: label.stageId,
+				step
+			}
+		}));
+	}
+	return findings;
+}
+function parseCodeTraceStages(value, trajectoryId) {
+	let parsed = value;
+	if (typeof value === "string") try {
+		parsed = JSON.parse(value);
+	} catch (error) {
+		throw new TypeError(`CodeTraceBench '${trajectoryId}' incorrect_stages is invalid JSON: ${error instanceof Error ? error.message : String(error)}`);
+	}
+	if (!Array.isArray(parsed)) throw new TypeError(`CodeTraceBench '${trajectoryId}' incorrect_stages must be an array`);
+	const stageIds = /* @__PURE__ */ new Set();
+	for (const stage of parsed) {
+		if (!stage || typeof stage !== "object" || !Number.isSafeInteger(stage.stage_id) || stage.stage_id < 1 || !optionalStepArray(stage.incorrect_step_ids) || !optionalStepArray(stage.unuseful_step_ids) || stage.reasoning !== void 0 && typeof stage.reasoning !== "string") throw new TypeError(`CodeTraceBench '${trajectoryId}' contains an invalid stage annotation`);
+		const stageId = stage.stage_id;
+		if (stageIds.has(stageId)) throw new TypeError(`CodeTraceBench '${trajectoryId}' repeats stage_id ${stageId}`);
+		stageIds.add(stageId);
+	}
+	return parsed;
+}
+function parseCodeTracerPredictionLabels(value, trajectoryId) {
+	let parsed = value;
+	if (typeof value === "string") try {
+		parsed = JSON.parse(value);
+	} catch (error) {
+		throw new TypeError(`CodeTracer prediction '${trajectoryId}' is invalid JSON: ${error instanceof Error ? error.message : String(error)}`);
+	}
+	if (!Array.isArray(parsed)) throw new TypeError(`CodeTracer prediction '${trajectoryId}' must be an array`);
+	if (parsed.length === 0) return [];
+	if (parsed.every(isCodeTraceStageAnnotation)) return parsed.flatMap((stage) => [...(stage.incorrect_step_ids ?? []).map((step) => ({
+		stageId: stage.stage_id,
+		area: "incorrect",
+		step,
+		...stage.reasoning === void 0 ? {} : { reasoning: stage.reasoning }
+	})), ...(stage.unuseful_step_ids ?? []).map((step) => ({
+		stageId: stage.stage_id,
+		area: "unuseful",
+		step,
+		...stage.reasoning === void 0 ? {} : { reasoning: stage.reasoning }
+	}))]);
+	const flat = [];
+	for (const [index, item] of parsed.entries()) {
+		if (isCodeTracerStepLabel(item)) {
+			flat.push(toCodeTracerPredictionLabel(item, codeTracerStageId(item, index + 1, trajectoryId), trajectoryId));
+			continue;
+		}
+		if (!isRecord$2(item) || !Array.isArray(item.labels)) throw new TypeError(`CodeTracer prediction '${trajectoryId}' contains an unsupported label row`);
+		const stageId = codeTracerStageId(item, index + 1, trajectoryId);
+		for (const label of item.labels) {
+			if (!isCodeTracerStepLabel(label)) throw new TypeError(`CodeTracer prediction '${trajectoryId}' contains an invalid step label`);
+			flat.push(toCodeTracerPredictionLabel(label, stageId, trajectoryId));
+		}
+	}
+	return flat;
+}
+function isCodeTraceStageAnnotation(value) {
+	return isRecord$2(value) && Number.isSafeInteger(value.stage_id) && value.stage_id > 0 && optionalStepArray(value.incorrect_step_ids) && optionalStepArray(value.unuseful_step_ids) && (value.reasoning === void 0 || typeof value.reasoning === "string");
+}
+function isCodeTracerStepLabel(value) {
+	return isRecord$2(value) && Number.isSafeInteger(value.step_id) && value.step_id > 0 && (value.stage === void 0 || typeof value.stage === "string" || typeof value.stage === "number") && (value.stage_name === void 0 || typeof value.stage_name === "string" || typeof value.stage_name === "number") && (value.stage_id === void 0 || typeof value.stage_id === "string" || typeof value.stage_id === "number") && (value.label === "incorrect" || value.label === "unuseful") && (value.rationale === void 0 || typeof value.rationale === "string") && (value.reason === void 0 || typeof value.reason === "string") && (value.note === void 0 || typeof value.note === "string") && (value.comment === void 0 || typeof value.comment === "string");
+}
+function toCodeTracerPredictionLabel(label, stageId, trajectoryId) {
+	if (!isCodeTracerStepLabel(label)) throw new TypeError(`CodeTracer prediction '${trajectoryId}' contains an invalid step label`);
+	return {
+		stageId,
+		area: label.label,
+		step: label.step_id,
+		...codeTracerReason(label, trajectoryId)
+	};
+}
+function codeTracerStageId(value, fallback, trajectoryId) {
+	const candidates = [
+		value.stage,
+		value.stage_name,
+		value.stage_id
+	].filter((candidate) => typeof candidate === "string" || typeof candidate === "number");
+	if (new Set(candidates).size > 1) throw new TypeError(`CodeTracer prediction '${trajectoryId}' has conflicting stage fields`);
+	return candidates[0] ?? fallback;
+}
+function codeTracerReason(label, trajectoryId) {
+	const candidates = [
+		label.rationale,
+		label.reason,
+		label.note,
+		label.comment
+	].filter((candidate) => candidate !== void 0);
+	if (new Set(candidates).size > 1) throw new TypeError(`CodeTracer prediction '${trajectoryId}' step ${label.step_id} has conflicting reason fields`);
+	return candidates[0] === void 0 ? {} : { reasoning: candidates[0] };
+}
+function parseTags(value, trajectoryId) {
+	if (value === void 0) return [];
+	let parsed = value;
+	if (typeof value === "string") try {
+		parsed = JSON.parse(value);
+	} catch (error) {
+		throw new TypeError(`CodeTraceBench '${trajectoryId}' tags is invalid JSON: ${error instanceof Error ? error.message : String(error)}`);
+	}
+	if (!Array.isArray(parsed)) throw new TypeError(`CodeTraceBench '${trajectoryId}' tags must be an array of strings`);
+	const tags = parsed.map((tag, index) => requiredCodeTraceString(tag, `CodeTraceBench '${trajectoryId}' tags[${index}]`));
+	if (new Set(tags).size !== tags.length) throw new TypeError(`CodeTraceBench '${trajectoryId}' tags must not repeat values`);
+	return tags;
+}
+function codeTraceLabelSet(value) {
+	if (value === void 0 || value === "incorrect-only") return "incorrect-only";
+	if (value === "incorrect-and-unuseful") return value;
+	throw new TypeError("CodeTraceBench labelSet must be 'incorrect-only' or 'incorrect-and-unuseful'");
+}
+function optionalStepArray(value) {
+	return value === void 0 || Array.isArray(value) && value.every(Number.isSafeInteger);
+}
+function requiredCodeTraceString(value, field) {
+	if (typeof value !== "string") throw new TypeError(`${field} must be a string`);
+	return nonEmpty$1(value, field);
+}
+function optionalCodeTraceString(value, field) {
+	if (value === void 0) return void 0;
+	return requiredCodeTraceString(value, field);
+}
+function optionalSourceRelativePath(value, trajectoryId) {
+	const path = optionalCodeTraceString(value, `CodeTraceBench '${trajectoryId}' source_relpath`);
+	if (path === void 0) return void 0;
+	const segments = path.replaceAll("\\", "/").split("/");
+	if (path.startsWith("/") || /^[a-zA-Z]:[\\/]/.test(path) || segments.some((segment) => segment === "..")) throw new TypeError(`CodeTraceBench '${trajectoryId}' source_relpath must stay within the artifact root`);
+	return path;
+}
+function codeTraceSolved(value, trajectoryId) {
+	if (value === void 0 || value === null || typeof value === "boolean") return value;
+	throw new TypeError(`CodeTraceBench '${trajectoryId}' solved must be a boolean or null`);
+}
+//#endregion
+//#region src/analyst/benchmark-agentrx-calibration.ts
+const AGENT_RX_UPSTREAM_REVISION = "f228165bfec60a801fd5fedd9d8ffe0f9de0c69d";
+function summarizeAgentRxCalibration(result, upstreamRevision) {
+	if (!upstreamRevision.trim()) throw new TypeError("AgentRx calibration requires an upstream revision");
+	return {
+		protocol: "official-agentrx-root-cause",
+		upstreamRevision,
+		rationale: "Matches AgentRx root-category accuracy, Python-rounded exact and tolerance step accuracy, unrounded mean step distance, normalized distance, and any, earliest, and terminal category accuracy. Failed runs and empty predictions score as no prediction.",
+		runners: result.provenance.runnerIds.map((runnerId) => summarizeRunner$1(runnerId, result.observations.filter((observation) => observation.runnerId === runnerId)))
+	};
+}
+function renderAgentRxCalibrationMarkdown(summary) {
+	return [
+		"## AgentRx Published Metrics",
+		"",
+		summary.rationale,
+		"",
+		`Upstream revision: \`${summary.upstreamRevision}\`.`,
+		"",
+		"| Runner | Completed/selected | Failed | Predictions | Missing predictions | Exact step | Within 1 | Within 2 | Within 3 | Within 4 | Within 5 | Mean step distance | Normalized distance | Normalized known/unknown | Root category | Any category | Earliest category | Terminal category |",
+		"| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
+		...summary.runners.map((runner) => `| ${escapeCell$2(runner.runnerId)} | ${runner.completedRuns}/${runner.selectedRuns} | ${runner.failedRuns} | ${runner.predictedRuns} | ${runner.missingPredictionRuns} | ${rate$3(runner.exactStepAccuracy)} | ${rate$3(runner.stepAccuracyWithin1)} | ${rate$3(runner.stepAccuracyWithin2)} | ${rate$3(runner.stepAccuracyWithin3)} | ${rate$3(runner.stepAccuracyWithin4)} | ${rate$3(runner.stepAccuracyWithin5)} | ${number$1(runner.meanStepDistance)} | ${number$1(runner.normalizedMeanStepDistance)} | ${runner.normalizedDistanceRuns}/${runner.normalizedDistanceUnknownRuns} | ${rate$3(runner.rootCauseCategoryAccuracy)} | ${rate$3(runner.anyFailureCategoryAccuracy)} | ${rate$3(runner.earliestFailureCategoryAccuracy)} | ${rate$3(runner.terminalFailureCategoryAccuracy)} |`)
+	].join("\n");
+}
+function summarizeRunner$1(runnerId, observations) {
+	const scored = observations.map(scoredObservation);
+	const normalized = scored.filter((row) => row.normalizedDistance !== null);
+	const predicted = scored.filter((row) => row.distance !== null);
+	return {
+		runnerId,
+		selectedRuns: observations.length,
+		completedRuns: observations.filter((observation) => !observation.error).length,
+		failedRuns: observations.filter((observation) => Boolean(observation.error)).length,
+		predictedRuns: predicted.length,
+		missingPredictionRuns: scored.length - predicted.length,
+		exactStepAccuracy: mean$2(scored.map((row) => Number(row.roundedDistance === 0))),
+		stepAccuracyWithin1: mean$2(scored.map((row) => Number(row.roundedDistance !== null && row.roundedDistance <= 1))),
+		stepAccuracyWithin2: mean$2(scored.map((row) => Number(row.roundedDistance !== null && row.roundedDistance <= 2))),
+		stepAccuracyWithin3: mean$2(scored.map((row) => Number(row.roundedDistance !== null && row.roundedDistance <= 3))),
+		stepAccuracyWithin4: mean$2(scored.map((row) => Number(row.roundedDistance !== null && row.roundedDistance <= 4))),
+		stepAccuracyWithin5: mean$2(scored.map((row) => Number(row.roundedDistance !== null && row.roundedDistance <= 5))),
+		meanStepDistance: mean$2(predicted.map((row) => row.distance)),
+		normalizedMeanStepDistance: mean$2(normalized.map((row) => row.normalizedDistance)),
+		normalizedDistanceRuns: normalized.length,
+		normalizedDistanceUnknownRuns: scored.length - normalized.length,
+		rootCauseCategoryAccuracy: mean$2(scored.map((row) => Number(row.rootCategoryMatch))),
+		anyFailureCategoryAccuracy: mean$2(scored.map((row) => Number(row.anyCategoryMatch))),
+		earliestFailureCategoryAccuracy: mean$2(scored.map((row) => Number(row.earliestCategoryMatch))),
+		terminalFailureCategoryAccuracy: mean$2(scored.map((row) => Number(row.terminalCategoryMatch)))
+	};
+}
+function scoredObservation(observation) {
+	const metadata = record(observation.caseMetadata);
+	const rootStep = requiredPositiveNumber(metadata.rootCauseStep, observation.caseId, "rootCauseStep");
+	const rootCategory = requiredString$1(metadata.rootCauseCategory, observation.caseId, "rootCauseCategory");
+	const allCategories = requiredStringArray(metadata.allFailureCategories, observation.caseId, "allFailureCategories");
+	const earliestCategory = requiredString$1(metadata.earliestFailureCategory, observation.caseId, "earliestFailureCategory");
+	const terminalCategory = requiredString$1(metadata.terminalFailureCategory, observation.caseId, "terminalFailureCategory");
+	const finding = observation.error ? void 0 : observation.findings[0];
+	const findingMetadata = record(finding?.metadata);
+	const stepMean = finding ? finiteNonNegative(findingMetadata.step_mean ?? findingMetadata.step, observation.caseId, "predicted step") : null;
+	const roundedDistance = stepMean === null ? null : Math.abs(roundAgentRxStep(stepMean) - rootStep);
+	const distance = stepMean === null ? null : Math.abs(stepMean - rootStep);
+	const trajectoryLength = metadata.trajectoryLength === void 0 ? null : requiredPositiveNumber(metadata.trajectoryLength, observation.caseId, "trajectoryLength");
+	const predictedCategory = finding?.area;
+	return {
+		roundedDistance,
+		distance,
+		normalizedDistance: trajectoryLength === null || distance === null ? null : distance / trajectoryLength,
+		rootCategoryMatch: predictedCategory === rootCategory,
+		anyCategoryMatch: predictedCategory !== void 0 && allCategories.includes(predictedCategory),
+		earliestCategoryMatch: predictedCategory === earliestCategory,
+		terminalCategoryMatch: predictedCategory === terminalCategory
+	};
+}
+function record(value) {
+	return typeof value === "object" && value !== null && !Array.isArray(value) ? value : {};
+}
+function requiredPositiveNumber(value, caseId, field) {
+	const numberValue = finiteNonNegative(value, caseId, field);
+	if (numberValue <= 0) throw new TypeError(`${caseId}: ${field} must be positive`);
+	return numberValue;
+}
+function finiteNonNegative(value, caseId, field) {
+	if (typeof value !== "number" || !Number.isFinite(value) || value < 0) throw new TypeError(`${caseId}: ${field} must be a finite non-negative number`);
+	return value;
+}
+function requiredString$1(value, caseId, field) {
+	if (typeof value !== "string" || !value.trim()) throw new TypeError(`${caseId}: ${field} must be a non-empty string`);
+	return value;
+}
+function requiredStringArray(value, caseId, field) {
+	if (!Array.isArray(value) || value.length === 0 || value.some((entry) => typeof entry !== "string" || !entry.trim())) throw new TypeError(`${caseId}: ${field} must be a non-empty string array`);
+	return value;
+}
+function mean$2(values) {
+	return values.length === 0 ? null : values.reduce((total, value) => total + value, 0) / values.length;
+}
+function rate$3(value) {
+	return value === null ? "n/a" : value.toFixed(3);
+}
+function number$1(value) {
+	return value === null ? "n/a" : value.toFixed(3);
+}
+function escapeCell$2(value) {
+	return value.replaceAll("|", "\\|").replaceAll("\n", " ");
+}
+//#endregion
+//#region src/analyst/benchmark-command-validation.ts
+const nonEmptyString = z.string().refine((value) => value.trim().length > 0, { message: "must be a non-empty string" });
+const safeInteger$1 = z.number().refine(Number.isSafeInteger, { message: "must be a safe integer" });
+const nonNegativeInteger = safeInteger$1.refine((value) => value >= 0, { message: "must be a non-negative safe integer" });
+const positiveInteger$1 = safeInteger$1.refine((value) => value > 0, { message: "must be a positive safe integer" });
+const nonNegativeNumber = z.number().nonnegative();
+const rate$2 = z.number().min(0).max(1);
+const nullableRate = rate$2.nullable();
+const nullableFiniteNumber = z.number().nullable();
+const sha256 = z.string().regex(/^[a-f0-9]{64}$/, "must be a lowercase SHA-256 digest");
+const revision = z.string().regex(/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/, "must be a lowercase 40 or 64 character revision");
+const timestamp = z.string().refine((value) => Number.isFinite(Date.parse(value)), { message: "must be a valid timestamp" });
+const stringArray = z.array(z.string());
+const nonEmptyStringArray = z.array(nonEmptyString);
+const metadata = z.record(z.string(), z.unknown());
+const errorSchema = z.strictObject({
+	class: nonEmptyString,
+	message: nonEmptyString,
+	code: nonEmptyString.optional(),
+	status: z.number().int().min(100).max(599).optional()
+});
+const evidenceSchema = z.strictObject({
+	kind: z.enum([
+		"span",
+		"event",
+		"artifact",
+		"finding",
+		"metric"
+	]),
+	uri: nonEmptyString,
+	excerpt: z.string().optional()
+});
+const findingSchema = z.strictObject({
+	schema_version: z.literal("1.0.0"),
+	finding_id: nonEmptyString,
+	analyst_id: nonEmptyString,
+	produced_at: timestamp,
+	severity: z.enum([
+		"critical",
+		"high",
+		"medium",
+		"low",
+		"info"
+	]),
+	area: nonEmptyString,
+	claim: nonEmptyString,
+	rationale: z.string().optional(),
+	evidence_refs: z.array(evidenceSchema),
+	recommended_action: z.string().optional(),
+	validation_plan: z.string().optional(),
+	confidence: rate$2,
+	subject: z.string().optional(),
+	derived_from_judge: z.boolean().optional(),
+	metadata: metadata.optional()
+});
+const tokenUsageSchema = z.strictObject({
+	input: nonNegativeInteger,
+	output: nonNegativeInteger,
+	reasoning: nonNegativeInteger.optional(),
+	cached: nonNegativeInteger.optional(),
+	cacheWrite: nonNegativeInteger.optional()
+}).superRefine((usage, context) => {
+	if (usage.reasoning !== void 0 && usage.reasoning > usage.output) context.addIssue({
+		code: "custom",
+		path: ["reasoning"],
+		message: "must not exceed output tokens"
+	});
+});
+const costSchema = z.discriminatedUnion("kind", [
+	z.strictObject({
+		kind: z.literal("observed"),
+		usd: nonNegativeNumber
+	}),
+	z.strictObject({
+		kind: z.literal("estimated"),
+		usd: nonNegativeNumber
+	}),
+	z.strictObject({
+		kind: z.literal("uncaptured"),
+		usd: z.null()
+	})
+]);
+const usageSchema = z.strictObject({
+	calls: nonNegativeInteger.nullable(),
+	tokens: tokenUsageSchema.nullable(),
+	cost: costSchema,
+	knownCostUsd: nonNegativeNumber.optional()
+});
+const findingScoreSchema = z.strictObject({
+	expectedIssueCount: nonNegativeInteger,
+	matchedIssueIds: nonEmptyStringArray,
+	missedIssueIds: nonEmptyStringArray,
+	supportedFindingIndexes: z.array(nonNegativeInteger),
+	unsupportedFindingIndexes: z.array(nonNegativeInteger),
+	unlabeledEvidence: z.array(evidenceSchema),
+	issueRecall: rate$2,
+	findingPrecision: rate$2,
+	f1: rate$2,
+	criticalStepAccuracy: nullableRate,
+	citationCoverage: nullableRate,
+	citationExcerptCoverage: nullableRate,
+	citationLabelAgreement: nullableRate,
+	predictionOnLabelEmptyCase: z.boolean()
+});
+const evidenceResolutionSchema = z.strictObject({
+	checked: nonNegativeInteger,
+	resolved: nonNegativeInteger,
+	unresolvedEvidence: z.array(evidenceSchema),
+	errors: z.array(z.strictObject({
+		evidence: evidenceSchema,
+		class: nonEmptyString,
+		message: nonEmptyString
+	})),
+	validity: nullableRate
+});
+const observationSchema = z.strictObject({
+	runnerId: nonEmptyString,
+	caseId: nonEmptyString,
+	clusterId: nonEmptyString,
+	labelState: z.enum([
+		"positive",
+		"trusted-negative",
+		"unlabeled"
+	]),
+	repetition: nonNegativeInteger,
+	executionIndex: nonNegativeInteger,
+	latencyMs: nonNegativeNumber.nullable(),
+	latencySource: z.enum([
+		"benchmark-clock",
+		"runner-reported",
+		"uncaptured"
+	]),
+	findings: z.array(findingSchema),
+	score: findingScoreSchema,
+	evidenceResolution: evidenceResolutionSchema.optional(),
+	caseTags: stringArray,
+	caseMetadata: metadata.optional(),
+	usage: usageSchema.optional(),
+	runnerMetadata: metadata.optional(),
+	error: errorSchema.optional()
+}).superRefine((observation, context) => {
+	const latencyIsMissing = observation.latencyMs === null;
+	if (observation.latencySource === "uncaptured" && !latencyIsMissing || observation.latencySource !== "uncaptured" && latencyIsMissing) context.addIssue({
+		code: "custom",
+		path: ["latencyMs"],
+		message: `must ${observation.latencySource === "uncaptured" ? "" : "not "}be null for '${observation.latencySource}' latency`
+	});
+});
+const latencyDistributionSchema = z.strictObject({
+	min: nonNegativeNumber,
+	mean: nonNegativeNumber,
+	p50: nonNegativeNumber,
+	p95: nonNegativeNumber,
+	max: nonNegativeNumber
+});
+const summarySchema = z.strictObject({
+	runnerId: nonEmptyString,
+	plannedRuns: nonNegativeInteger,
+	completedRuns: nonNegativeInteger,
+	failedRuns: nonNegativeInteger,
+	issueBearingRuns: nonNegativeInteger,
+	trustedNegativeRuns: nonNegativeInteger,
+	unlabeledRuns: nonNegativeInteger,
+	issueRecall: nullableRate,
+	findingPrecision: nullableRate,
+	f1: nullableRate,
+	macroIssueRecall: nullableRate,
+	macroFindingPrecision: nullableRate,
+	macroF1: nullableRate,
+	criticalStepAccuracy: nullableRate,
+	citationCoverage: nullableRate,
+	citationExcerptCoverage: nullableRate,
+	citationLabelAgreement: nullableRate,
+	citationResolution: nullableRate,
+	citationResolutionUnknownRuns: nonNegativeInteger,
+	unresolvedCitations: nonNegativeInteger,
+	citationResolutionErrors: nonNegativeInteger,
+	trustedNegativeFalsePositiveRate: nullableRate,
+	trustedNegativeFailureRate: nullableRate,
+	unlabeledPredictionRate: nullableRate,
+	unlabeledFailureRate: nullableRate,
+	predictionAgreement: nullableRate,
+	predictionAgreementCases: nonNegativeInteger,
+	matchedLabelAgreement: nullableRate,
+	matchedLabelAgreementCases: nonNegativeInteger,
+	latencyMs: latencyDistributionSchema.nullable(),
+	benchmarkClockLatencyRuns: nonNegativeInteger,
+	runnerReportedLatencyRuns: nonNegativeInteger,
+	latencyUnknownRuns: nonNegativeInteger,
+	calls: nonNegativeInteger,
+	callsUnknownRuns: nonNegativeInteger,
+	inputTokens: nonNegativeInteger,
+	outputTokens: nonNegativeInteger,
+	reasoningTokens: nonNegativeInteger,
+	cachedTokens: nonNegativeInteger,
+	cacheWriteTokens: nonNegativeInteger,
+	tokenUsageUnknownRuns: nonNegativeInteger,
+	reasoningTokenUsageUnknownRuns: nonNegativeInteger,
+	cachedTokenUsageUnknownRuns: nonNegativeInteger,
+	cacheWriteTokenUsageUnknownRuns: nonNegativeInteger,
+	knownCostUsd: nonNegativeNumber,
+	costUnknownRuns: nonNegativeInteger
+});
+const provenanceSchema = z.strictObject({
+	id: nonEmptyString.optional(),
+	dataset: z.strictObject({
+		id: nonEmptyString,
+		revision: nonEmptyString,
+		split: nonEmptyString.optional()
+	}).optional(),
+	command: nonEmptyString.optional(),
+	environment: z.record(z.string(), z.string()).optional(),
+	metadata: metadata.optional(),
+	startedAt: timestamp,
+	endedAt: timestamp,
+	caseCount: positiveInteger$1,
+	runnerIds: nonEmptyStringArray.min(1),
+	repetitions: positiveInteger$1,
+	maxConcurrency: positiveInteger$1,
+	runnerOrderSeed: safeInteger$1
+}).superRefine((provenance, context) => {
+	if (Date.parse(provenance.endedAt) < Date.parse(provenance.startedAt)) context.addIssue({
+		code: "custom",
+		path: ["endedAt"],
+		message: "must not precede startedAt"
+	});
+});
+const resultSchema = z.strictObject({
+	provenance: provenanceSchema,
+	observations: z.array(observationSchema),
+	summaries: z.array(summarySchema)
+});
+const comparisonMetricSchema = z.strictObject({
+	metric: z.enum([
+		"completion",
+		"issueRecall",
+		"findingPrecision",
+		"f1",
+		"criticalStepAccuracy",
+		"citationCoverage",
+		"citationExcerptCoverage",
+		"citationLabelAgreement",
+		"citationResolution",
+		"trustedNegativeAccuracy",
+		"latencyMs",
+		"calls",
+		"inputTokens",
+		"outputTokens",
+		"reasoningTokens",
+		"cachedTokens",
+		"cacheWriteTokens",
+		"costUsd"
+	]),
+	direction: z.enum(["higher", "lower"]),
+	pairedCases: nonNegativeInteger,
+	pairedClusters: nonNegativeInteger,
+	eligibleObservations: nonNegativeInteger,
+	pairedObservations: nonNegativeInteger,
+	baselineMissingObservations: nonNegativeInteger,
+	candidateMissingObservations: nonNegativeInteger,
+	asymmetricMissingObservations: nonNegativeInteger,
+	survivorOnly: z.boolean(),
+	baselineMean: nullableFiniteNumber,
+	candidateMean: nullableFiniteNumber,
+	meanDelta: nullableFiniteNumber,
+	intervalLow: nullableFiniteNumber,
+	intervalHigh: nullableFiniteNumber,
+	confidence: z.number().gt(0).lt(1),
+	resamples: positiveInteger$1,
+	minimumSampleMet: z.boolean(),
+	populationInferenceEligible: z.boolean(),
+	inferenceLimitations: stringArray
+});
+const comparisonSchema = z.strictObject({
+	baselineRunnerId: nonEmptyString,
+	candidateRunnerId: nonEmptyString,
+	metrics: z.array(comparisonMetricSchema)
+});
+const valueDistributionSchema = z.strictObject({
+	total: nonNegativeInteger,
+	missing: nonNegativeInteger,
+	counts: z.record(z.string(), nonNegativeInteger)
+});
+const distributionsSchema = z.strictObject({
+	class: valueDistributionSchema,
+	agent: valueDistributionSchema,
+	model: valueDistributionSchema,
+	difficulty: valueDistributionSchema,
+	solved: valueDistributionSchema
+});
+const selectionReportSchema = z.strictObject({
+	method: z.enum(["census", "deterministic-hash"]),
+	seed: safeInteger$1,
+	sourceCount: positiveInteger$1,
+	selectedCount: positiveInteger$1,
+	stratified: z.literal(false),
+	representativeOfInput: z.boolean(),
+	source: distributionsSchema,
+	selected: distributionsSchema
+});
+const verificationOutcomeSchema = z.strictObject({
+	status: z.enum([
+		"passed",
+		"failed",
+		"unavailable"
+	]),
+	reason: z.enum([
+		"missing-result",
+		"result-output-unavailable",
+		"result-parse-error",
+		"result-label-disagreement"
+	]).optional(),
+	parseError: errorSchema.optional(),
+	sources: z.array(z.strictObject({
+		path: nonEmptyString,
+		format: z.enum([
+			"terminal-bench",
+			"swe-bench",
+			"swe-multi"
+		]),
+		status: z.enum([
+			"passed",
+			"failed",
+			"unavailable"
+		])
+	})),
+	passedCheckCount: nonNegativeInteger,
+	failedCheckCount: nonNegativeInteger,
+	passedChecks: stringArray,
+	failedChecks: stringArray
+});
+const verificationArtifactRole = z.enum([
+	"final-test-output",
+	"final-result",
+	"final-metrics"
+]);
+const verificationArtifactSchema = z.strictObject({
+	traceId: nonEmptyString,
+	status: z.enum(["present", "missing"]),
+	outcome: verificationOutcomeSchema,
+	outcomeSpanId: nonEmptyString,
+	caseDirectory: nonEmptyString,
+	caseDirectoriesSearched: nonEmptyStringArray,
+	totalBytes: nonNegativeInteger,
+	maxBytes: positiveInteger$1,
+	files: z.array(z.strictObject({
+		role: verificationArtifactRole,
+		path: nonEmptyString,
+		relativePath: nonEmptyString,
+		sha256,
+		bytes: nonNegativeInteger,
+		spanId: nonEmptyString
+	})),
+	missingRoles: z.array(verificationArtifactRole),
+	searched: z.strictObject({
+		"final-test-output": stringArray,
+		"final-result": stringArray,
+		"final-metrics": stringArray
+	})
+});
+const verificationAvailabilitySchema = z.strictObject({
+	cases: nonNegativeInteger,
+	resultFilesPresent: nonNegativeInteger,
+	resultFilesMissing: nonNegativeInteger,
+	outcomes: z.strictObject({
+		passed: nonNegativeInteger,
+		failed: nonNegativeInteger,
+		unavailable: nonNegativeInteger
+	})
+});
+const codeTraceCalibrationSchema = z.strictObject({
+	protocol: z.literal("labeled-positive-and-solved-negative"),
+	rationale: nonEmptyString,
+	runners: z.array(z.strictObject({
+		runnerId: nonEmptyString,
+		selectedRuns: nonNegativeInteger,
+		positiveRuns: nonNegativeInteger,
+		trustedNegativeRuns: nonNegativeInteger,
+		unlabeledRuns: nonNegativeInteger,
+		failedLabelEmptyRuns: nonNegativeInteger,
+		unknownLabelEmptyRuns: nonNegativeInteger,
+		completedRuns: nonNegativeInteger,
+		failedRuns: nonNegativeInteger,
+		expectedIncorrectSteps: nonNegativeInteger,
+		predictedIncorrectSteps: nonNegativeInteger,
+		matchedIncorrectSteps: nonNegativeInteger,
+		officialAllRowF1: nullableRate,
+		officialAllRowRuns: nonNegativeInteger,
+		precision: nullableRate,
+		recall: nullableRate,
+		f1: nullableRate,
+		trustedNegativeFalsePositiveRate: nullableRate,
+		trustedNegativeFailureRate: nullableRate,
+		unlabeledPredictionRate: nullableRate,
+		unlabeledFailureRate: nullableRate
+	}))
+});
+const agentRxCalibrationSchema = z.strictObject({
+	protocol: z.literal("official-agentrx-root-cause"),
+	upstreamRevision: revision,
+	rationale: nonEmptyString,
+	runners: z.array(z.strictObject({
+		runnerId: nonEmptyString,
+		selectedRuns: nonNegativeInteger,
+		completedRuns: nonNegativeInteger,
+		failedRuns: nonNegativeInteger,
+		predictedRuns: nonNegativeInteger,
+		missingPredictionRuns: nonNegativeInteger,
+		exactStepAccuracy: nullableRate,
+		stepAccuracyWithin1: nullableRate,
+		stepAccuracyWithin2: nullableRate,
+		stepAccuracyWithin3: nullableRate,
+		stepAccuracyWithin4: nullableRate,
+		stepAccuracyWithin5: nullableRate,
+		meanStepDistance: nonNegativeNumber.nullable(),
+		normalizedMeanStepDistance: nullableRate,
+		normalizedDistanceRuns: nonNegativeInteger,
+		normalizedDistanceUnknownRuns: nonNegativeInteger,
+		rootCauseCategoryAccuracy: nullableRate,
+		anyFailureCategoryAccuracy: nullableRate,
+		earliestFailureCategoryAccuracy: nullableRate,
+		terminalFailureCategoryAccuracy: nullableRate
+	}))
+});
+const artifactSchema = z.strictObject({
+	kind: z.literal("agent-eval/analyst-benchmark-result"),
+	runIdentitySha256: sha256,
+	inputs: z.strictObject({
+		dataset: z.enum(["agentrx", "codetracebench"]),
+		datasetRevision: revision,
+		datasetSplit: nonEmptyString,
+		labelsSha256: sha256,
+		sourceRowCount: positiveInteger$1,
+		traceFiles: z.array(z.strictObject({
+			traceId: nonEmptyString,
+			relativePath: nonEmptyString,
+			sha256
+		})),
+		verificationArtifacts: z.array(verificationArtifactSchema),
+		verificationAvailability: verificationAvailabilitySchema,
+		selection: z.strictObject({
+			limit: positiveInteger$1,
+			seed: safeInteger$1,
+			selectedCaseIds: nonEmptyStringArray.min(1),
+			report: selectionReportSchema
+		}),
+		execution: z.strictObject({
+			repetitions: positiveInteger$1,
+			concurrency: positiveInteger$1,
+			model: nonEmptyString,
+			maxOutputTokens: positiveInteger$1,
+			timeoutMs: positiveInteger$1,
+			maxCostUsd: nonNegativeNumber,
+			maxArtifactBytes: positiveInteger$1,
+			analystProtocolSha256: sha256,
+			implementationSha256: sha256,
+			dependencyLockSha256: sha256
+		})
+	}),
+	result: resultSchema,
+	comparisons: z.array(comparisonSchema),
+	codeTraceCalibration: codeTraceCalibrationSchema.optional(),
+	agentRxCalibration: agentRxCalibrationSchema.optional()
+}).superRefine((artifact, context) => {
+	const isCodeTrace = artifact.inputs.dataset === "codetracebench";
+	if (isCodeTrace && !artifact.codeTraceCalibration) context.addIssue({
+		code: "custom",
+		path: ["codeTraceCalibration"],
+		message: "is required for CodeTraceBench artifacts"
+	});
+	if (isCodeTrace && artifact.agentRxCalibration) context.addIssue({
+		code: "custom",
+		path: ["agentRxCalibration"],
+		message: "is not allowed for CodeTraceBench artifacts"
+	});
+	if (!isCodeTrace && !artifact.agentRxCalibration) context.addIssue({
+		code: "custom",
+		path: ["agentRxCalibration"],
+		message: "is required for AgentRx artifacts"
+	});
+	if (!isCodeTrace && artifact.codeTraceCalibration) context.addIssue({
+		code: "custom",
+		path: ["codeTraceCalibration"],
+		message: "is not allowed for AgentRx artifacts"
+	});
+});
+function assertAnalystBenchmarkObservation(value, context) {
+	assertSchema(observationSchema, value, context);
+}
+function assertAnalystBenchmarkArtifact(value, context) {
+	assertSchema(artifactSchema, value, context);
+}
+function assertSchema(schema, value, context) {
+	const result = schema.safeParse(value);
+	if (result.success) return;
+	throw new TypeError(formatIssue(result.error.issues[0], context));
+}
+function formatIssue(issue, context) {
+	const path = issue.path.length === 0 ? context : `${context}.${issue.path.join(".")}`;
+	if (issue.code === "unrecognized_keys") return `${path} contains unknown field '${issue.keys[0]}'`;
+	return `${path} ${issue.message}`;
+}
+//#endregion
+//#region src/analyst/benchmark-command-artifact.ts
+const ANALYST_BENCHMARK_MANIFEST_FILE = "manifest.json";
+const ANALYST_BENCHMARK_OBSERVATIONS_FILE = "observations.jsonl";
+const ANALYST_BENCHMARK_LOCAL_RECEIPT_FILE = "run.local.json";
+const ANALYST_BENCHMARK_COST_LEDGER_FILE = "cost-ledger.jsonl";
+function observationKey(observation) {
+	return `${observation.runnerId}\u0000${observation.caseId}\u0000${observation.repetition}`;
+}
+function parseJson(text, source) {
+	try {
+		return JSON.parse(text);
+	} catch {
+		throw new Error(`invalid JSON in ${source}`);
+	}
+}
+function assertExactKeys(value, allowed, context, optional = []) {
+	const allowedSet = new Set(allowed);
+	const optionalSet = new Set(optional);
+	for (const key of Object.keys(value)) if (!allowedSet.has(key)) throw new TypeError(`${context} contains unknown field '${key}'`);
+	for (const key of allowed) if (!optionalSet.has(key) && !(key in value)) throw new TypeError(`${context} is missing field '${key}'`);
+}
+function digestCanonical(value) {
+	return createHash("sha256").update(canonicalJson(value)).digest("hex");
+}
+function canonicalJson(value) {
+	if (value === null || typeof value === "string" || typeof value === "boolean") return JSON.stringify(value);
+	if (typeof value === "number") {
+		if (!Number.isFinite(value)) throw new TypeError("cannot hash a non-finite number");
+		return JSON.stringify(value);
+	}
+	if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
+	if (isRecord$1(value)) return `{${Object.keys(value).filter((key) => value[key] !== void 0).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`).join(",")}}`;
+	throw new TypeError(`cannot hash ${typeof value}`);
+}
+function isRecord$1(value) {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function isSha256(value) {
+	return typeof value === "string" && /^[a-f0-9]{64}$/.test(value);
+}
+//#endregion
+//#region src/analyst/benchmark-implementation.ts
+const ANALYST_BENCHMARK_IMPLEMENTATION_DIGEST_ALGORITHM = "sha256-canonical-source-manifest";
+const ANALYST_BENCHMARK_DEPENDENCY_LOCK_DIGEST_ALGORITHM = "sha256-canonical-file-manifest";
+const ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES = Object.freeze(["package.json", "pnpm-lock.yaml"]);
+const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = "d06e3e3f171cc6bd5cf36b14325f507f34b1c6a9d5129ce8a85bf3c1681f8223";
+/** The published benchmark evidence was produced at this package version.
+* A release changes package.json's version field, which is part of the lock
+* manifest but cannot change benchmark behavior, so the evidence stays bound
+* to the digest at its creation. A test proves the current lock differs from
+* the evidence lock by the version stamp alone; any real dependency change
+* still forces a new benchmark run or explicit retirement of the evidence. */
+const ANALYST_BENCHMARK_EVIDENCE_PACKAGE_VERSION = "0.137.0";
+const ANALYST_BENCHMARK_EVIDENCE_DEPENDENCY_LOCK_SHA256 = "1e03f2daed356d60316aabefb407ec1e437ac94d408d61eea4ae096e9c6fbb5b";
+const ANALYST_BENCHMARK_IMPLEMENTATION_FILES = Object.freeze([
+	"src/analyst/benchmark-agentrx-calibration.ts",
+	"src/analyst/benchmark-command-artifact.ts",
+	"src/analyst/benchmark-command-persistence.ts",
+	"src/analyst/benchmark-command-result.ts",
+	"src/analyst/benchmark-command-validation.ts",
+	"src/analyst/benchmark-command.ts",
+	"src/analyst/benchmark-comparison.ts",
+	"src/analyst/benchmark-dataset-agentrx.ts",
+	"src/analyst/benchmark-dataset-codetrace.ts",
+	"src/analyst/benchmark-dataset-utils.ts",
+	"src/analyst/benchmark-datasets.ts",
+	"src/analyst/benchmark-evidence-validation.ts",
+	"src/analyst/benchmark-public-adapters.ts",
+	"src/analyst/benchmark-public-calibration.ts",
+	"src/analyst/benchmark-public-data.ts",
+	"src/analyst/benchmark-public-errors.ts",
+	"src/analyst/benchmark-public-model.ts",
+	"src/analyst/benchmark-public-types.ts",
+	"src/analyst/benchmark-real-model.ts",
+	"src/analyst/benchmark-report.ts",
+	"src/analyst/benchmark-response-cache.ts",
+	"src/analyst/benchmark-scoring.ts",
+	"src/analyst/benchmark-summary.ts",
+	"src/analyst/benchmark-verification-artifacts.ts",
+	"src/analyst/benchmark-verification-outcome.ts",
+	"src/analyst/benchmark.ts",
+	"src/analyst/types.ts",
+	"src/analyst/usage-receipt.ts",
+	"src/campaign/search-ledger-errors.ts",
+	"src/campaign/search-ledger-file.ts",
+	"src/campaign/single-run-lock.ts",
+	"src/campaign/storage.ts",
+	"src/concurrency.ts",
+	"src/cost-ledger.ts",
+	"src/errors.ts",
+	"src/judge-calibration.ts",
+	"src/ledger-core/atomic-file-lock.ts",
+	"src/ledger-core/canonical.ts",
+	"src/ledger-core/index.ts",
+	"src/ledger-core/journal-file.ts",
+	"src/ledger-core/journal.ts",
+	"src/ledger-core/trusted-head.ts",
+	"src/llm-client.ts",
+	"src/math/normal.ts",
+	"src/math/special-functions.ts",
+	"src/math/student-t.ts",
+	"src/metrics.ts",
+	"src/statistics.ts",
+	"src/trace-analyst/errors.ts",
+	"src/trace-analyst/otlp-span.ts",
+	"src/trace-analyst/shared-abortable-task.ts",
+	"src/trace-analyst/store-boundary.ts",
+	"src/trace-analyst/store-bounds.ts",
+	"src/trace-analyst/store-contract.ts",
+	"src/trace-analyst/store-otlp.ts",
+	"src/trace-analyst/store-schemas.ts",
+	"src/trace-analyst/store.ts",
+	"src/trace-analyst/types.ts",
+	"src/trace/attribute-vocabulary.ts",
+	"src/trace/otlp-attributes.ts",
+	"src/trace/raw-provider-sink.ts"
+]);
+const ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 = "4dba263b6256a30d56c7fdb2d992d3a953c0035d731f359b704db806f68f75ac";
+function analystBenchmarkImplementationDigest() {
+	return ANALYST_BENCHMARK_IMPLEMENTATION_SHA256;
+}
+function analystBenchmarkDependencyLockDigest() {
+	return ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256;
+}
+//#endregion
+//#region src/analyst/benchmark-evidence-validation.ts
+const MIN_ACTION_EXCERPT_CHARACTERS = 12;
+const MAX_LABEL_SCAN_DEPTH = 64;
+const MAX_SERIALIZED_JSON_SCAN_BYTES = 64 * 1024;
+const MAX_SERIALIZED_JSON_SCAN_DEPTH = 8;
+const BENCHMARK_LABEL_KEYS = /* @__PURE__ */ new Set([
+	"category_reason",
+	"failure_category",
+	"failure_summary",
+	"incorrect_stages",
+	"incorrect_step_ids",
+	"root_cause",
+	"root_cause_failure_id",
+	"root_cause_reason",
+	"step_reason",
+	"unuseful_step_ids"
+]);
+const BENCHMARK_LABEL_KEY_TOKENS = [...BENCHMARK_LABEL_KEYS].sort((left, right) => right.length - left.length);
+const BENCHMARK_LABEL_PATH_MARKERS = [
+	"bench_manifest.verified",
+	"codetracer_labels.json",
+	"/ground_truth/",
+	"\\ground_truth\\"
+];
+function assertNoBenchmarkLabelsInTrace(options) {
+	let scannedValues = 0;
+	for (const [index, line] of options.otlpText.split(/\r?\n/).entries()) {
+		if (!line.trim()) continue;
+		let value;
+		try {
+			value = JSON.parse(line);
+		} catch (error) {
+			throw new TypeError(`trace '${options.traceId}' line ${index + 1} is invalid JSON: ${error instanceof Error ? error.message : String(error)}`);
+		}
+		scannedValues += scanValue(value, options.traceId, `$line[${index + 1}]`, 0, 0);
+	}
+	if (scannedValues === 0) throw new Error(`trace '${options.traceId}' contains no JSON values`);
+	return {
+		passed: true,
+		scannedBytes: Buffer.byteLength(options.otlpText),
+		scannedValues
+	};
+}
+function assertNoBenchmarkLabelsInArtifact(options) {
+	const normalizedPath = options.relativePath.toLowerCase();
+	for (const marker of BENCHMARK_LABEL_PATH_MARKERS) if (normalizedPath.includes(marker.toLowerCase())) throw new Error(`trace '${options.traceId}' verification artifact path contains benchmark label marker '${marker}'`);
+	const normalizedContent = options.content.toLowerCase();
+	for (const key of BENCHMARK_LABEL_KEYS) if (normalizedContent.includes(key)) throw new Error(`trace '${options.traceId}' verification artifact contains benchmark label key '${key}'`);
+	for (const marker of BENCHMARK_LABEL_PATH_MARKERS) if (normalizedContent.includes(marker.toLowerCase())) throw new Error(`trace '${options.traceId}' verification artifact contains benchmark label path marker '${marker}'`);
+}
+async function validateCodeTraceFindingEvidence(options) {
+	const citations = options.findings.flatMap((finding) => finding.evidence_refs.map((evidence) => ({
+		evidence,
+		findingId: finding.finding_id,
+		location: codeTraceStepFromEvidence(evidence.uri)
+	})));
+	if (citations.length === 0) return;
+	for (const citation of citations) if (!citation.location || citation.location.traceId !== options.trajectoryId) throw new Error(`model finding '${citation.findingId}' cites non-case evidence '${citation.evidence.uri}'`);
+	const spanIds = [...new Set(citations.map((citation) => `step-${citation.location.step}`))];
+	const spans = /* @__PURE__ */ new Map();
+	for (let offset = 0; offset < spanIds.length; offset += TRACE_ANALYSIS_LIMITS.viewSpans) {
+		const requested = spanIds.slice(offset, offset + TRACE_ANALYSIS_LIMITS.viewSpans);
+		const result = await options.store.viewSpans({
+			trace_id: options.trajectoryId,
+			span_ids: requested
+		}, options.signal ? { signal: options.signal } : void 0);
+		if (result.missing_span_ids.length > 0 || result.omitted_span_ids.length > 0) {
+			const unavailable = [...result.missing_span_ids, ...result.omitted_span_ids];
+			throw new Error(`model finding evidence is unavailable in the case trace: ${unavailable.join(", ")}`);
+		}
+		for (const span of result.spans) spans.set(span.span_id, span);
+	}
+	for (const citation of citations) {
+		const spanId = `step-${citation.location.step}`;
+		const span = spans.get(spanId);
+		if (!span) throw new Error(`model finding '${citation.findingId}' cites missing span '${spanId}'`);
+		if (span.kind !== "LLM") throw new Error(`model finding '${citation.findingId}' cites '${spanId}', which is ${span.kind}, not an assistant LLM span`);
+		assertExactActionExcerpt(citation.findingId, citation.evidence, spanId, span.attributes.content);
+	}
+}
+async function resolveAssistantStepEvidence(options) {
+	const steps = [...new Set(options.steps)];
+	for (const step of steps) if (!Number.isSafeInteger(step) || step < 1) throw new TypeError(`assistant evidence step must be a positive safe integer: ${step}`);
+	const spanIds = steps.map((step) => `step-${step}`);
+	const spans = /* @__PURE__ */ new Map();
+	for (let offset = 0; offset < spanIds.length; offset += TRACE_ANALYSIS_LIMITS.viewSpans) {
+		const requested = spanIds.slice(offset, offset + TRACE_ANALYSIS_LIMITS.viewSpans);
+		const result = await options.store.viewSpans({
+			trace_id: options.trajectoryId,
+			span_ids: requested
+		}, options.signal ? { signal: options.signal } : void 0);
+		if (result.missing_span_ids.length > 0 || result.omitted_span_ids.length > 0) {
+			const unavailable = [...result.missing_span_ids, ...result.omitted_span_ids];
+			throw new Error(`model selected unavailable assistant steps: ${unavailable.join(", ")}`);
+		}
+		for (const span of result.spans) spans.set(span.span_id, span);
+	}
+	const evidence = /* @__PURE__ */ new Map();
+	for (const step of steps) {
+		const spanId = `step-${step}`;
+		const span = spans.get(spanId);
+		if (!span) throw new Error(`model selected missing assistant step '${spanId}'`);
+		if (span.kind !== "LLM") throw new Error(`model selected '${spanId}', which is ${span.kind}, not an assistant LLM span`);
+		const content = span.attributes.content;
+		if (typeof content !== "string" || content.trim().length === 0) throw new Error(`model selected '${spanId}' without action content`);
+		evidence.set(step, {
+			kind: "span",
+			uri: codeTraceStepEvidenceUri(options.trajectoryId, step),
+			excerpt: content.trim().slice(0, 512)
+		});
+	}
+	return evidence;
+}
+function scanValue(value, traceId, path, depth, serializedDepth) {
+	if (depth > MAX_LABEL_SCAN_DEPTH) throw new Error(`trace '${traceId}' exceeds benchmark label scan depth at ${path}`);
+	if (Array.isArray(value)) return 1 + value.reduce((count, entry, index) => count + scanValue(entry, traceId, `${path}[${index}]`, depth + 1, serializedDepth), 0);
+	if (typeof value === "object" && value !== null) {
+		let count = 1;
+		for (const [key, entry] of Object.entries(value)) {
+			if (BENCHMARK_LABEL_KEYS.has(key.toLowerCase())) throw new Error(`trace '${traceId}' exposes benchmark label key '${key}' at ${path}`);
+			count += scanValue(entry, traceId, `${path}.${key}`, depth + 1, serializedDepth);
+		}
+		return count;
+	}
+	if (typeof value === "string") {
+		const normalized = value.toLowerCase();
+		const labelKey = BENCHMARK_LABEL_KEY_TOKENS.find((candidate) => normalized.includes(candidate));
+		if (labelKey) throw new Error(`trace '${traceId}' exposes benchmark label key '${labelKey}' inside a string at ${path}`);
+		const marker = BENCHMARK_LABEL_PATH_MARKERS.find((candidate) => normalized.includes(candidate));
+		if (marker) throw new Error(`trace '${traceId}' exposes benchmark label path marker '${marker}' at ${path}`);
+		const trimmed = value.trim();
+		if (serializedDepth < MAX_SERIALIZED_JSON_SCAN_DEPTH && Buffer.byteLength(trimmed) <= MAX_SERIALIZED_JSON_SCAN_BYTES && looksLikeSerializedJson(trimmed)) {
+			let parsed;
+			try {
+				parsed = JSON.parse(trimmed);
+			} catch {
+				return 1;
+			}
+			return 1 + scanValue(parsed, traceId, `${path}<serialized-json>`, depth + 1, serializedDepth + 1);
+		}
+	}
+	return 1;
+}
+function looksLikeSerializedJson(value) {
+	return value.startsWith("{") && value.endsWith("}") || value.startsWith("[") && value.endsWith("]") || value.startsWith("\"") && value.endsWith("\"");
+}
+function codeTraceStepFromEvidence(uri) {
+	const match = /^trace:\/\/([^/]+)\/span\/step-(\d+)$/.exec(uri);
+	if (!match) return null;
+	try {
+		const traceId = decodeURIComponent(match[1]);
+		const step = Number(match[2]);
+		return traceId && Number.isSafeInteger(step) && step > 0 ? {
+			traceId,
+			step
+		} : null;
+	} catch {
+		return null;
+	}
+}
+function codeTraceStepEvidenceUri(traceId, step) {
+	return `trace://${encodeURIComponent(traceId)}/span/step-${step}`;
+}
+function assertExactActionExcerpt(findingId, evidence, spanId, content) {
+	if (typeof content !== "string" || content.length === 0) throw new Error(`model finding '${findingId}' cites '${spanId}' without action content`);
+	const excerpt = evidence.excerpt?.trim();
+	if (!excerpt) throw new Error(`model finding '${findingId}' must quote action content from '${spanId}'`);
+	const requiredLength = Math.min(MIN_ACTION_EXCERPT_CHARACTERS, content.trim().length);
+	if (excerpt.length < requiredLength) throw new Error(`model finding '${findingId}' excerpt for '${spanId}' is too short; expected at least ${requiredLength} characters`);
+	if (!content.includes(excerpt)) throw new Error(`model finding '${findingId}' excerpt is not present in '${spanId}' action content`);
+}
+//#endregion
+//#region src/analyst/benchmark-public-adapters.ts
+function emptyPublicBenchmarkRunner() {
+	return {
+		id: "empty",
+		analyze() {
+			return {
+				findings: [],
+				usage: {
+					calls: 0,
+					tokens: {
+						input: 0,
+						output: 0
+					},
+					cost: {
+						kind: "observed",
+						usd: 0
+					}
+				},
+				metadata: { baseline: "emit-no-findings" }
+			};
+		}
+	};
+}
+function adaptPublicBenchmarkFindings(dataset, trajectoryId, findings, analystId) {
+	return dataset === "agentrx" ? adaptAgentRxFindings(trajectoryId, findings, analystId) : adaptCodeTraceFindings(trajectoryId, findings, analystId);
+}
+function adaptAgentRxFindings(trajectoryId, findings, analystId) {
+	if (findings.length === 0) return [];
+	if (findings.length !== 1) throw new Error(`AgentRx model analyst must emit zero or one root cause, received ${findings.length}`);
+	const source = findings[0];
+	if (!source.subject) throw new Error("AgentRx model analyst finding is missing its failure-category subject");
+	const steps = exactFindingSteps(trajectoryId, source);
+	if (steps.length !== 1) throw new Error(`AgentRx model analyst must cite exactly one root-cause step, received ${steps.length}`);
+	const [adapted] = agentRxPredictionsToFindings(trajectoryId, [{
+		failure_case: source.subject,
+		step_number: steps[0],
+		description: source.rationale ?? source.claim
+	}], {
+		analystId,
+		producedAt: source.produced_at,
+		confidence: source.confidence
+	});
+	if (!adapted) throw new Error("AgentRx output adapter produced no root-cause finding");
+	return [{
+		...adapted,
+		metadata: {
+			...adapted.metadata,
+			sourceFindingId: source.finding_id
+		}
+	}];
+}
+function adaptCodeTraceFindings(trajectoryId, findings, analystId) {
+	const clean = findings.filter((finding) => finding.subject === "clean");
+	if (clean.length > 0) {
+		if (findings.length !== 1) throw new Error("CodeTraceBench model analyst mixed a clean verdict with incorrect steps");
+		exactFindingSteps(trajectoryId, clean[0]);
+		return [];
+	}
+	const byStep = /* @__PURE__ */ new Map();
+	for (const source of findings) {
+		const steps = exactFindingSteps(trajectoryId, source);
+		for (const step of steps) {
+			if (byStep.has(step)) continue;
+			byStep.set(step, makeFinding({
+				analyst_id: analystId,
+				area: "incorrect",
+				subject: `incorrect-step-${step}`,
+				claim: `Step ${step} is incorrect. ${source.claim}`,
+				rationale: source.rationale,
+				severity: source.severity,
+				confidence: source.confidence,
+				evidence_refs: [{
+					kind: "span",
+					uri: codeTraceStepEvidenceUri(trajectoryId, step),
+					excerpt: source.evidence_refs.find((evidence) => evidence.uri === codeTraceStepEvidenceUri(trajectoryId, step))?.excerpt
+				}],
+				recommended_action: source.recommended_action,
+				metadata: { sourceFindingId: source.finding_id },
+				produced_at: source.produced_at,
+				id_basis: `incorrect-step-${step}`
+			}));
+		}
+	}
+	return [...byStep].sort(([left], [right]) => left - right).map(([, finding]) => finding);
+}
+function exactFindingSteps(trajectoryId, finding) {
+	if (finding.evidence_refs.length === 0) throw new Error(`model finding '${finding.finding_id}' has no step evidence`);
+	const steps = finding.evidence_refs.map((evidence) => {
+		const parsed = codeTraceStepFromEvidence(evidence.uri);
+		if (!parsed || parsed.traceId !== trajectoryId) throw new Error(`model finding '${finding.finding_id}' cites non-case evidence '${evidence.uri}'`);
+		return parsed.step;
+	});
+	return [...new Set(steps)];
+}
+//#endregion
+//#region src/analyst/benchmark-public-types.ts
+function requiredString(value, field) {
+	const trimmed = value.trim();
+	if (!trimmed) throw new TypeError(`${field} must be a non-empty string`);
+	return trimmed;
+}
+function positiveSafeInteger(value, field) {
+	if (!Number.isSafeInteger(value) || value < 1) throw new RangeError(`${field} must be a positive safe integer`);
+	return value;
+}
+function safeInteger(value, field) {
+	if (!Number.isSafeInteger(value)) throw new RangeError(`${field} must be a safe integer`);
+	return value;
+}
+function isRecord(value) {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+//#endregion
+//#region src/analyst/benchmark-verification-outcome.ts
+const MAX_REPORTED_CHECKS = 20;
+const SWE_MULTI_NO_TEST_RESULTS = "After applying the fix patch, no test results were captured when executing the test command.";
+const checkNameSchema = z.string().min(1);
+const checkListSchema = z.array(checkNameSchema).superRefine((checks, context) => {
+	const seen = /* @__PURE__ */ new Set();
+	for (const [index, check] of checks.entries()) {
+		if (seen.has(check)) context.addIssue({
+			code: "custom",
+			path: [index],
+			message: `duplicate check '${check}'`
+		});
+		seen.add(check);
+	}
+});
+const nonNegativeCountSchema = z.number().int().min(0).max(Number.MAX_SAFE_INTEGER);
+const terminalBenchSchema = z.object({
+	is_resolved: z.boolean().nullable(),
+	failure_mode: z.string().min(1),
+	parser_results: z.record(z.string().min(1), z.enum(["passed", "failed"])).nullable()
+}).passthrough();
+const directSweBenchSchema = z.object({
+	resolved: z.boolean(),
+	passed_tests: checkListSchema,
+	failed_tests: checkListSchema
+}).passthrough();
+const nestedSweBenchCategorySchema = z.object({
+	success: checkListSchema,
+	failure: checkListSchema
+}).passthrough();
+const nestedSweBenchInstanceSchema = z.object({
+	resolved: z.boolean(),
+	tests_status: z.record(z.string().min(1), nestedSweBenchCategorySchema).refine((value) => Object.keys(value).length > 0, "must contain at least one test category")
+}).passthrough();
+const nestedSweBenchSchema = z.record(z.string().min(1), nestedSweBenchInstanceSchema).refine((value) => Object.keys(value).length > 0, "must contain at least one instance");
+const sweMultiCheckResultSchema = z.object({
+	passed_count: nonNegativeCountSchema,
+	failed_count: nonNegativeCountSchema,
+	skipped_count: nonNegativeCountSchema,
+	passed_tests: checkListSchema,
+	failed_tests: checkListSchema,
+	skipped_tests: checkListSchema
+}).passthrough();
+const sweMultiSchema = z.object({
+	valid: z.boolean(),
+	error_msg: z.string(),
+	fix_patch_result: sweMultiCheckResultSchema
+}).passthrough();
+function parseVerificationOutcome(files) {
+	if (files.length === 0) throw new Error("final verification outcome requires at least one result file");
+	const sources = [];
+	const passedChecks = /* @__PURE__ */ new Set();
+	const failedChecks = /* @__PURE__ */ new Set();
+	const unavailableReasons = /* @__PURE__ */ new Set();
+	for (const file of files) {
+		let value;
+		try {
+			value = JSON.parse(file.content);
+		} catch (error) {
+			throw new TypeError(`final verification result is not valid JSON: ${file.relativePath}: ${errorMessage(error)}`);
+		}
+		const parsed = parseResult(value, file.relativePath);
+		sources.push({
+			path: file.relativePath,
+			format: parsed.format,
+			status: parsed.status
+		});
+		if (parsed.reason) unavailableReasons.add(parsed.reason);
+		for (const check of parsed.passedChecks) passedChecks.add(check);
+		for (const check of parsed.failedChecks) failedChecks.add(check);
+	}
+	if (new Set(sources.map((source) => source.status)).size !== 1) throw new Error(`final verification result files disagree: ${sources.map((source) => `${source.path}=${source.status}`).join(", ")}`);
+	if (unavailableReasons.size > 1) throw new Error(`final verification result files disagree on why the outcome is unavailable: ${[...unavailableReasons].join(", ")}`);
+	const [unavailableReason] = unavailableReasons;
+	const passed = [...passedChecks].sort();
+	const failed = [...failedChecks].sort();
+	assertDisjointChecks(passed, failed, "final verification result");
+	return {
+		status: sources[0].status,
+		...unavailableReason ? { reason: unavailableReason } : {},
+		sources,
+		passedCheckCount: passed.length,
+		failedCheckCount: failed.length,
+		passedChecks: passed.slice(0, MAX_REPORTED_CHECKS),
+		failedChecks: failed.slice(0, MAX_REPORTED_CHECKS)
+	};
+}
+function parseResult(value, path) {
+	const record = asRecord(value);
+	if (!record) throw unsupported(path);
+	const discriminators = [
+		"is_resolved",
+		"resolved",
+		"valid"
+	].filter((field) => Object.hasOwn(record, field));
+	if (discriminators.length > 1) throw new TypeError(`final verification result is ambiguous: ${path}: found ${discriminators.join(", ")}`);
+	if (discriminators[0] === "is_resolved") return parseTerminalBench(record, path);
+	if (discriminators[0] === "resolved") return parseDirectSweBench(record, path);
+	if (discriminators[0] === "valid") return parseSweMulti(record, path);
+	if (Object.entries(record).some(([, candidate]) => {
+		const nested = asRecord(candidate);
+		return nested !== null && Object.hasOwn(nested, "resolved");
+	})) return parseNestedSweBench(record, path);
+	throw unsupported(path);
+}
+function parseTerminalBench(value, path) {
+	const record = parseSchema(terminalBenchSchema, value, path, "Terminal-Bench");
+	if (record.is_resolved === null) {
+		if (record.parser_results !== null) throw malformed(path, "Terminal-Bench", "parser_results must be null when is_resolved is null");
+		if (record.failure_mode === "unset") throw malformed(path, "Terminal-Bench", "failure_mode cannot be 'unset' when unresolved");
+		return {
+			format: "terminal-bench",
+			status: "unavailable",
+			reason: record.failure_mode === "parse_error" ? "result-parse-error" : "result-output-unavailable",
+			passedChecks: [],
+			failedChecks: []
+		};
+	}
+	if (record.parser_results === null) throw malformed(path, "Terminal-Bench", "parser_results must be an object when is_resolved is boolean");
+	const checks = stringStatusChecks(record.parser_results);
+	if (checks.passedChecks.length + checks.failedChecks.length === 0) throw malformed(path, "Terminal-Bench", "parser_results must contain at least one check");
+	assertOutcomeConsistency(record.is_resolved, checks, path, "Terminal-Bench is_resolved", true);
+	return {
+		format: "terminal-bench",
+		status: status(record.is_resolved),
+		...checks
+	};
+}
+function parseDirectSweBench(value, path) {
+	const record = parseSchema(directSweBenchSchema, value, path, "SWE-bench");
+	const checks = {
+		passedChecks: record.passed_tests,
+		failedChecks: record.failed_tests
+	};
+	assertOutcomeConsistency(record.resolved, checks, path, "SWE-bench resolved");
+	return {
+		format: "swe-bench",
+		status: status(record.resolved),
+		...checks
+	};
+}
+function parseNestedSweBench(value, path) {
+	const record = parseSchema(nestedSweBenchSchema, value, path, "SWE-bench instance report");
+	const instances = Object.entries(record);
+	const statuses = new Set(instances.map(([, instance]) => status(instance.resolved)));
+	if (statuses.size !== 1) throw new Error(`final verification report contains conflicting instance outcomes: ${path}: ${instances.map(([id, instance]) => `${id}=${status(instance.resolved)}`).join(", ")}`);
+	const passedChecks = [];
+	const failedChecks = [];
+	for (const [instanceId, instance] of instances) {
+		const checks = nestedSweBenchChecks(instanceId, instance.tests_status);
+		assertOutcomeConsistency(instance.resolved, checks, path, `SWE-bench instance '${instanceId}' resolved`);
+		passedChecks.push(...checks.passedChecks);
+		failedChecks.push(...checks.failedChecks);
+	}
+	return {
+		format: "swe-bench",
+		status: [...statuses][0],
+		passedChecks,
+		failedChecks
+	};
+}
+function parseSweMulti(value, path) {
+	const record = parseSchema(sweMultiSchema, value, path, "SWE-Multi");
+	const fix = record.fix_patch_result;
+	assertCount(fix.passed_count, fix.passed_tests, "passed", path);
+	assertCount(fix.failed_count, fix.failed_tests, "failed", path);
+	assertCount(fix.skipped_count, fix.skipped_tests, "skipped", path);
+	assertDisjointChecks(fix.passed_tests, fix.failed_tests, `SWE-Multi result ${path}`);
+	assertDisjointChecks(fix.passed_tests, fix.skipped_tests, `SWE-Multi result ${path}`);
+	assertDisjointChecks(fix.failed_tests, fix.skipped_tests, `SWE-Multi result ${path}`);
+	const checks = {
+		passedChecks: fix.passed_tests,
+		failedChecks: fix.failed_tests
+	};
+	if (record.valid === false && isSweMultiOutputUnavailable(record)) return {
+		format: "swe-multi",
+		status: "unavailable",
+		reason: "result-output-unavailable",
+		...checks
+	};
+	assertOutcomeConsistency(record.valid, checks, path, "SWE-Multi valid");
+	return {
+		format: "swe-multi",
+		status: status(record.valid),
+		...checks
+	};
+}
+function isSweMultiOutputUnavailable(record) {
+	const fix = record.fix_patch_result;
+	return (record.error_msg === SWE_MULTI_NO_TEST_RESULTS || record.error_msg.startsWith(`${SWE_MULTI_NO_TEST_RESULTS} `)) && fix.passed_count === 0 && fix.failed_count === 0 && fix.skipped_count === 0;
+}
+function stringStatusChecks(record) {
+	const passedChecks = [];
+	const failedChecks = [];
+	for (const [name, result] of Object.entries(record)) if (result === "passed") passedChecks.push(name);
+	else failedChecks.push(name);
+	return {
+		passedChecks,
+		failedChecks
+	};
+}
+function nestedSweBenchChecks(instanceId, testsStatus) {
+	const passedChecks = [];
+	const failedChecks = [];
+	for (const [category, result] of Object.entries(testsStatus)) {
+		passedChecks.push(...result.success.map((name) => `${instanceId}:${category}:${name}`));
+		failedChecks.push(...result.failure.map((name) => `${instanceId}:${category}:${name}`));
+	}
+	return {
+		passedChecks,
+		failedChecks
+	};
+}
+function assertOutcomeConsistency(passed, checks, path, field, requireFailedCheck = false) {
+	assertDisjointChecks(checks.passedChecks, checks.failedChecks, `${field} in ${path}`);
+	if (passed && checks.failedChecks.length > 0) throw malformed(path, field, `cannot be true while failed checks are reported: ${checks.failedChecks.join(", ")}`);
+	if (passed && checks.passedChecks.length === 0) throw malformed(path, field, "cannot be true without at least one passed check");
+	if (!passed && requireFailedCheck && checks.failedChecks.length === 0) throw malformed(path, field, "cannot be false without at least one failed check");
+}
+function assertCount(count, checks, kind, path) {
+	if (count !== checks.length) throw malformed(path, "SWE-Multi", `${kind}_count=${count} does not match ${kind}_tests length ${checks.length}`);
+}
+function assertDisjointChecks(left, right, source) {
+	const rightSet = new Set(right);
+	const contradictions = [...new Set(left.filter((check) => rightSet.has(check)))].sort();
+	if (contradictions.length > 0) throw new Error(`${source} marks checks as both passed and failed: ${contradictions.join(", ")}`);
+}
+function parseSchema(schema, value, path, format) {
+	const parsed = schema.safeParse(value);
+	if (parsed.success) return parsed.data;
+	throw malformed(path, format, parsed.error.issues.map((issue) => `${issue.path.join(".") || "<root>"}: ${issue.message}`).join("; "));
+}
+function malformed(path, format, details) {
+	return /* @__PURE__ */ new TypeError(`malformed ${format} verification result: ${path}: ${details}`);
+}
+function asRecord(value) {
+	return typeof value === "object" && value !== null && !Array.isArray(value) ? value : null;
+}
+function status(value) {
+	return value ? "passed" : "failed";
+}
+function unsupported(path) {
+	return /* @__PURE__ */ new TypeError(`final verification result has no supported outcome field: ${path}; expected is_resolved, resolved, valid, or a SWE-bench instance report`);
+}
+function errorMessage(error) {
+	return error instanceof Error ? error.message : String(error);
+}
+//#endregion
+//#region src/analyst/benchmark-verification-artifacts.ts
+const DEFAULT_MAX_VERIFICATION_ARTIFACT_BYTES = 8 * 1024 * 1024;
+const SEARCHED_ARTIFACTS = {
+	"final-test-output": [
+		"panes/post-test.txt",
+		"sessions/tests.log",
+		"test_output.txt"
+	],
+	"final-result": [
+		"results.json",
+		"result.json",
+		"report.json",
+		"*_result.json"
+	],
+	"final-metrics": ["*_metrics.json"]
+};
+const REQUIRED_ROLES = /* @__PURE__ */ new Set(["final-result"]);
+const UTF8 = new TextDecoder$1("utf-8", { fatal: true });
+async function loadCodeTraceVerificationArtifacts(options) {
+	const maxBytes = positiveInteger(options.maxBytes ?? 8388608, "max verification artifact bytes");
+	const sourceRelativePath = nonEmpty(options.row.source_relpath, `CodeTraceBench '${options.row.traj_id}' source_relpath`);
+	const artifactRoot = await realpath(resolve(options.artifactDir));
+	const caseDirectoriesSearched = [resolve(artifactRoot, options.row.traj_id, sourceRelativePath), resolve(artifactRoot, sourceRelativePath)].filter((path, index, paths) => paths.indexOf(path) === index);
+	for (const path of caseDirectoriesSearched) assertContained(artifactRoot, path, sourceRelativePath);
+	const existingCaseDirectories = /* @__PURE__ */ new Set();
+	for (const path of caseDirectoriesSearched) try {
+		const canonicalPath = await realpath(path);
+		assertContained(artifactRoot, canonicalPath, sourceRelativePath);
+		if (!(await stat(canonicalPath)).isDirectory()) throw new TypeError(`CodeTraceBench '${options.row.traj_id}' artifact case path is not a directory: ${canonicalPath}`);
+		existingCaseDirectories.add(canonicalPath);
+	} catch (error) {
+		if (!isMissing(error)) throw error;
+	}
+	if (existingCaseDirectories.size === 0) return missingArtifacts(options.row.traj_id, caseDirectoriesSearched[0], caseDirectoriesSearched, maxBytes);
+	if (existingCaseDirectories.size > 1) throw new Error(`CodeTraceBench '${options.row.traj_id}' artifact directory is ambiguous: ${[...existingCaseDirectories].join(", ")}`);
+	const [caseDirectory] = existingCaseDirectories;
+	const candidates = await artifactCandidates(caseDirectory);
+	const files = [];
+	let totalBytes = 0;
+	for (const candidate of candidates) {
+		const { bytes, canonicalPath } = await readArtifactSnapshot({
+			artifactRoot,
+			candidatePath: candidate.path,
+			relativePath: candidate.relativePath,
+			traceId: options.row.traj_id,
+			totalBytes,
+			maxBytes
+		});
+		totalBytes += bytes.byteLength;
+		let content;
+		try {
+			content = UTF8.decode(bytes);
+		} catch {
+			throw new TypeError(`CodeTraceBench '${options.row.traj_id}' verification artifact is not UTF-8 text: ${canonicalPath}`);
+		}
+		if (!content.trim()) throw new Error(`CodeTraceBench '${options.row.traj_id}' verification artifact is empty: ${canonicalPath}`);
+		const relativePath = candidate.relativePath;
+		files.push({
+			role: candidate.role,
+			path: canonicalPath,
+			relativePath,
+			sha256: sha256Digest(bytes),
+			bytes: bytes.byteLength,
+			spanId: verificationSpanId(candidate.role, relativePath),
+			content
+		});
+	}
+	const roles = new Set(files.map((file) => file.role));
+	const missingRoles = Object.keys(SEARCHED_ARTIFACTS).filter((role) => !roles.has(role));
+	const hasFinalVerification = [...REQUIRED_ROLES].every((role) => roles.has(role));
+	const outcome = hasFinalVerification ? loadVerificationOutcome(files.filter((file) => file.role === "final-result").map((file) => ({
+		relativePath: file.relativePath,
+		content: file.content
+	})), options.row) : unavailableOutcome("missing-result");
+	const outcomeSpanId = verificationOutcomeSpanId(options.row.traj_id, outcome);
+	return {
+		manifest: {
+			traceId: options.row.traj_id,
+			status: hasFinalVerification ? "present" : "missing",
+			outcome,
+			outcomeSpanId,
+			caseDirectory,
+			caseDirectoriesSearched,
+			totalBytes,
+			maxBytes,
+			files: files.map(({ content: _content, ...file }) => file),
+			missingRoles,
+			searched: searchedArtifacts()
+		},
+		outcome,
+		files
+	};
+}
+function appendVerificationArtifactsToOtlp(otlpText, traceId, artifacts, afterTimestamp) {
+	if (!otlpText.trim()) throw new Error(`trace '${traceId}' OTLP input is empty`);
+	if (artifacts.manifest.traceId !== traceId) throw new Error(`verification artifacts for trace '${artifacts.manifest.traceId}' cannot be attached to '${traceId}'`);
+	if (!artifacts.outcome || !artifacts.manifest.outcomeSpanId) throw new Error(`trace '${traceId}' has no final verification artifacts to attach`);
+	const afterMs = Date.parse(afterTimestamp);
+	if (!Number.isFinite(afterMs)) throw new TypeError(`trace '${traceId}' latest timestamp is invalid: ${afterTimestamp}`);
+	const outcome = artifacts.outcome;
+	const outcomeLine = JSON.stringify({
+		trace_id: traceId,
+		span_id: artifacts.manifest.outcomeSpanId,
+		parent_span_id: null,
+		name: `final verification outcome: ${outcome.status}`,
+		start_time: timestampAfter(afterMs, 1, traceId),
+		end_time: timestampAfter(afterMs, 2, traceId),
+		status: { code: outcome.status === "passed" ? "STATUS_CODE_OK" : outcome.status === "failed" ? "STATUS_CODE_ERROR" : "STATUS_CODE_UNSET" },
+		resource: { attributes: { "service.name": "agent-eval-public-benchmark" } },
+		attributes: {
+			"openinference.span.kind": "EVALUATOR",
+			"benchmark.evidence.role": "final-verification",
+			"benchmark.verification.outcome": outcome.status,
+			"benchmark.verification.passed_check_count": outcome.passedCheckCount,
+			"benchmark.verification.failed_check_count": outcome.failedCheckCount,
+			"benchmark.verification.passed_checks": JSON.stringify(outcome.passedChecks),
+			"benchmark.verification.failed_checks": JSON.stringify(outcome.failedChecks),
+			"benchmark.verification.sources": JSON.stringify(outcome.sources),
+			...outcome.reason ? { "benchmark.verification.reason": outcome.reason } : {},
+			...outcome.parseError ? { "benchmark.verification.parse_error": JSON.stringify(outcome.parseError) } : {}
+		}
+	});
+	const artifactLines = artifacts.files.filter((artifact) => artifact.role === "final-test-output").map((artifact, index) => JSON.stringify({
+		trace_id: traceId,
+		span_id: artifact.spanId,
+		parent_span_id: null,
+		name: `final verification artifact: ${artifact.relativePath}`,
+		start_time: timestampAfter(afterMs, index * 2 + 3, traceId),
+		end_time: timestampAfter(afterMs, index * 2 + 4, traceId),
+		status: { code: "STATUS_CODE_UNSET" },
+		resource: { attributes: { "service.name": "agent-eval-public-benchmark" } },
+		attributes: {
+			"openinference.span.kind": "EVALUATOR",
+			"benchmark.evidence.role": "final-verification-artifact",
+			"benchmark.verification.outcome": outcome.status,
+			"artifact.role": artifact.role,
+			"artifact.path": artifact.relativePath,
+			"artifact.sha256": artifact.sha256,
+			"artifact.bytes": artifact.bytes,
+			"artifact.content": artifact.content
+		}
+	}));
+	return `${otlpText.trimEnd()}\n${[outcomeLine, ...artifactLines].join("\n")}\n`;
+}
+function timestampAfter(afterMs, offsetMs, traceId) {
+	const date = new Date(afterMs + offsetMs);
+	if (!Number.isFinite(date.getTime())) throw new RangeError(`trace '${traceId}' cannot place final verification after its latest span`);
+	return date.toISOString();
+}
+function sha256Digest(value) {
+	return createHash("sha256").update(value).digest("hex");
+}
+async function readArtifactSnapshot(options) {
+	const safeOpenFlags = constants.O_RDONLY | (process.platform === "win32" ? 0 : constants.O_NOFOLLOW | constants.O_NONBLOCK);
+	let handle;
+	try {
+		handle = await open(options.candidatePath, safeOpenFlags);
+	} catch (error) {
+		if (isNodeError$1(error, "ELOOP")) throw new Error(`CodeTraceBench '${options.traceId}' verification artifact must not be a symbolic link: ${options.relativePath}`);
+		throw error;
+	}
+	try {
+		const before = await handle.stat();
+		if (!before.isFile()) throw new TypeError(`CodeTraceBench '${options.traceId}' verification artifact is not a regular file: ${options.relativePath}`);
+		const bytes = checkedFileSize(before.size, options);
+		const descriptorPath = await openedDescriptorPath(handle.fd);
+		if (descriptorPath !== null) assertContained(options.artifactRoot, descriptorPath, options.relativePath);
+		const canonicalPath = await realpath(options.candidatePath);
+		assertContained(options.artifactRoot, canonicalPath, options.relativePath);
+		if (!sameFile(before, await stat(canonicalPath))) throw changedArtifact(options.traceId, options.relativePath);
+		const content = Buffer.allocUnsafe(bytes);
+		let offset = 0;
+		while (offset < content.byteLength) {
+			const { bytesRead } = await handle.read(content, offset, content.byteLength - offset, offset);
+			if (bytesRead === 0) break;
+			offset += bytesRead;
+		}
+		const eofProbe = Buffer.allocUnsafe(1);
+		const { bytesRead: trailingBytes } = await handle.read(eofProbe, 0, eofProbe.byteLength, content.byteLength);
+		const after = await handle.stat();
+		if (offset !== content.byteLength || trailingBytes !== 0 || !sameSnapshot(before, after)) throw changedArtifact(options.traceId, options.relativePath);
+		return {
+			canonicalPath,
+			bytes: content
+		};
+	} finally {
+		await handle.close();
+	}
+}
+function checkedFileSize(bytes, options) {
+	if (!Number.isSafeInteger(bytes) || bytes < 0) throw new RangeError(`CodeTraceBench '${options.traceId}' verification artifact has an invalid byte size: ${options.relativePath}`);
+	if (bytes > options.maxBytes) throw new RangeError(`CodeTraceBench '${options.traceId}' verification artifact '${options.relativePath}' requires ${bytes} bytes, over the ${options.maxBytes}-byte per-file limit`);
+	if (options.totalBytes > options.maxBytes - bytes) throw new RangeError(`CodeTraceBench '${options.traceId}' verification artifacts require ${options.totalBytes + bytes} bytes, over the ${options.maxBytes}-byte cumulative limit`);
+	return bytes;
+}
+async function openedDescriptorPath(fileDescriptor) {
+	if (process.platform !== "linux") return null;
+	try {
+		return await realpath(`/proc/self/fd/${fileDescriptor}`);
+	} catch (error) {
+		if (isNodeError$1(error, "ENOENT") || isNodeError$1(error, "ENOTDIR") || isNodeError$1(error, "EACCES")) return null;
+		throw error;
+	}
+}
+function sameFile(left, right) {
+	return left.isFile() && right.isFile() && left.dev === right.dev && left.ino === right.ino && left.size === right.size;
+}
+function sameSnapshot(left, right) {
+	return sameFile(left, right) && left.mode === right.mode && left.mtimeMs === right.mtimeMs && left.ctimeMs === right.ctimeMs;
+}
+function changedArtifact(traceId, relativePath) {
+	return /* @__PURE__ */ new Error(`CodeTraceBench '${traceId}' verification artifact changed while being read: ${relativePath}`);
+}
+async function artifactCandidates(caseDirectory) {
+	const rootFiles = (await readdir(caseDirectory, { withFileTypes: true })).filter((entry) => entry.isFile() || entry.isSymbolicLink()).map((entry) => entry.name);
+	const testOutput = await firstExisting(caseDirectory, SEARCHED_ARTIFACTS["final-test-output"]);
+	const finalResults = [...SEARCHED_ARTIFACTS["final-result"].filter((name) => !name.includes("*")).map((name) => resolve(caseDirectory, name)), ...rootFiles.filter((name) => name.endsWith("_result.json")).map((name) => resolve(caseDirectory, name))];
+	const finalMetrics = rootFiles.filter((name) => name.endsWith("_metrics.json")).map((name) => resolve(caseDirectory, name));
+	const candidates = [
+		...testOutput.map((path) => candidate("final-test-output", caseDirectory, path)),
+		...(await existing(finalResults)).map((path) => candidate("final-result", caseDirectory, path)),
+		...(await existing(finalMetrics)).map((path) => candidate("final-metrics", caseDirectory, path))
+	];
+	const seen = /* @__PURE__ */ new Set();
+	return candidates.filter((entry) => {
+		if (seen.has(entry.path)) return false;
+		seen.add(entry.path);
+		return true;
+	}).sort((left, right) => artifactRoleOrder(left.role) - artifactRoleOrder(right.role) || left.relativePath.localeCompare(right.relativePath));
+}
+async function firstExisting(caseDirectory, candidates) {
+	for (const relativePath of candidates) {
+		const path = resolve(caseDirectory, relativePath);
+		if (await isFile(path)) return [path];
+	}
+	return [];
+}
+async function existing(paths) {
+	const out = [];
+	for (const path of paths) if (await isFile(path)) out.push(path);
+	return out;
+}
+async function isFile(path) {
+	try {
+		return (await stat(path)).isFile();
+	} catch (error) {
+		if (isMissing(error)) return false;
+		throw error;
+	}
+}
+function candidate(role, caseDirectory, path) {
+	return {
+		role,
+		path,
+		relativePath: slashRelative$1(caseDirectory, path)
+	};
+}
+function missingArtifacts(traceId, caseDirectory, caseDirectoriesSearched, maxBytes) {
+	const outcome = unavailableOutcome("missing-result");
+	return {
+		manifest: {
+			traceId,
+			status: "missing",
+			outcome,
+			outcomeSpanId: verificationOutcomeSpanId(traceId, outcome),
+			caseDirectory,
+			caseDirectoriesSearched,
+			totalBytes: 0,
+			maxBytes,
+			files: [],
+			missingRoles: Object.keys(SEARCHED_ARTIFACTS),
+			searched: searchedArtifacts()
+		},
+		outcome,
+		files: []
+	};
+}
+function verificationOutcomeSpanId(traceId, outcome) {
+	return `benchmark-verification-outcome-${sha256Digest(`${traceId}\u0000${JSON.stringify(outcome.sources)}\u0000${outcome.status}`).slice(0, 16)}`;
+}
+function unavailableOutcome(reason) {
+	return {
+		status: "unavailable",
+		reason,
+		sources: [],
+		passedCheckCount: 0,
+		failedCheckCount: 0,
+		passedChecks: [],
+		failedChecks: []
+	};
+}
+function loadVerificationOutcome(files, row) {
+	try {
+		const outcome = parseVerificationOutcome(files);
+		if (outcome.status === "unavailable" || typeof row.solved !== "boolean") return outcome;
+		const labelStatus = row.solved ? "passed" : "failed";
+		if (outcome.status === labelStatus) return outcome;
+		return {
+			...outcome,
+			status: "unavailable",
+			reason: "result-label-disagreement",
+			parseError: {
+				class: "ResultLabelDisagreementError",
+				message: `CodeTraceBench '${row.traj_id}' solved=${row.solved} disagrees with parsed final verification status '${outcome.status}' from ${outcome.sources.map((source) => `${source.path}=${source.status}`).join(", ")}`
+			}
+		};
+	} catch (error) {
+		return {
+			...unavailableOutcome("result-parse-error"),
+			parseError: {
+				class: error instanceof Error ? error.constructor.name : "Error",
+				message: error instanceof Error ? error.message : String(error)
+			}
+		};
+	}
+}
+function verificationSpanId(role, relativePath) {
+	return `benchmark-verification-${sha256Digest(`${role}\u0000${relativePath}`).slice(0, 16)}`;
+}
+function artifactRoleOrder(role) {
+	return role === "final-test-output" ? 0 : role === "final-result" ? 1 : 2;
+}
+function assertContained(root, candidate, source) {
+	const rel = relative(root, candidate);
+	if (isAbsolute(rel) || rel === ".." || rel.startsWith(`..${sep}`)) throw new Error(`verification artifact path escapes --artifact-dir: ${source}`);
+}
+function searchedArtifacts() {
+	return {
+		"final-test-output": [...SEARCHED_ARTIFACTS["final-test-output"]],
+		"final-result": [...SEARCHED_ARTIFACTS["final-result"]],
+		"final-metrics": [...SEARCHED_ARTIFACTS["final-metrics"]]
+	};
+}
+function slashRelative$1(root, path) {
+	return relative(root, path).split(sep).join("/");
+}
+function nonEmpty(value, field) {
+	if (typeof value !== "string" || !value.trim()) throw new TypeError(`${field} must be a non-empty string`);
+	return value.trim();
+}
+function positiveInteger(value, field) {
+	if (!Number.isSafeInteger(value) || value < 1) throw new RangeError(`${field} must be a positive safe integer`);
+	return value;
+}
+function isMissing(error) {
+	return isNodeError$1(error, "ENOENT");
+}
+function isNodeError$1(error, code) {
+	return error instanceof Error && "code" in error && error.code === code;
+}
+//#endregion
+//#region src/analyst/benchmark-public-data.ts
+const DEFAULT_MAX_PUBLIC_BENCHMARK_LABEL_BYTES = 256 * 1024 * 1024;
+const INPUT_OPEN_FLAGS = constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0);
+async function loadPublicBenchmarkRows(path) {
+	return parsePublicBenchmarkRows((await readImmutableInputSnapshot(resolve(path), DEFAULT_MAX_PUBLIC_BENCHMARK_LABEL_BYTES)).text, path);
+}
+function parsePublicBenchmarkRows(text, path) {
+	const trimmed = text.trim();
+	if (!trimmed) throw new Error(`public analyst benchmark dataset is empty: ${path}`);
+	let parsed;
+	try {
+		parsed = JSON.parse(trimmed);
+	} catch {
+		return parseJsonl(trimmed, path);
+	}
+	if (Array.isArray(parsed)) return records(parsed, path);
+	if (isRecord(parsed) && Array.isArray(parsed.data)) return records(parsed.data, `${path}.data`);
+	if (isRecord(parsed) && Array.isArray(parsed.cases)) return records(parsed.cases, `${path}.cases`);
+	if (isRecord(parsed)) return [parsed];
+	throw new TypeError(`public analyst benchmark dataset must contain JSON objects: ${path}`);
+}
+function selectPublicBenchmarkRows(dataset, rows, options) {
+	positiveSafeInteger(options.limit, "limit");
+	safeInteger(options.seed, "seed");
+	if (rows.length === 0) throw new Error("public analyst benchmark dataset has no rows");
+	const byId = /* @__PURE__ */ new Map();
+	for (const row of rows) {
+		const id = publicBenchmarkRowId(dataset, row);
+		if (byId.has(id)) throw new Error(`public analyst benchmark dataset repeats trajectory id '${id}'`);
+		byId.set(id, row);
+	}
+	return [...byId].sort(([left], [right]) => selectionKey(options.seed, left).localeCompare(selectionKey(options.seed, right)) || left.localeCompare(right)).slice(0, Math.min(options.limit, byId.size)).map(([, row]) => row);
+}
+function publicBenchmarkDistributions(dataset, rows) {
+	const values = {
+		class: [],
+		agent: [],
+		model: [],
+		difficulty: [],
+		solved: []
+	};
+	for (const row of rows) {
+		const benchmarkCase = dataset === "agentrx" ? agentRxBenchmarkCase(row, void 0) : codeTraceBenchCase(row, void 0);
+		values.class.push(dataset === "codetracebench" ? benchmarkCase.expectedIssues.length > 0 ? "positive" : row.solved === true ? "trusted-negative" : row.solved === false ? "unlabeled-failure" : "unlabeled-unknown" : benchmarkCase.expectedIssues[0]?.areas?.[0]);
+		values.agent.push(scalarDistributionValue(row.agent) ?? (dataset === "agentrx" ? rootAgent(row) : void 0));
+		values.model.push(scalarDistributionValue(row.model));
+		values.difficulty.push(scalarDistributionValue(row.difficulty));
+		values.solved.push(scalarDistributionValue(row.solved));
+	}
+	return {
+		class: valueDistribution(values.class),
+		agent: valueDistribution(values.agent),
+		model: valueDistribution(values.model),
+		difficulty: valueDistribution(values.difficulty),
+		solved: valueDistribution(values.solved)
+	};
+}
+function publicBenchmarkSelectionReport(dataset, source, selected, seed) {
+	const census = source.length === selected.length;
+	return {
+		method: census ? "census" : "deterministic-hash",
+		seed,
+		sourceCount: source.length,
+		selectedCount: selected.length,
+		stratified: false,
+		representativeOfInput: census,
+		source: publicBenchmarkDistributions(dataset, source),
+		selected: publicBenchmarkDistributions(dataset, selected)
+	};
+}
+async function preparePublicAnalystBenchmark(options) {
+	const labelsPath = resolve(options.labelsPath);
+	const traceRoot = resolve(options.traceDir);
+	const artifactRoot = options.artifactDir ? resolve(options.artifactDir) : void 0;
+	const labelSnapshot = await readImmutableInputSnapshot(labelsPath, DEFAULT_MAX_PUBLIC_BENCHMARK_LABEL_BYTES);
+	const rows = parsePublicBenchmarkRows(labelSnapshot.text, labelsPath);
+	const selected = selectPublicBenchmarkRows(options.dataset, rows, {
+		limit: options.limit,
+		seed: options.seed
+	});
+	const stores = await indexSelectedSingleTraceFiles(traceRoot, new Set(selected.map((row) => publicBenchmarkRowId(options.dataset, row))));
+	const resolver = traceStoreEvidenceResolver((input) => {
+		if (!input.traceStore) throw new Error("prepared benchmark case has no trace store");
+		return input.traceStore;
+	});
+	const traceFiles = [];
+	const verificationArtifacts = [];
+	const cases = [];
+	for (const row of selected) {
+		const trajectoryId = publicBenchmarkRowId(options.dataset, row);
+		const indexed = stores.get(trajectoryId);
+		if (!indexed) throw new Error(`public analyst benchmark trace directory has no single-trace OTLP JSONL for '${trajectoryId}'`);
+		let modelVisibleOtlp = indexed.text;
+		let traceStore = indexed.store;
+		let artifactDir;
+		let verificationManifest;
+		if (options.dataset === "codetracebench") {
+			if (!options.artifactDir?.trim()) throw new Error("--artifact-dir is required for CodeTraceBench so final verification evidence is not omitted");
+			const artifacts = await loadCodeTraceVerificationArtifacts({
+				artifactDir: options.artifactDir,
+				row,
+				maxBytes: options.maxArtifactBytes ?? 8388608
+			});
+			for (const artifact of artifacts.files) assertNoBenchmarkLabelsInArtifact({
+				traceId: trajectoryId,
+				relativePath: artifact.relativePath,
+				content: artifact.content
+			});
+			verificationManifest = shareableVerificationManifest(artifacts.manifest, artifactRoot ?? resolve(options.artifactDir));
+			const collisions = await indexed.store.hasSpans({
+				trace_id: trajectoryId,
+				span_ids: [...artifacts.manifest.files.map((file) => file.spanId), artifacts.manifest.outcomeSpanId]
+			});
+			if (collisions.length > 0) throw new Error(`CodeTraceBench '${trajectoryId}' trace already contains benchmark verification span '${collisions[0]}'`);
+			modelVisibleOtlp = appendVerificationArtifactsToOtlp(indexed.text, trajectoryId, artifacts, indexed.latestTimestamp);
+			traceStore = otlpTextToTraceAnalysisStore(modelVisibleOtlp);
+			artifactDir = artifacts.manifest.status === "present" ? artifacts.manifest.caseDirectory : void 0;
+			verificationArtifacts.push(verificationManifest);
+		}
+		const labelLeakScan = assertNoBenchmarkLabelsInTrace({
+			traceId: trajectoryId,
+			otlpText: modelVisibleOtlp
+		});
+		const input = {
+			traceStore,
+			artifactDir
+		};
+		const benchmarkCase = options.dataset === "agentrx" ? agentRxBenchmarkCase(row, input, { stepCount: indexed.stepCount }) : codeTraceBenchCase(row, input);
+		for (const evidence of benchmarkCase.labeledEvidence ?? []) if (!await resolver({
+			caseId: benchmarkCase.id,
+			caseInput: input,
+			evidence: {
+				kind: evidence.kind ?? "span",
+				uri: evidence.uri
+			}
+		})) throw new Error(`${benchmarkCase.id}: missing labeled span ${spanIdFromEvidence(evidence.uri) ?? evidence.uri} in ${indexed.path}`);
+		cases.push({
+			...benchmarkCase,
+			metadata: {
+				...benchmarkCase.metadata,
+				traceFileRelativePath: slashRelative(traceRoot, indexed.path),
+				traceFileSha256: indexed.sha256,
+				labelLeakScan,
+				...verificationManifest ? { verificationArtifacts: verificationManifest } : {}
+			}
+		});
+		traceFiles.push({
+			traceId: trajectoryId,
+			relativePath: slashRelative(traceRoot, indexed.path),
+			sha256: indexed.sha256
+		});
+	}
+	return {
+		cases,
+		sourceRowCount: rows.length,
+		selectedCaseIds: cases.map((testCase) => testCase.id),
+		labelsSha256: labelSnapshot.sha256,
+		traceFiles,
+		verificationArtifacts,
+		selection: publicBenchmarkSelectionReport(options.dataset, rows, selected, options.seed)
+	};
+}
+async function indexSelectedSingleTraceFiles(traceDir, selectedTraceIds) {
+	if (selectedTraceIds.size === 0) throw new Error("public analyst benchmark selected no trace ids");
+	const files = (await readdir(traceDir, { withFileTypes: true })).filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")).map((entry) => resolve(traceDir, entry.name)).sort();
+	if (files.length === 0) throw new Error(`public analyst benchmark trace directory has no JSONL files: ${traceDir}`);
+	const indexed = /* @__PURE__ */ new Map();
+	for (const path of files) {
+		const snapshot = await readImmutableInputSnapshot(path, DEFAULT_MAX_TRACE_FILE_BYTES);
+		const store = createOtlpBufferTraceStore(snapshot.bytes);
+		const overview = await store.getOverview();
+		if (overview.total_traces !== 1 || overview.sample_trace_ids.length !== 1) throw new Error(`public analyst benchmark trace file must contain exactly one trace: ${path} contains ${overview.total_traces}`);
+		if (!overview.time_range) throw new Error(`public analyst benchmark trace file has no valid timestamps: ${path}`);
+		const traceId = overview.sample_trace_ids[0];
+		if (!selectedTraceIds.has(traceId)) continue;
+		if (indexed.has(traceId)) throw new Error(`public analyst benchmark trace id '${traceId}' appears in multiple files`);
+		indexed.set(traceId, {
+			path,
+			sha256: snapshot.sha256,
+			store,
+			latestTimestamp: overview.time_range.latest,
+			text: snapshot.text,
+			stepCount: traceStepCount(snapshot.text, path)
+		});
+	}
+	return indexed;
+}
+async function readImmutableInputSnapshot(path, maxBytes) {
+	if (!Number.isSafeInteger(maxBytes) || maxBytes < 1) throw new RangeError("benchmark input maxBytes must be a positive safe integer");
+	const handle = await open(path, INPUT_OPEN_FLAGS);
+	try {
+		return await readImmutableInputHandle(handle, path, maxBytes);
+	} finally {
+		await handle.close();
+	}
+}
+async function readImmutableInputHandle(handle, path, maxBytes) {
+	const before = await handle.stat({ bigint: true });
+	if (!before.isFile()) throw new TypeError(`public analyst benchmark input must be a regular file: ${path}`);
+	if (before.size > BigInt(maxBytes)) throw new RangeError(`public analyst benchmark input exceeds ${maxBytes} bytes: ${path} has ${before.size}`);
+	const size = Number(before.size);
+	const bytes = Buffer.allocUnsafe(size);
+	let offset = 0;
+	while (offset < size) {
+		const result = await handle.read(bytes, offset, size - offset, offset);
+		if (result.bytesRead === 0) throw new Error(`public analyst benchmark input changed while being read: ${path}`);
+		offset += result.bytesRead;
+	}
+	const overflow = Buffer.allocUnsafe(1);
+	const extra = await handle.read(overflow, 0, 1, size);
+	const after = await handle.stat({ bigint: true });
+	if (extra.bytesRead !== 0 || before.dev !== after.dev || before.ino !== after.ino || before.size !== after.size || before.mtimeNs !== after.mtimeNs || before.ctimeNs !== after.ctimeNs) throw new Error(`public analyst benchmark input changed while being read: ${path}`);
+	let text;
+	try {
+		text = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
+	} catch (error) {
+		throw new TypeError(`public analyst benchmark input is not valid UTF-8: ${path}: ${error instanceof Error ? error.message : String(error)}`);
+	}
+	return Object.freeze({
+		bytes,
+		sha256: sha256Digest(bytes),
+		text
+	});
+}
+function traceStepCount(text, path) {
+	const steps = parseJsonl(text, path).map((row) => row.span_id).filter((spanId) => typeof spanId === "string").map((spanId) => /^step-(\d+)$/.exec(spanId)?.[1]).filter((step) => step !== void 0).map(Number).filter((step) => Number.isSafeInteger(step) && step > 0);
+	if (steps.length === 0) throw new Error(`public analyst benchmark trace has no step-<n> spans: ${path}`);
+	return Math.max(...steps);
+}
+function shareableVerificationManifest(manifest, artifactRoot) {
+	return {
+		...manifest,
+		caseDirectory: slashRelative(artifactRoot, manifest.caseDirectory),
+		caseDirectoriesSearched: manifest.caseDirectoriesSearched.map((path) => slashRelative(artifactRoot, path)),
+		files: manifest.files.map((file) => ({
+			...file,
+			path: slashRelative(artifactRoot, file.path)
+		}))
+	};
+}
+function slashRelative(root, path) {
+	const value = relative(root, path);
+	if (!value || value === ".." || value.startsWith(`..${sep}`)) {
+		if (!value) return ".";
+		throw new Error(`benchmark artifact path escapes its declared root: ${path}`);
+	}
+	return value.replaceAll("\\", "/");
+}
+function parseJsonl(text, path) {
+	const rows = [];
+	for (const [index, line] of text.split(/\r?\n/).entries()) {
+		const trimmed = line.trim();
+		if (!trimmed) continue;
+		let parsed;
+		try {
+			parsed = JSON.parse(trimmed);
+		} catch (error) {
+			throw new Error(`${path}:${index + 1}: invalid JSON: ${error instanceof Error ? error.message : String(error)}`);
+		}
+		if (!isRecord(parsed)) throw new TypeError(`${path}:${index + 1}: dataset row must be a JSON object`);
+		rows.push(parsed);
+	}
+	if (rows.length === 0) throw new Error(`public analyst benchmark dataset is empty: ${path}`);
+	return rows;
+}
+function records(values, path) {
+	return values.map((value, index) => {
+		if (!isRecord(value)) throw new TypeError(`${path}[${index}] must be a JSON object`);
+		return value;
+	});
+}
+function publicBenchmarkRowId(dataset, row) {
+	const value = dataset === "agentrx" ? row.trajectory_id : row.traj_id;
+	if (typeof value !== "string" && typeof value !== "number" || !String(value).trim()) throw new TypeError(`${dataset} dataset row requires a non-empty ${dataset === "agentrx" ? "trajectory_id" : "traj_id"}`);
+	return String(value);
+}
+function spanIdFromEvidence(uri) {
+	const match = /\/span\/([^/]+)$/.exec(uri);
+	return match?.[1] ? decodeURIComponent(match[1]) : null;
+}
+function selectionKey(seed, id) {
+	return sha256Digest(`${seed}\u0000${id}`);
+}
+function valueDistribution(values) {
+	const counts = /* @__PURE__ */ new Map();
+	let missing = 0;
+	for (const value of values) {
+		if (value === void 0) {
+			missing += 1;
+			continue;
+		}
+		counts.set(value, (counts.get(value) ?? 0) + 1);
+	}
+	return {
+		total: values.length,
+		missing,
+		counts: Object.fromEntries([...counts].sort(([left], [right]) => left.localeCompare(right)))
+	};
+}
+function scalarDistributionValue(value) {
+	if (typeof value === "string") return value.trim() || void 0;
+	if (typeof value === "number" || typeof value === "boolean") return String(value);
+}
+function rootAgent(row) {
+	const rootCauseId = row.root_cause_failure_id ?? row.root_cause?.failure_id;
+	return scalarDistributionValue(row.failures.find((failure) => String(failure.failure_id) === String(rootCauseId))?.failed_agent);
+}
+//#endregion
+//#region src/analyst/benchmark-public-errors.ts
+function publicBenchmarkError(error, secrets = []) {
+	if (error instanceof LlmCallError) return {
+		class: "LlmCallError",
+		code: error.code,
+		status: error.status,
+		message: `Provider request failed with HTTP ${error.status}.`
+	};
+	if (error instanceof LlmResponseError) return {
+		class: "LlmResponseError",
+		code: error.code,
+		message: "Provider response did not satisfy the structured output contract."
+	};
+	if (error instanceof z.ZodError) return {
+		class: "ModelOutputValidationError",
+		message: "Provider response did not match the benchmark output schema."
+	};
+	if (error instanceof SyntaxError) return {
+		class: "ModelOutputParseError",
+		message: "Provider response was not valid JSON."
+	};
+	if (error instanceof CostCeilingReachedError || error instanceof CostAccountingIncompleteError || error instanceof CostReservationExceededError) return {
+		class: error.constructor.name,
+		code: error.code,
+		message: redactSensitiveText(error.message, secrets)
+	};
+	if (error instanceof Error && error.name === "AbortError") return {
+		class: "ProviderTimeoutError",
+		message: "Provider request timed out."
+	};
+	if (error instanceof Error && /(?:assistant steps?|finding evidence|selected missing|selected unavailable|no readable spans|requires a trace store)/i.test(error.message)) return {
+		class: "BenchmarkEvidenceError",
+		message: redactSensitiveText(error.message, secrets)
+	};
+	if (error instanceof AgentEvalError) return {
+		class: error.constructor.name,
+		code: error.code,
+		message: redactSensitiveText(error.message, secrets)
+	};
+	if (error instanceof Error) return {
+		class: error.constructor.name || "Error",
+		message: redactSensitiveText(error.message, secrets)
+	};
+	return {
+		class: "Error",
+		message: "Benchmark analyst execution failed."
+	};
+}
+function redactSensitiveText(value, secrets) {
+	let redacted = value;
+	for (const secret of secrets) if (secret) redacted = redacted.replaceAll(secret, "[REDACTED]");
+	return redacted.replace(/\bBearer\s+[^\s"',;]+/gi, "Bearer [REDACTED]").replace(/\b(api[_-]?key|access[_-]?token|refresh[_-]?token|password|secret)\b\s*[:=]\s*[^\s"',;]+/gi, "$1=[REDACTED]").slice(0, 500);
+}
+//#endregion
+//#region src/analyst/benchmark-response-cache.ts
+const SHA256 = /^[a-f0-9]{64}$/;
+const CostReceiptInputSchema = z.object({
+	model: z.string().min(1),
+	inputTokens: z.number().int().nonnegative(),
+	outputTokens: z.number().int().nonnegative(),
+	reasoningTokens: z.number().int().nonnegative().optional(),
+	cachedTokens: z.number().int().nonnegative().optional(),
+	cacheWriteTokens: z.number().int().nonnegative().optional(),
+	customTokenPricing: z.object({
+		inputUsdPerMillion: z.number().nonnegative(),
+		cachedInputUsdPerMillion: z.number().nonnegative().optional(),
+		cacheWriteUsdPerMillion: z.number().nonnegative().optional(),
+		outputUsdPerMillion: z.number().nonnegative()
+	}).strict().optional(),
+	actualCostUsd: z.number().nonnegative().optional(),
+	estimatedCostUsd: z.number().nonnegative().optional(),
+	costUnknown: z.boolean().optional(),
+	usageUnknown: z.boolean().optional()
+}).strict();
+const BenchmarkErrorSchema = z.object({
+	class: z.string().min(1),
+	message: z.string(),
+	code: z.string().min(1).optional(),
+	status: z.number().int().min(100).max(599).optional()
+}).strict();
+const ResponseMetadataSchema = z.object({
+	providerModel: z.string().min(1),
+	providerDurationMs: z.number().nonnegative(),
+	finishReason: z.string().nullable(),
+	producedAt: z.string().datetime()
+}).strict();
+const CacheIdentityShape = {
+	kind: z.literal("agent-eval/public-benchmark-model-response"),
+	callId: z.string().min(1),
+	runIdentitySha256: z.string().regex(SHA256),
+	caseId: z.string().min(1),
+	repetition: z.number().int().nonnegative()
+};
+const SuccessCacheEntryWithoutDigestSchema = z.object({
+	...CacheIdentityShape,
+	status: z.literal("succeeded"),
+	response: z.json(),
+	metadata: ResponseMetadataSchema,
+	receipt: CostReceiptInputSchema
+}).strict();
+const FailureCacheEntryWithoutDigestSchema = z.object({
+	...CacheIdentityShape,
+	status: z.literal("failed"),
+	error: BenchmarkErrorSchema,
+	receipt: CostReceiptInputSchema
+}).strict();
+const CacheEntryWithoutDigestSchema = z.discriminatedUnion("status", [SuccessCacheEntryWithoutDigestSchema, FailureCacheEntryWithoutDigestSchema]);
+const CacheEntrySchema = z.discriminatedUnion("status", [SuccessCacheEntryWithoutDigestSchema.extend({ entrySha256: z.string().regex(SHA256) }), FailureCacheEntryWithoutDigestSchema.extend({ entrySha256: z.string().regex(SHA256) })]);
+function publicBenchmarkCallId(identity) {
+	assertCacheIdentity(identity);
+	return `analyst-benchmark-${hashCanonical({
+		runIdentitySha256: identity.runIdentitySha256,
+		caseId: identity.caseId,
+		repetition: identity.repetition
+	}).slice(7)}`;
+}
+function readPublicBenchmarkResponseCache(cacheDirectory, identity) {
+	const callId = publicBenchmarkCallId(identity);
+	const path = responseCachePath(cacheDirectory, callId);
+	if (!existsSync(path)) return void 0;
+	const metadata = lstatSync(path);
+	if (!metadata.isFile() || metadata.isSymbolicLink()) throw new ValidationError(`benchmark response cache must be a real file: ${path}`);
+	let parsed;
+	try {
+		parsed = JSON.parse(readFileSync(path, "utf8"));
+	} catch (error) {
+		throw new ValidationError(`benchmark response cache contains invalid JSON: ${path}`, { cause: error });
+	}
+	const entry = parseCacheEntry(parsed, path);
+	if (entry.callId !== callId || entry.runIdentitySha256 !== identity.runIdentitySha256 || entry.caseId !== identity.caseId || entry.repetition !== identity.repetition) throw new ValidationError(`benchmark response cache identity does not match: ${path}`);
+	return entry;
+}
+function writePublicBenchmarkResponseCache(cacheDirectory, entry) {
+	const expectedCallId = publicBenchmarkCallId(entry);
+	if (entry.callId !== expectedCallId) throw new ValidationError("benchmark response cache callId does not match its identity");
+	const validated = JSON.parse(JSON.stringify(CacheEntryWithoutDigestSchema.parse(entry)));
+	const complete = {
+		...validated,
+		entrySha256: hashCanonical(validated).slice(7)
+	};
+	const path = responseCachePath(cacheDirectory, complete.callId);
+	const content = `${canonicalString(complete)}\n`;
+	withLedgerFileLock(path, fileContext(), () => {
+		if (existsSync(path)) {
+			const existing = readPublicBenchmarkResponseCache(cacheDirectory, complete);
+			if (!existing || canonicalString(existing) !== canonicalString(complete)) throw new ValidationError(`benchmark response cache conflicts with existing file: ${path}`);
+			return;
+		}
+		writeLedgerFileAtomically(path, content, fileContext());
+	});
+	return complete;
+}
+function parseCacheEntry(value, path) {
+	let parsed;
+	try {
+		parsed = CacheEntrySchema.parse(value);
+	} catch (error) {
+		throw new ValidationError(`benchmark response cache has an invalid shape: ${path}`, { cause: error });
+	}
+	const { entrySha256, ...withoutDigest } = parsed;
+	if (entrySha256 !== hashCanonical(withoutDigest).slice(7)) throw new ValidationError(`benchmark response cache digest does not match: ${path}`);
+	return parsed;
+}
+function responseCachePath(cacheDirectory, callId) {
+	const directory = nodePath.resolve(cacheDirectory);
+	const path = nodePath.resolve(directory, `${hashCanonical(callId).slice(7)}.json`);
+	if (!isPathInsideDirectory(directory, path, nodePath)) throw new ValidationError("benchmark response cache path escapes its directory");
+	return path;
+}
+function isPathInsideDirectory(directory, candidate, pathOperations = nodePath) {
+	const relative = pathOperations.relative(directory, candidate);
+	return relative !== "" && relative !== ".." && !relative.startsWith(`..${pathOperations.sep}`) && !pathOperations.isAbsolute(relative);
+}
+function assertCacheIdentity(identity) {
+	if (!SHA256.test(identity.runIdentitySha256)) throw new ValidationError("benchmark response cache requires a SHA-256 run identity");
+	if (!identity.caseId.trim()) throw new ValidationError("benchmark response cache requires a case id");
+	if (!Number.isSafeInteger(identity.repetition) || identity.repetition < 0) throw new ValidationError("benchmark response cache repetition must be non-negative");
+}
+function fileContext() {
+	return {
+		subject: "benchmark response cache",
+		integrityError: (message, options) => new ValidationError(message, options)
+	};
+}
+//#endregion
+//#region src/analyst/benchmark-public-model.ts
+const TRACE_PROJECTION_ATTRIBUTE_BYTE_CAPS = [
+	4096,
+	2048,
+	1024,
+	512,
+	256,
+	128,
+	64
+];
+function createPublicBenchmarkModelRunner(dataset, config) {
+	const model = requiredString(config.model, "model");
+	const baseUrl = requiredString(config.baseUrl, "baseUrl");
+	const apiKey = requiredString(config.apiKey, "apiKey");
+	const maxOutputTokens = positiveSafeInteger(config.maxOutputTokens, "maxOutputTokens");
+	const timeoutMs = positiveSafeInteger(config.timeoutMs, "timeoutMs");
+	const costLedger = config.costLedger ?? new CostLedger();
+	const durability = config.durability ? {
+		runIdentitySha256: requiredString(config.durability.runIdentitySha256, "durability.runIdentitySha256"),
+		responseCacheDir: requiredString(config.durability.responseCacheDir, "durability.responseCacheDir")
+	} : void 0;
+	const actor = dataset === "agentrx" ? "agentrx-root-cause-localizer" : "codetracebench-step-localizer";
+	const outputAdapter = dataset === "agentrx" ? "agentrx-taxonomy-and-root-step" : "codetracebench-incorrect-step";
+	const llmOptions = {
+		baseUrl,
+		apiKey,
+		maximumAttempts: 1,
+		jsonSchemaTransport: "json-object",
+		jsonPayloadMode: "exact",
+		thinking: "disabled",
+		...config.fetchImpl ? { fetch: config.fetchImpl } : {}
+	};
+	return {
+		id: "model",
+		async analyze(input, context) {
+			const trajectoryId = trajectoryIdFromCaseId(dataset, context.caseId);
+			const costTags = {
+				analystId: actor,
+				benchmarkCaseId: context.caseId,
+				benchmarkRepetition: String(context.repetition)
+			};
+			let rawPredictions = [];
+			let modelFindings = [];
+			let providerModel = model;
+			let producedAt;
+			let modelMetadata = {
+				analysisMode: "single-pass",
+				outputAdapter,
+				protocolSha256: publicBenchmarkProtocolSha256(dataset)
+			};
+			try {
+				if (!input.traceStore) throw new Error(`${dataset} model runner requires a trace store`);
+				const preparedContext = await prepareSingleTraceContext(input.traceStore, context);
+				if (preparedContext === void 0) throw new Error(`${dataset} trace '${trajectoryId}' has no readable spans`);
+				const request = {
+					model,
+					messages: [{
+						role: "system",
+						content: publicBenchmarkSystemPrompt(dataset)
+					}, {
+						role: "user",
+						content: `TRACE DATA:\n${preparedContext}\n\nReturn the analysis JSON object.`
+					}],
+					jsonMode: true,
+					thinking: "disabled",
+					maxTokens: maxOutputTokens,
+					timeoutMs
+				};
+				const cacheIdentity = durability ? {
+					runIdentitySha256: durability.runIdentitySha256,
+					caseId: context.caseId,
+					repetition: context.repetition
+				} : void 0;
+				const callId = cacheIdentity ? publicBenchmarkCallId(cacheIdentity) : void 0;
+				const cached = cacheIdentity ? readPublicBenchmarkResponseCache(durability.responseCacheDir, cacheIdentity) : void 0;
+				if (cached) {
+					const receipt = settleCachedResponse(costLedger, cached);
+					modelMetadata = {
+						...modelMetadata,
+						responseSource: "durable-cache",
+						cost: costReceiptMetadata(receipt)
+					};
+					if (cached.status === "failed") return {
+						findings: [],
+						usage: usageReceiptFromCostLedger(costLedger, {
+							channel: "analyst",
+							tags: costTags
+						}),
+						error: cached.error,
+						metadata: modelMetadata
+					};
+					const response = parsePublicBenchmarkModelResponse(dataset, cached.response);
+					rawPredictions = response.findings;
+					providerModel = cached.metadata.providerModel;
+					producedAt = cached.metadata.producedAt;
+					modelMetadata = {
+						...modelMetadata,
+						report: response.report,
+						providerModel: cached.metadata.providerModel,
+						providerDurationMs: cached.metadata.providerDurationMs,
+						finishReason: cached.metadata.finishReason
+					};
+				} else {
+					assertNoSettledResponseWithoutCache(costLedger, callId);
+					let completedResult;
+					const paid = await costLedger.runPaidCall({
+						...callId ? { callId } : {},
+						channel: "analyst",
+						phase: "analyst.public-benchmark",
+						actor,
+						model,
+						signal: context.signal,
+						maximumCharge: maximumChargeForLlmRequest(request, llmOptions),
+						tags: costTags,
+						execute: async (signal, providerCallId) => {
+							try {
+								const completed = await callLlmJson(request, {
+									...llmOptions,
+									signal,
+									idempotencyKey: providerCallId
+								});
+								completedResult = completed.result;
+								const response = parsePublicBenchmarkModelResponse(dataset, completed.value);
+								const responseProducedAt = (/* @__PURE__ */ new Date()).toISOString();
+								if (cacheIdentity) writePublicBenchmarkResponseCache(durability.responseCacheDir, {
+									kind: "agent-eval/public-benchmark-model-response",
+									...cacheIdentity,
+									callId: providerCallId,
+									status: "succeeded",
+									response,
+									metadata: {
+										providerModel: completed.result.model,
+										providerDurationMs: completed.result.durationMs,
+										finishReason: completed.result.finishReason ?? null,
+										producedAt: responseProducedAt
+									},
+									receipt: costReceiptFromLlm(completed.result)
+								});
+								return {
+									...completed,
+									response,
+									producedAt: responseProducedAt
+								};
+							} catch (error) {
+								if (cacheIdentity) writePublicBenchmarkResponseCache(durability.responseCacheDir, {
+									kind: "agent-eval/public-benchmark-model-response",
+									...cacheIdentity,
+									callId: providerCallId,
+									status: "failed",
+									error: publicBenchmarkError(error, [apiKey]),
+									receipt: receiptForProviderFailure(error, completedResult, model)
+								});
+								throw error;
+							}
+						},
+						receipt: ({ result }) => costReceiptFromLlm(result),
+						receiptFromError: (error) => receiptForProviderFailure(error, completedResult, model)
+					});
+					if (!paid.succeeded) throw paid.error;
+					const response = paid.value.response;
+					rawPredictions = response.findings;
+					providerModel = paid.value.result.model;
+					producedAt = paid.value.producedAt;
+					modelMetadata = {
+						...modelMetadata,
+						responseSource: "provider",
+						report: response.report,
+						providerModel: paid.value.result.model,
+						providerDurationMs: paid.value.result.durationMs,
+						finishReason: paid.value.result.finishReason ?? null,
+						cost: costReceiptMetadata(paid.receipt)
+					};
+				}
+				modelFindings = await publicBenchmarkPredictionsToFindings({
+					dataset,
+					trajectoryId,
+					predictions: rawPredictions,
+					store: input.traceStore,
+					analystId: "model",
+					providerModel,
+					producedAt: requiredString(producedAt ?? "", "finding producedAt"),
+					...context.signal ? { signal: context.signal } : {}
+				});
+				if (dataset === "codetracebench") await validateCodeTraceFindingEvidence({
+					trajectoryId,
+					findings: modelFindings,
+					store: input.traceStore,
+					...context.signal ? { signal: context.signal } : {}
+				});
+				return {
+					findings: modelFindings,
+					usage: usageReceiptFromCostLedger(costLedger, {
+						channel: "analyst",
+						tags: costTags
+					}),
+					metadata: modelMetadata
+				};
+			} catch (error) {
+				if (context.signal?.aborted) throw error;
+				if (isPaidCallControlError(error)) throw error;
+				return {
+					findings: [],
+					usage: usageReceiptFromCostLedger(costLedger, {
+						channel: "analyst",
+						tags: costTags
+					}),
+					error: publicBenchmarkError(error, [apiKey]),
+					metadata: {
+						...modelMetadata,
+						rawPredictions,
+						acceptedFindings: modelFindings
+					}
+				};
+			}
+		}
+	};
+}
+function settleCachedResponse(costLedger, cached) {
+	const settled = costLedger.list().find((receipt) => receipt.callId === cached.callId);
+	const pending = costLedger.listPending?.().find((record) => record.callId === cached.callId);
+	if (settled && pending) throw new CostCallConflictError(`benchmark response '${cached.callId}' is both pending and settled`, { callId: cached.callId });
+	const receipt = pending ? costLedger.reconcile(cached.callId, cached.receipt, { ...cached.status === "failed" ? { failed: true } : {} }) : settled;
+	if (!receipt) throw new CostCallConflictError(`benchmark response cache '${cached.callId}' has no matching cost record`, { callId: cached.callId });
+	assertCacheReceiptMatches(cached, receipt);
+	return receipt;
+}
+function assertNoSettledResponseWithoutCache(costLedger, callId) {
+	if (!callId) return;
+	if (costLedger.list().some((receipt) => receipt.callId === callId)) throw new CostCallConflictError(`settled benchmark call '${callId}' has no durable response cache`, { callId });
+}
+function assertCacheReceiptMatches(cached, receipt) {
+	const expected = cached.receipt;
+	if (receipt.callId !== cached.callId || receipt.model !== expected.model || receipt.inputTokens !== expected.inputTokens || receipt.outputTokens !== expected.outputTokens || (receipt.reasoningTokens ?? 0) !== (expected.reasoningTokens ?? 0) || (receipt.cachedTokens ?? 0) !== (expected.cachedTokens ?? 0) || (receipt.cacheWriteTokens ?? 0) !== (expected.cacheWriteTokens ?? 0) || expected.actualCostUsd !== void 0 && receipt.actualCostUsd !== expected.actualCostUsd || expected.estimatedCostUsd !== void 0 && receipt.estimatedCostUsd !== expected.estimatedCostUsd || expected.costUnknown === true && !receipt.costUnknown || expected.usageUnknown === true && !receipt.usageUnknown || cached.status === "succeeded" && receipt.error !== void 0 || cached.status === "failed" && receipt.error === void 0) throw new CostCallConflictError(`benchmark response cache receipt does not match cost record '${cached.callId}'`, {
+		callId: cached.callId,
+		receipt
+	});
+}
+function isPaidCallControlError(error) {
+	return error instanceof CostAccountingIncompleteError || error instanceof CostCallConflictError || error instanceof CostCeilingReachedError || error instanceof CostLedgerPersistenceError || error instanceof CostReceiptCaptureError || error instanceof CostReservationExceededError;
+}
+function receiptForProviderFailure(error, completedResult, model) {
+	if (completedResult) return costReceiptFromLlm(completedResult);
+	if (error instanceof Error) {
+		const captured = costReceiptFromLlmError(error);
+		if (captured) return captured;
+	}
+	return {
+		model,
+		inputTokens: 0,
+		outputTokens: 0,
+		costUnknown: true,
+		usageUnknown: true
+	};
+}
+function costReceiptMetadata(receipt) {
+	if (receipt.actualCostUsd !== void 0) return {
+		source: "provider",
+		actualCostUsd: receipt.actualCostUsd
+	};
+	if (receipt.estimatedCostUsd !== void 0) return {
+		source: "external-estimate",
+		estimatedCostUsd: receipt.estimatedCostUsd
+	};
+	if (receipt.pricing) return {
+		source: "agent-eval-model-pricing",
+		estimatedCostUsd: receipt.costUsd,
+		ratesPerThousandTokens: receipt.pricing
+	};
+	return {
+		source: "unknown",
+		estimatedCostUsd: null
+	};
+}
+const ModelSeveritySchema = z.enum([
+	"critical",
+	"high",
+	"medium",
+	"low",
+	"info"
+]);
+const PublicBenchmarkPredictionSchema = z.object({
+	step: z.number().int().positive(),
+	severity: ModelSeveritySchema,
+	claim: z.string().min(1),
+	confidence: z.number().min(0).max(1),
+	rationale: z.string().min(1).optional(),
+	recommended_action: z.string().min(1).optional()
+}).strict();
+const AgentRxCategorySchema = z.enum([
+	"instruction-plan-adherence-failure",
+	"invention-of-new-information",
+	"invalid-invocation",
+	"misinterpretation-of-tool-output-handoff-failure",
+	"intent-plan-misalignment",
+	"underspecified-user-intent",
+	"intent-not-supported",
+	"guardrails-triggered",
+	"system-failure",
+	"inconclusive"
+]);
+const CodeTraceModelResponseSchema = z.object({
+	report: z.string().min(1).max(4e3),
+	findings: z.array(PublicBenchmarkPredictionSchema).max(200)
+}).strict();
+const AgentRxModelResponseSchema = z.object({
+	report: z.string().min(1).max(4e3),
+	findings: z.array(PublicBenchmarkPredictionSchema.extend({ category: AgentRxCategorySchema }).strict()).max(1)
+}).strict();
+function parsePublicBenchmarkModelResponse(dataset, value) {
+	return dataset === "agentrx" ? AgentRxModelResponseSchema.parse(value) : CodeTraceModelResponseSchema.parse(value);
+}
+async function publicBenchmarkPredictionsToFindings(options) {
+	if (options.predictions.length === 0) return [];
+	const evidenceByStep = await resolveAssistantStepEvidence({
+		trajectoryId: options.trajectoryId,
+		steps: options.predictions.map((prediction) => prediction.step),
+		store: options.store,
+		...options.signal ? { signal: options.signal } : {}
+	});
+	if (options.dataset === "agentrx") {
+		const prediction = options.predictions[0];
+		if (!prediction.category) throw new Error("AgentRx model output is missing its failure category");
+		const [finding] = agentRxPredictionsToFindings(options.trajectoryId, [{
+			failure_case: prediction.category,
+			step_number: prediction.step,
+			description: prediction.rationale ?? prediction.claim
+		}], {
+			analystId: options.analystId,
+			producedAt: options.producedAt,
+			confidence: prediction.confidence
+		});
+		if (!finding) throw new Error("AgentRx output adapter produced no root-cause finding");
+		return [{
+			...finding,
+			evidence_refs: [evidenceByStep.get(prediction.step)],
+			metadata: {
+				...finding.metadata,
+				model: options.providerModel
+			}
+		}];
+	}
+	const byStep = /* @__PURE__ */ new Map();
+	for (const prediction of options.predictions) if (!byStep.has(prediction.step)) byStep.set(prediction.step, prediction);
+	return [...byStep].sort(([left], [right]) => left - right).map(([step, prediction]) => makeFinding({
+		analyst_id: options.analystId,
+		area: "incorrect",
+		subject: `incorrect-step-${step}`,
+		claim: `Step ${step} is incorrect. ${prediction.claim}`,
+		rationale: prediction.rationale,
+		severity: prediction.severity,
+		confidence: prediction.confidence,
+		evidence_refs: [evidenceByStep.get(step)],
+		recommended_action: prediction.recommended_action,
+		metadata: {
+			analysis_mode: "single-pass",
+			model: options.providerModel
+		},
+		produced_at: options.producedAt,
+		id_basis: `incorrect-step-${step}`
+	}));
+}
+function publicBenchmarkSystemPrompt(dataset) {
+	return `${dataset === "agentrx" ? AGENT_RX_PROMPT : CODE_TRACE_BENCH_ANALYST_PROMPT}
+
+Each finding must contain only:
+- "step": a positive integer matching an existing assistant LLM span named step-<n>
+- "severity": "critical", "high", "medium", "low", or "info"
+- "claim": one sentence
+- "confidence": a number from 0 through 1
+- optional "rationale" and "recommended_action" strings
+${dataset === "agentrx" ? `- "category": one allowed failure category listed above` : ""}
+
+Return exactly one JSON object with:
+- "report": a concise evidence-based explanation, at most 4000 characters
+- "findings": the strict finding array
+Use an empty findings array when the trace does not support a finding.
+Do not return a bare array, markdown, trace URIs, copied excerpts, or fields not listed above.
+The runner constructs exact trace URIs and action previews from each selected step.`;
+}
+function publicBenchmarkProtocolSha256(dataset) {
+	return sha256Digest(JSON.stringify({
+		dataset,
+		systemPrompt: publicBenchmarkSystemPrompt(dataset),
+		transport: {
+			attempts: 1,
+			jsonMode: true,
+			thinking: "disabled"
+		},
+		traceProjectionAttributeByteCaps: TRACE_PROJECTION_ATTRIBUTE_BYTE_CAPS,
+		evidence: {
+			location: "model-selected-positive-integer-assistant-step",
+			uri: "deterministic-trace-uri",
+			excerpt: `exact-action-prefix-512`
+		}
+	}));
+}
+async function prepareSingleTraceContext(store, context) {
+	const storeContext = context.signal ? { signal: context.signal } : void 0;
+	const overview = await store.getOverview(void 0, storeContext);
+	if (overview.total_traces !== 1 || overview.sample_trace_ids.length !== 1) throw new Error(`public model benchmark requires exactly one trace, received ${overview.total_traces}`);
+	const traceId = overview.sample_trace_ids[0];
+	for (const perAttributeByteCap of TRACE_PROJECTION_ATTRIBUTE_BYTE_CAPS) {
+		const viewed = await store.viewTrace({
+			trace_id: traceId,
+			per_attribute_byte_cap: perAttributeByteCap
+		}, storeContext);
+		if (!viewed.spans) continue;
+		return JSON.stringify({
+			trace_id: traceId,
+			per_attribute_byte_cap: perAttributeByteCap,
+			spans: viewed.spans
+		});
+	}
+}
+const AGENT_RX_PROMPT = `Analyze exactly one failed agent trajectory.
+Find the first unrecoverable critical failure, not every later symptom.
+Inspect the complete supplied trace data.
+Emit zero findings only when the trace does not contain enough evidence.
+Otherwise emit exactly one finding.
+Its category MUST be exactly one of:
+instruction-plan-adherence-failure
+invention-of-new-information
+invalid-invocation
+misinterpretation-of-tool-output-handoff-failure
+intent-plan-misalignment
+underspecified-user-intent
+intent-not-supported
+guardrails-triggered
+system-failure
+inconclusive
+Its step is the positive integer n from the first unrecoverable assistant span named step-<n>.`;
+const CODE_TRACE_BENCH_ANALYST_PROMPT = `Analyze exactly one coding-agent trajectory and its attached final verification.
+Your task is the CodeTraceBench incorrect-step task: identify every wrong state-changing action, bad hypothesis that drives an action, and regression.
+An incorrect step remains incorrect when the agent later recovers or the final verification passes.
+Inspect the complete supplied trace data.
+Use the final-verification outcome as evidence about the final state, not as a rule for whether earlier steps were incorrect.
+For each candidate, inspect the assistant action and its following observation.
+Label a failed command when the assistant caused it through a wrong action or unsupported hypothesis.
+Label the later corrective action only when that action is itself wrong.
+Do not label a diagnostic probe merely because it exposes an earlier defect.
+Do not label a redundant but correct read or search; CodeTraceBench scores unuseful steps separately, and this run scores incorrect steps only.
+Do not label a step solely because final verification failed.
+When final verification is unavailable, use only directly observed trajectory evidence.
+Emit one finding per incorrect assistant step.
+Each finding's step MUST be the positive integer n from an existing assistant LLM span named step-<n>.
+Never select an EVALUATOR, TOOL, CHAIN, final-verification, benchmark-verification, or message-<n> span.
+Before emitting a finding, inspect its candidate span's attributes.content and describe only the action shown there.
+When the trajectory has no incorrect steps, return an empty findings array.`;
+function trajectoryIdFromCaseId(dataset, caseId) {
+	const prefix = dataset === "agentrx" ? "agentrx:" : "codetrace:";
+	if (!caseId.startsWith(prefix) || caseId.length === prefix.length) throw new Error(`unexpected ${dataset} benchmark case id '${caseId}'`);
+	return caseId.slice(prefix.length);
+}
+//#endregion
+//#region src/analyst/benchmark-command-persistence.ts
+const ANALYST_BENCHMARK_INITIALIZATION_COMPLETE_FILE = "initialization-complete.json";
+async function openOutputDirectory(outDir, resume) {
+	const directory = resolve(outDir);
+	if (resume) {
+		let outputStat;
+		try {
+			outputStat = await lstat(directory);
+		} catch (error) {
+			if (isNodeError(error, "ENOENT")) throw new Error(`cannot resume missing benchmark output directory: ${directory}`);
+			throw error;
+		}
+		if (!outputStat.isDirectory() || outputStat.isSymbolicLink()) throw new Error(`benchmark output must be a real directory: ${directory}`);
+	} else {
+		await mkdir(dirname(directory), { recursive: true });
+		try {
+			await mkdir(directory);
+		} catch (error) {
+			if (isNodeError(error, "EEXIST")) throw new Error(`refusing to use existing benchmark output directory: ${directory}`);
+			throw error;
+		}
+		await syncDirectory(dirname(directory));
+	}
+	return {
+		directory,
+		initializationComplete: resolve(directory, ANALYST_BENCHMARK_INITIALIZATION_COMPLETE_FILE),
+		manifest: resolve(directory, ANALYST_BENCHMARK_MANIFEST_FILE),
+		observations: resolve(directory, ANALYST_BENCHMARK_OBSERVATIONS_FILE),
+		costLedger: resolve(directory, ANALYST_BENCHMARK_COST_LEDGER_FILE),
+		modelResponses: resolve(directory, "model-responses"),
+		localReceipt: resolve(directory, ANALYST_BENCHMARK_LOCAL_RECEIPT_FILE),
+		result: resolve(directory, "result.json"),
+		report: resolve(directory, "report.md")
+	};
+}
+async function prepareOutputLockPath(outDir) {
+	const directory = resolve(outDir);
+	await mkdir(dirname(directory), { recursive: true });
+	return `${directory}.lock`;
+}
+function createRunIdentity(config, prepared) {
+	const caseDefinitions = prepared.cases.map((testCase) => ({
+		id: testCase.id,
+		clusterId: testCase.clusterId,
+		labelState: testCase.labelState,
+		expectedIssues: testCase.expectedIssues,
+		labeledEvidence: testCase.labeledEvidence ?? [],
+		tags: testCase.tags ?? [],
+		metadata: testCase.metadata ?? {}
+	}));
+	return {
+		config: {
+			dataset: config.dataset,
+			datasetRevision: config.revision,
+			datasetSplit: config.split,
+			model: {
+				id: config.model.model,
+				maxOutputTokens: config.model.maxOutputTokens,
+				timeoutMs: config.model.timeoutMs
+			},
+			limit: config.limit,
+			seed: config.seed,
+			concurrency: config.concurrency,
+			repetitions: config.repetitions,
+			maxCostUsd: config.maxCostUsd,
+			maxArtifactBytes: config.maxArtifactBytes,
+			analystProtocolSha256: publicBenchmarkProtocolSha256(config.dataset),
+			implementationSha256: ANALYST_BENCHMARK_IMPLEMENTATION_SHA256,
+			dependencyLockSha256: ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256,
+			runnerIds: ["empty", "model"]
+		},
+		inputs: {
+			labelsSha256: prepared.labelsSha256,
+			sourceRowCount: prepared.sourceRowCount,
+			selectedCaseIds: [...prepared.selectedCaseIds],
+			traceFiles: prepared.traceFiles.map((traceFile) => ({ ...traceFile })),
+			verificationArtifactsSha256: digestCanonical(prepared.verificationArtifacts),
+			caseDefinitionsSha256: digestCanonical(caseDefinitions)
+		}
+	};
+}
+function createLocalRunReceipt(config, paths) {
+	return {
+		kind: "agent-eval/analyst-benchmark-local-run",
+		local: {
+			labelsPath: resolve(config.labelsPath),
+			traceDir: resolve(config.traceDir),
+			...config.artifactDir ? { artifactDir: resolve(config.artifactDir) } : {},
+			outputDir: paths.directory,
+			baseUrl: config.model.baseUrl,
+			apiKeyEnvironment: config.apiKeyEnv
+		},
+		command: config.command,
+		environment: {
+			node: process.version,
+			platform: platform(),
+			arch: arch()
+		},
+		files: {
+			manifest: paths.manifest,
+			observations: paths.observations,
+			costLedger: paths.costLedger,
+			modelResponses: paths.modelResponses,
+			result: paths.result,
+			report: paths.report
+		}
+	};
+}
+async function initializeRunFiles(paths, identity, identitySha256, localIdentitySha256, localReceiptInput) {
+	const existingManifest = await readOptionalRegularFile(paths.manifest, "benchmark run manifest");
+	const manifest = existingManifest ? await readAndValidateManifestContent(paths.manifest, existingManifest, identity, identitySha256, localIdentitySha256) : {
+		kind: "agent-eval/analyst-benchmark-run",
+		createdAt: (/* @__PURE__ */ new Date()).toISOString(),
+		identitySha256,
+		localIdentitySha256,
+		identity
+	};
+	const localReceipt = {
+		...localReceiptInput,
+		runIdentitySha256: identitySha256,
+		localIdentitySha256
+	};
+	const manifestContent = `${JSON.stringify(manifest, null, 2)}\n`;
+	const localReceiptContent = `${JSON.stringify(localReceipt, null, 2)}\n`;
+	const initializationCompleteContent = renderInitializationComplete(manifest);
+	await assertAbsentOrExact(paths.observations, "", "benchmark observation log");
+	await assertAbsentOrExact(paths.localReceipt, localReceiptContent, "benchmark local run receipt");
+	await assertAbsentOrExact(paths.manifest, manifestContent, "benchmark run manifest");
+	for (const path of [
+		paths.costLedger,
+		paths.modelResponses,
+		paths.result,
+		paths.report
+	]) if (await regularFileExists(path)) throw new Error(`benchmark initialization marker is missing but later run artifact exists: ${path}`);
+	if (await regularFileExists(paths.initializationComplete)) throw new Error(`benchmark initialization marker already exists during partial initialization: ${paths.initializationComplete}`);
+	await writeExclusiveOrVerify(paths.observations, "");
+	await writeExclusiveOrVerify(paths.localReceipt, localReceiptContent);
+	await writeExclusiveOrVerify(paths.manifest, manifestContent);
+	await writeExclusiveOrVerify(paths.initializationComplete, initializationCompleteContent);
+	return manifest;
+}
+async function readAndValidateResumeFiles(paths, currentIdentity, currentIdentitySha256, currentLocalIdentitySha256, localReceiptInput) {
+	if (!await regularFileExists(paths.initializationComplete)) return initializeRunFiles(paths, currentIdentity, currentIdentitySha256, currentLocalIdentitySha256, localReceiptInput);
+	const manifest = await readAndValidateManifest(paths.manifest, currentIdentity, currentIdentitySha256, currentLocalIdentitySha256);
+	const localReceiptContent = await readRegularFile(paths.localReceipt, "benchmark local run receipt");
+	const value = parseJson(localReceiptContent, paths.localReceipt);
+	if (!isRecord$1(value)) throw new TypeError(`benchmark local run receipt must be an object: ${paths.localReceipt}`);
+	assertExactKeys(value, [
+		"kind",
+		"runIdentitySha256",
+		"localIdentitySha256",
+		"local",
+		"command",
+		"environment",
+		"files"
+	], "benchmark local run receipt");
+	if (value.kind !== "agent-eval/analyst-benchmark-local-run" || value.runIdentitySha256 !== currentIdentitySha256 || value.localIdentitySha256 !== currentLocalIdentitySha256 || !isRecord$1(value.local)) throw new Error("benchmark local run receipt does not match the requested resume");
+	const expectedLocalReceipt = {
+		...localReceiptInput,
+		runIdentitySha256: currentIdentitySha256,
+		localIdentitySha256: currentLocalIdentitySha256
+	};
+	if (digestCanonical(value.local) !== currentLocalIdentitySha256 || canonicalJson(value.local) !== canonicalJson(localReceiptInput.local)) throw new Error("benchmark local paths or endpoint do not match the requested resume");
+	if (localReceiptContent !== `${JSON.stringify(expectedLocalReceipt, null, 2)}\n`) throw new Error(`benchmark local run receipt does not exactly match: ${paths.localReceipt}`);
+	if (await readRegularFile(paths.initializationComplete, "benchmark initialization marker") !== renderInitializationComplete(manifest)) throw new Error(`benchmark initialization marker does not match the run manifest: ${paths.initializationComplete}`);
+	return manifest;
+}
+async function readAndValidateManifest(path, currentIdentity, currentIdentitySha256, currentLocalIdentitySha256) {
+	const content = await readRegularFile(path, "benchmark run manifest");
+	const manifest = await readAndValidateManifestContent(path, content, currentIdentity, currentIdentitySha256, currentLocalIdentitySha256);
+	if (content !== `${JSON.stringify(manifest, null, 2)}\n`) throw new Error(`benchmark run manifest does not exactly match: ${path}`);
+	return manifest;
+}
+async function readAndValidateManifestContent(path, content, currentIdentity, currentIdentitySha256, currentLocalIdentitySha256) {
+	const value = parseJson(content, path);
+	if (!isRecord$1(value)) throw new TypeError(`benchmark run manifest must be an object: ${path}`);
+	assertExactKeys(value, [
+		"kind",
+		"createdAt",
+		"identitySha256",
+		"localIdentitySha256",
+		"identity"
+	], "benchmark run manifest");
+	if (value.kind !== "agent-eval/analyst-benchmark-run") throw new TypeError(`unsupported benchmark run manifest: ${path}`);
+	if (typeof value.createdAt !== "string" || !Number.isFinite(Date.parse(value.createdAt))) throw new TypeError(`benchmark run manifest has an invalid createdAt: ${path}`);
+	if (!isSha256(value.identitySha256) || !isSha256(value.localIdentitySha256) || !isRecord$1(value.identity)) throw new TypeError(`benchmark run manifest has an invalid identity: ${path}`);
+	if (digestCanonical(value.identity) !== value.identitySha256) throw new Error(`benchmark run manifest identity digest does not match its contents: ${path}`);
+	if (currentIdentitySha256 !== value.identitySha256 || currentLocalIdentitySha256 !== value.localIdentitySha256 || canonicalJson(currentIdentity) !== canonicalJson(value.identity)) throw new Error(`benchmark resume configuration or inputs do not match ${ANALYST_BENCHMARK_MANIFEST_FILE}`);
+	return {
+		kind: "agent-eval/analyst-benchmark-run",
+		createdAt: value.createdAt,
+		identitySha256: currentIdentitySha256,
+		localIdentitySha256: currentLocalIdentitySha256,
+		identity: currentIdentity
+	};
+}
+function renderInitializationComplete(manifest) {
+	return `${JSON.stringify({
+		kind: "agent-eval/analyst-benchmark-initialization-complete",
+		runIdentitySha256: manifest.identitySha256,
+		localIdentitySha256: manifest.localIdentitySha256,
+		createdAt: manifest.createdAt
+	}, null, 2)}\n`;
+}
+async function assertAbsentOrExact(path, expected, label) {
+	const existing = await readOptionalRegularFile(path, label);
+	if (existing !== void 0 && existing !== expected) throw new Error(`${label} does not exactly match interrupted initialization: ${path}`);
+}
+async function readOptionalRegularFile(path, label) {
+	if (!await regularFileExists(path)) return void 0;
+	return readRegularFile(path, label);
+}
+function createObservationAppender(path, runIdentitySha256, progress) {
+	let writes = Promise.resolve();
+	const seen = new Set(progress.observations.map(observationKey));
+	return (observation) => {
+		const write = writes.then(async () => {
+			assertAnalystBenchmarkObservation(observation, "benchmark observation");
+			const key = observationKey(observation);
+			if (seen.has(key)) throw new Error(`refusing duplicate benchmark observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}'`);
+			const rowWithoutDigest = {
+				sequence: progress.nextSequence,
+				runIdentitySha256,
+				previousRowSha256: progress.previousRowSha256,
+				observation
+			};
+			const row = {
+				...rowWithoutDigest,
+				rowSha256: digestCanonical(rowWithoutDigest)
+			};
+			await appendDurable(path, `${JSON.stringify(row)}\n`);
+			progress.nextSequence += 1;
+			progress.previousRowSha256 = row.rowSha256;
+			progress.observations.push(observation);
+			seen.add(key);
+		});
+		writes = write;
+		return write;
+	};
+}
+async function readProgress(path, runIdentitySha256, caseIds, repetitions) {
+	const rawLines = (await readRegularFile(path, "benchmark observation log")).split("\n");
+	if (rawLines.at(-1) === "") rawLines.pop();
+	const observations = [];
+	const seen = /* @__PURE__ */ new Set();
+	const executionIndexes = /* @__PURE__ */ new Set();
+	let previousRowSha256 = null;
+	const allowedCases = new Set(caseIds);
+	const plannedObservationCount = caseIds.length * 2 * repetitions;
+	for (const [index, line] of rawLines.entries()) {
+		if (!line.trim()) throw new Error(`benchmark observation log contains an empty row at line ${index + 1}`);
+		const parsed = parseJson(line, `${path}:${index + 1}`);
+		if (!isRecord$1(parsed)) throw new TypeError(`benchmark observation row ${index + 1} must be an object`);
+		assertExactKeys(parsed, [
+			"sequence",
+			"runIdentitySha256",
+			"previousRowSha256",
+			"observation",
+			"rowSha256"
+		], `benchmark observation row ${index + 1}`);
+		assertAnalystBenchmarkObservation(parsed.observation, `benchmark observation row ${index + 1}.observation`);
+		const observation = parsed.observation;
+		const key = observationKey(observation);
+		if (seen.has(key)) throw new Error(`duplicate benchmark observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}' at line ${index + 1}`);
+		if (parsed.sequence !== index) throw new Error(`benchmark observation row ${index + 1} has sequence ${String(parsed.sequence)}; expected ${index}`);
+		if (parsed.runIdentitySha256 !== runIdentitySha256) throw new Error(`benchmark observation row ${index + 1} belongs to another run`);
+		if (parsed.previousRowSha256 !== previousRowSha256) throw new Error(`benchmark observation row ${index + 1} breaks the digest chain`);
+		if (!isSha256(parsed.rowSha256)) throw new TypeError(`benchmark observation row ${index + 1} has an invalid digest`);
+		if (digestCanonical({
+			sequence: parsed.sequence,
+			runIdentitySha256: parsed.runIdentitySha256,
+			previousRowSha256: parsed.previousRowSha256,
+			observation
+		}) !== parsed.rowSha256) throw new Error(`benchmark observation row ${index + 1} digest does not match its contents`);
+		if (!allowedCases.has(observation.caseId) || observation.runnerId !== "empty" && observation.runnerId !== "model" || observation.repetition >= repetitions || observation.executionIndex >= plannedObservationCount) throw new Error(`benchmark observation row ${index + 1} does not match a planned case, runner, and repetition`);
+		if (executionIndexes.has(observation.executionIndex)) throw new Error(`duplicate benchmark executionIndex ${observation.executionIndex} at line ${index + 1}`);
+		observations.push(observation);
+		seen.add(key);
+		executionIndexes.add(observation.executionIndex);
+		previousRowSha256 = parsed.rowSha256;
+	}
+	return {
+		observations,
+		nextSequence: observations.length,
+		previousRowSha256
+	};
+}
+async function writeExclusiveOrVerify(path, content) {
+	try {
+		await writeExclusive(path, content);
+	} catch (error) {
+		if (!isNodeError(error, "EEXIST")) throw error;
+		if (await readRegularFile(path, "existing benchmark artifact") !== content) throw new Error(`refusing to replace existing benchmark artifact: ${path}`);
+	}
+}
+async function regularFileExists(path) {
+	try {
+		const fileStat = await lstat(path);
+		if (!fileStat.isFile() || fileStat.isSymbolicLink()) throw new Error(`benchmark artifact path must be a real file: ${path}`);
+		return true;
+	} catch (error) {
+		if (isNodeError(error, "ENOENT")) return false;
+		throw error;
+	}
+}
+async function appendDurable(path, content) {
+	const handle = await open(path, constants.O_APPEND | constants.O_WRONLY | constants.O_NOFOLLOW);
+	try {
+		await handle.writeFile(content, "utf8");
+		await handle.sync();
+	} finally {
+		await handle.close();
+	}
+}
+async function writeExclusive(path, content) {
+	const temporary = `${path}.tmp-${process.pid}-${randomUUID()}`;
+	let handle;
+	try {
+		handle = await open(temporary, "wx");
+		await handle.writeFile(content, "utf8");
+		await handle.sync();
+		await handle.close();
+		handle = void 0;
+		await link(temporary, path);
+		await syncDirectory(dirname(path));
+	} finally {
+		await handle?.close().catch(() => void 0);
+		await unlink(temporary).catch(() => void 0);
+	}
+}
+async function readRegularFile(path, label) {
+	let fileStat;
+	try {
+		fileStat = await lstat(path);
+	} catch (error) {
+		if (isNodeError(error, "ENOENT")) throw new Error(`${label} is missing: ${path}`);
+		throw error;
+	}
+	if (!fileStat.isFile() || fileStat.isSymbolicLink()) throw new Error(`${label} must be a real file: ${path}`);
+	return readFile(path, "utf8");
+}
+async function syncDirectory(path) {
+	const directory = await open(path, "r");
+	try {
+		await directory.sync();
+	} finally {
+		await directory.close();
+	}
+}
+function isNodeError(error, code) {
+	return error instanceof Error && "code" in error && error.code === code;
+}
+//#endregion
+//#region src/analyst/benchmark-comparison.ts
+function compareAnalystRunners(result, options) {
+	const confidence = options.confidence ?? .95;
+	const resamples = options.resamples ?? 2e3;
+	assertComparisonControls(confidence, resamples);
+	const runnerIds = new Set(result.summaries.map((summary) => summary.runnerId));
+	if (!runnerIds.has(options.baselineRunnerId)) throw new TypeError(`unknown baseline analyst runner '${options.baselineRunnerId}'`);
+	if (!runnerIds.has(options.candidateRunnerId)) throw new TypeError(`unknown candidate analyst runner '${options.candidateRunnerId}'`);
+	if (options.baselineRunnerId === options.candidateRunnerId) throw new TypeError("baseline and candidate analyst runners must be different");
+	const baseline = observationsByCase(result.observations, options.baselineRunnerId);
+	const candidate = observationsByCase(result.observations, options.candidateRunnerId);
+	const populationRepresentativenessProven = result.provenance.metadata?.populationRepresentativenessProven === true;
+	const metrics = METRICS.map((metric) => compareMetric({
+		metric,
+		baseline,
+		candidate,
+		confidence,
+		resamples,
+		seed: options.seed,
+		populationRepresentativenessProven
+	}));
+	return {
+		baselineRunnerId: options.baselineRunnerId,
+		candidateRunnerId: options.candidateRunnerId,
+		metrics
+	};
+}
+function compareMetric(options) {
+	const pairedCases = [];
+	let eligibleObservations = 0;
+	let pairedObservations = 0;
+	let baselineMissingObservations = 0;
+	let candidateMissingObservations = 0;
+	let asymmetricMissingObservations = 0;
+	const caseIds = /* @__PURE__ */ new Set([...options.baseline.keys(), ...options.candidate.keys()]);
+	for (const caseId of caseIds) {
+		const baselineByRepetition = new Map((options.baseline.get(caseId) ?? []).map((observation) => [observation.repetition, observation]));
+		const candidateByRepetition = new Map((options.candidate.get(caseId) ?? []).map((observation) => [observation.repetition, observation]));
+		const caseBefore = [];
+		const caseAfter = [];
+		let clusterId;
+		const repetitions = /* @__PURE__ */ new Set([...baselineByRepetition.keys(), ...candidateByRepetition.keys()]);
+		for (const repetition of repetitions) {
+			const baselineObservation = baselineByRepetition.get(repetition);
+			const candidateObservation = candidateByRepetition.get(repetition);
+			const identity = baselineObservation ?? candidateObservation;
+			if (!identity || !metricApplies(identity, options.metric)) continue;
+			if (baselineObservation && candidateObservation) assertSameCaseIdentity(baselineObservation, candidateObservation);
+			eligibleObservations += 1;
+			clusterId = identity.clusterId;
+			const baselineValue = baselineObservation ? metricValue(baselineObservation, options.metric) : null;
+			const candidateValue = candidateObservation ? metricValue(candidateObservation, options.metric) : null;
+			const baselineMissing = baselineValue === null;
+			const candidateMissing = candidateValue === null;
+			if (baselineMissing) baselineMissingObservations += 1;
+			if (candidateMissing) candidateMissingObservations += 1;
+			if (baselineMissing !== candidateMissing) asymmetricMissingObservations += 1;
+			if (baselineMissing || candidateMissing) continue;
+			caseBefore.push(baselineValue);
+			caseAfter.push(candidateValue);
+			pairedObservations += 1;
+		}
+		if (caseBefore.length === 0 || !clusterId) continue;
+		pairedCases.push({
+			clusterId,
+			baseline: mean$1(caseBefore),
+			candidate: mean$1(caseAfter)
+		});
+	}
+	const byCluster = /* @__PURE__ */ new Map();
+	for (const pairedCase of pairedCases) {
+		const rows = byCluster.get(pairedCase.clusterId) ?? [];
+		rows.push(pairedCase);
+		byCluster.set(pairedCase.clusterId, rows);
+	}
+	const before = [...byCluster.values()].map((rows) => mean$1(rows.map((row) => row.baseline)));
+	const after = [...byCluster.values()].map((rows) => mean$1(rows.map((row) => row.candidate)));
+	const interval = before.length === 0 ? null : pairedBootstrap(before, after, {
+		confidence: options.confidence,
+		resamples: options.resamples,
+		statistic: "mean",
+		seed: options.seed
+	});
+	const survivorOnly = pairedObservations < eligibleObservations;
+	const limitations = [];
+	if (!interval?.gateEligible) limitations.push("fewer-than-20-independent-clusters");
+	if (!options.populationRepresentativenessProven) limitations.push("population-representativeness-not-proven");
+	if (survivorOnly) limitations.push("missing-observations");
+	const comparison = {
+		metric: options.metric,
+		direction: LOWER_IS_BETTER.has(options.metric) ? "lower" : "higher",
+		pairedCases: pairedCases.length,
+		pairedClusters: before.length,
+		eligibleObservations,
+		pairedObservations,
+		baselineMissingObservations,
+		candidateMissingObservations,
+		asymmetricMissingObservations,
+		survivorOnly,
+		baselineMean: before.length === 0 ? null : mean$1(before),
+		candidateMean: after.length === 0 ? null : mean$1(after),
+		meanDelta: interval?.mean ?? null,
+		intervalLow: interval?.low ?? null,
+		intervalHigh: interval?.high ?? null,
+		confidence: options.confidence,
+		resamples: options.resamples,
+		minimumSampleMet: interval?.gateEligible ?? false,
+		populationInferenceEligible: limitations.length === 0,
+		inferenceLimitations: limitations
+	};
+	assertValidComparison(comparison);
+	return comparison;
+}
+const METRICS = [
+	"completion",
+	"issueRecall",
+	"findingPrecision",
+	"f1",
+	"criticalStepAccuracy",
+	"citationCoverage",
+	"citationExcerptCoverage",
+	"citationLabelAgreement",
+	"citationResolution",
+	"trustedNegativeAccuracy",
+	"latencyMs",
+	"calls",
+	"inputTokens",
+	"outputTokens",
+	"reasoningTokens",
+	"cachedTokens",
+	"cacheWriteTokens",
+	"costUsd"
+];
+const LOWER_IS_BETTER = /* @__PURE__ */ new Set([
+	"latencyMs",
+	"calls",
+	"inputTokens",
+	"outputTokens",
+	"reasoningTokens",
+	"cachedTokens",
+	"cacheWriteTokens",
+	"costUsd"
+]);
+function observationsByCase(observations, runnerId) {
+	const byCase = /* @__PURE__ */ new Map();
+	for (const observation of observations) {
+		if (observation.runnerId !== runnerId) continue;
+		const rows = byCase.get(observation.caseId) ?? [];
+		rows.push(observation);
+		byCase.set(observation.caseId, rows);
+	}
+	return byCase;
+}
+function assertSameCaseIdentity(baseline, candidate) {
+	if (baseline.clusterId !== candidate.clusterId || baseline.labelState !== candidate.labelState) throw new Error(`analyst comparison case identity differs for '${baseline.caseId}' repetition ${baseline.repetition}`);
+}
+function metricApplies(observation, metric) {
+	if (metric === "trustedNegativeAccuracy") return observation.labelState === "trusted-negative";
+	if (metric === "issueRecall" || metric === "findingPrecision" || metric === "f1") return observation.labelState === "positive";
+	if (metric === "criticalStepAccuracy") return observation.labelState === "positive" && observation.score.criticalStepAccuracy !== null;
+	return true;
+}
+function metricValue(observation, metric) {
+	if (metric === "completion") return observation.error ? 0 : 1;
+	if (metric === "latencyMs") return observation.latencyMs;
+	if (metric === "trustedNegativeAccuracy") {
+		if (observation.error) return 0;
+		return observation.score.predictionOnLabelEmptyCase ? 0 : 1;
+	}
+	if (observation.error && (metric === "issueRecall" || metric === "findingPrecision" || metric === "f1" || metric === "criticalStepAccuracy")) return 0;
+	if (observation.error && (metric === "citationCoverage" || metric === "citationExcerptCoverage" || metric === "citationLabelAgreement" || metric === "citationResolution")) return null;
+	if (metric === "issueRecall") return observation.score.issueRecall;
+	if (metric === "findingPrecision") return observation.score.findingPrecision;
+	if (metric === "f1") return observation.score.f1;
+	if (metric === "criticalStepAccuracy") return observation.score.criticalStepAccuracy;
+	if (metric === "citationCoverage") return observation.score.citationCoverage;
+	if (metric === "citationExcerptCoverage") return observation.score.citationExcerptCoverage;
+	if (metric === "citationLabelAgreement") return observation.score.citationLabelAgreement;
+	if (metric === "citationResolution") return observation.evidenceResolution?.validity ?? null;
+	if (metric === "calls") return observation.usage?.calls ?? null;
+	if (metric === "inputTokens") return observation.usage?.tokens?.input ?? null;
+	if (metric === "outputTokens") return observation.usage?.tokens?.output ?? null;
+	if (metric === "reasoningTokens") return observation.usage?.tokens?.reasoning ?? null;
+	if (metric === "cachedTokens") return observation.usage?.tokens?.cached ?? null;
+	if (metric === "cacheWriteTokens") return observation.usage?.tokens?.cacheWrite ?? null;
+	if (observation.usage?.cost.kind === "uncaptured") return null;
+	return observation.usage?.cost.usd ?? null;
+}
+function mean$1(values) {
+	return values.reduce((sum, value) => sum + value, 0) / values.length;
+}
+function assertComparisonControls(confidence, resamples) {
+	if (!Number.isSafeInteger(resamples) || resamples <= 0 || resamples > 1e6) throw new Error(`compareAnalystRunners: resamples must be a positive safe integer no greater than 1000000, got ${String(resamples)}`);
+	if (!Number.isFinite(confidence) || confidence <= 0 || confidence >= 1) throw new Error(`compareAnalystRunners: confidence must be a finite number in (0,1), got ${String(confidence)}`);
+}
+function assertValidComparison(comparison) {
+	if ([
+		"pairedCases",
+		"pairedClusters",
+		"eligibleObservations",
+		"pairedObservations",
+		"baselineMissingObservations",
+		"candidateMissingObservations",
+		"asymmetricMissingObservations",
+		"confidence",
+		"resamples"
+	].some((field) => !Number.isFinite(comparison[field])) || [
+		"baselineMean",
+		"candidateMean",
+		"meanDelta",
+		"intervalLow",
+		"intervalHigh"
+	].some((field) => comparison[field] !== null && !Number.isFinite(comparison[field]))) throw new Error(`compareAnalystRunners: ${comparison.metric} produced non-finite comparison output`);
+	if (comparison.intervalLow !== null && comparison.intervalHigh !== null && comparison.intervalLow > comparison.intervalHigh) throw new Error(`compareAnalystRunners: ${comparison.metric} produced an invalid confidence interval`);
+}
+//#endregion
+//#region src/analyst/benchmark-public-calibration.ts
+function summarizeCodeTraceCalibration(result) {
+	return {
+		protocol: "labeled-positive-and-solved-negative",
+		rationale: "Uses rows with incorrect-step labels as positives and solved label-empty rows as trusted negatives. Failed label-empty rows remain in the published result but are not treated as clean controls.",
+		runners: result.provenance.runnerIds.map((runnerId) => summarizeRunner(runnerId, result.observations.filter((observation) => observation.runnerId === runnerId)))
+	};
+}
+function renderCodeTraceCalibrationMarkdown(summary) {
+	return [
+		"## CodeTraceBench Calibrated View",
+		"",
+		summary.rationale,
+		"",
+		"| Runner | Completed/selected | Failed | Positive runs | Trusted negative runs | Unlabeled runs | Failed label-empty | Unknown label-empty | Matched/expected steps | Predicted steps | Precision | Recall | F1 | Official all-row F1 | Official rows | Trusted-negative false positives | Trusted-negative failures | Unlabeled predictions | Unlabeled failures |",
+		"| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
+		...summary.runners.map((runner) => `| ${escapeCell$1(runner.runnerId)} | ${runner.completedRuns}/${runner.selectedRuns} | ${runner.failedRuns} | ${runner.positiveRuns} | ${runner.trustedNegativeRuns} | ${runner.unlabeledRuns} | ${runner.failedLabelEmptyRuns} | ${runner.unknownLabelEmptyRuns} | ${runner.matchedIncorrectSteps}/${runner.expectedIncorrectSteps} | ${runner.predictedIncorrectSteps} | ${rate$1(runner.precision)} | ${rate$1(runner.recall)} | ${rate$1(runner.f1)} | ${rate$1(runner.officialAllRowF1)} | ${runner.officialAllRowRuns} | ${rate$1(runner.trustedNegativeFalsePositiveRate)} | ${rate$1(runner.trustedNegativeFailureRate)} | ${rate$1(runner.unlabeledPredictionRate)} | ${rate$1(runner.unlabeledFailureRate)} |`)
+	].join("\n");
+}
+function summarizeRunner(runnerId, observations) {
+	const positive = observations.filter((observation) => observation.labelState === "positive");
+	const trustedNegative = observations.filter((observation) => observation.labelState === "trusted-negative");
+	const excluded = observations.filter((observation) => observation.labelState === "unlabeled");
+	const selected = [...positive, ...trustedNegative];
+	const expected = sum(positive.map((observation) => observation.score.expectedIssueCount));
+	const predicted = sum(selected.map((observation) => observation.error ? 0 : observation.findings.length));
+	const matched = sum(positive.map((observation) => observation.score.matchedIssueIds.length));
+	const precision = predicted === 0 ? expected > 0 ? 0 : null : matched / predicted;
+	const recall = ratio(matched, expected);
+	const completedTrustedNegative = trustedNegative.filter((observation) => !observation.error);
+	const completedExcluded = excluded.filter((observation) => !observation.error);
+	const officialRows = observations.map(officialCodeTraceF1);
+	return {
+		runnerId,
+		selectedRuns: selected.length,
+		positiveRuns: positive.length,
+		trustedNegativeRuns: trustedNegative.length,
+		unlabeledRuns: excluded.length,
+		failedLabelEmptyRuns: excluded.filter((observation) => observation.caseMetadata?.solved === false).length,
+		unknownLabelEmptyRuns: excluded.filter((observation) => observation.caseMetadata?.solved !== false).length,
+		completedRuns: selected.filter((observation) => !observation.error).length,
+		failedRuns: selected.filter((observation) => observation.error).length,
+		expectedIncorrectSteps: expected,
+		predictedIncorrectSteps: predicted,
+		matchedIncorrectSteps: matched,
+		officialAllRowF1: mean(officialRows),
+		officialAllRowRuns: officialRows.length,
+		precision,
+		recall,
+		f1: harmonicMean(precision, recall),
+		trustedNegativeFalsePositiveRate: ratio(completedTrustedNegative.filter((observation) => observation.score.predictionOnLabelEmptyCase).length, completedTrustedNegative.length),
+		trustedNegativeFailureRate: ratio(trustedNegative.filter((observation) => observation.error).length, trustedNegative.length),
+		unlabeledPredictionRate: ratio(completedExcluded.filter((observation) => observation.findings.length > 0).length, completedExcluded.length),
+		unlabeledFailureRate: ratio(excluded.filter((observation) => Boolean(observation.error)).length, excluded.length)
+	};
+}
+function officialCodeTraceF1(observation) {
+	const trajectoryId = observation.caseMetadata?.trajectoryId;
+	if (typeof trajectoryId !== "string" || !trajectoryId.trim()) throw new TypeError(`${observation.caseId}: CodeTraceBench trajectoryId metadata is missing`);
+	const expected = new Set([...observation.score.matchedIssueIds, ...observation.score.missedIssueIds].map((issueId) => {
+		const match = /^incorrect:(\d+)$/.exec(issueId);
+		if (!match) throw new TypeError(`${observation.caseId}: invalid incorrect-step label '${issueId}'`);
+		return Number(match[1]);
+	}));
+	const predicted = /* @__PURE__ */ new Set();
+	if (!observation.error) for (const finding of observation.findings) {
+		if (finding.area !== "incorrect") continue;
+		for (const evidence of finding.evidence_refs) {
+			const location = codeTraceStepFromEvidence(evidence.uri);
+			if (!location || location.traceId !== trajectoryId) throw new TypeError(`${observation.caseId}: invalid CodeTraceBench prediction evidence '${evidence.uri}'`);
+			predicted.add(location.step);
+		}
+	}
+	let matched = 0;
+	for (const step of predicted) if (expected.has(step)) matched += 1;
+	return harmonicMean(predicted.size === 0 ? 0 : matched / predicted.size, expected.size === 0 ? 0 : matched / expected.size) ?? 0;
+}
+function sum(values) {
+	return values.reduce((total, value) => total + value, 0);
+}
+function ratio(numerator, denominator) {
+	return denominator === 0 ? null : numerator / denominator;
+}
+function mean(values) {
+	return values.length === 0 ? null : sum(values) / values.length;
+}
+function harmonicMean(left, right) {
+	if (left === null || right === null) return null;
+	return left + right === 0 ? 0 : 2 * left * right / (left + right);
+}
+function rate$1(value) {
+	return value === null ? "n/a" : value.toFixed(3);
+}
+function escapeCell$1(value) {
+	return value.replaceAll("|", "\\|").replaceAll("\n", " ");
+}
+//#endregion
+//#region src/analyst/benchmark-command-result.ts
+async function readAnalystBenchmarkArtifact(path) {
+	const value = parseJson(await readRegularFile(path, "analyst benchmark result"), path);
+	assertAnalystBenchmarkArtifact(value, "analyst benchmark result");
+	return value;
+}
+function assertCompletedArtifactMatchesRun(artifact, manifest, observations, prepared) {
+	if (artifact.runIdentitySha256 !== manifest.identitySha256) throw new Error("completed benchmark result belongs to another run");
+	assertSameObservations(artifact.result.observations, observations);
+	const expectedCount = manifest.identity.inputs.selectedCaseIds.length * manifest.identity.config.runnerIds.length * manifest.identity.config.repetitions;
+	if (observations.length !== expectedCount) throw new Error(`completed benchmark result has ${observations.length} observations; expected ${expectedCount}`);
+	const { config, inputs } = manifest.identity;
+	const verificationAvailability = {
+		cases: prepared.verificationArtifacts.length,
+		resultFilesPresent: prepared.verificationArtifacts.filter((artifact) => artifact.status === "present").length,
+		resultFilesMissing: prepared.verificationArtifacts.filter((artifact) => artifact.status === "missing").length,
+		outcomes: {
+			passed: prepared.verificationArtifacts.filter((artifact) => artifact.outcome.status === "passed").length,
+			failed: prepared.verificationArtifacts.filter((artifact) => artifact.outcome.status === "failed").length,
+			unavailable: prepared.verificationArtifacts.filter((artifact) => artifact.outcome.status === "unavailable").length
+		}
+	};
+	const expectedInputs = {
+		dataset: config.dataset,
+		datasetRevision: config.datasetRevision,
+		datasetSplit: config.datasetSplit,
+		labelsSha256: inputs.labelsSha256,
+		sourceRowCount: inputs.sourceRowCount,
+		traceFiles: inputs.traceFiles.map((traceFile) => ({ ...traceFile })),
+		verificationArtifacts: prepared.verificationArtifacts,
+		verificationAvailability,
+		selection: {
+			limit: config.limit,
+			seed: config.seed,
+			selectedCaseIds: [...inputs.selectedCaseIds],
+			report: prepared.selection
+		},
+		execution: {
+			repetitions: config.repetitions,
+			concurrency: config.concurrency,
+			model: config.model.id,
+			maxOutputTokens: config.model.maxOutputTokens,
+			timeoutMs: config.model.timeoutMs,
+			maxCostUsd: config.maxCostUsd,
+			maxArtifactBytes: config.maxArtifactBytes,
+			analystProtocolSha256: config.analystProtocolSha256,
+			implementationSha256: config.implementationSha256,
+			dependencyLockSha256: config.dependencyLockSha256
+		}
+	};
+	if (canonicalJson(artifact.inputs) !== canonicalJson(expectedInputs)) throw new Error("completed benchmark result inputs do not match the run manifest");
+	const provenance = artifact.result.provenance;
+	const expectedDatasetId = config.dataset === "agentrx" ? "microsoft/AgentRx" : "NJU-LINK/CodeTraceBench";
+	const expectedOutputAdapter = config.dataset === "agentrx" ? "agentrx-taxonomy-and-root-step" : "codetracebench-incorrect-step";
+	if (provenance.id !== `${config.dataset}-real-model-analyst` || provenance.startedAt !== manifest.createdAt || !Number.isFinite(Date.parse(provenance.endedAt)) || Date.parse(provenance.endedAt) < Date.parse(provenance.startedAt) || canonicalJson(provenance.dataset) !== canonicalJson({
+		id: expectedDatasetId,
+		revision: config.datasetRevision,
+		split: config.datasetSplit
+	}) || provenance.caseCount !== inputs.selectedCaseIds.length || canonicalJson(provenance.runnerIds) !== canonicalJson(config.runnerIds) || provenance.repetitions !== config.repetitions || provenance.maxConcurrency !== Math.min(config.concurrency, expectedCount) || provenance.runnerOrderSeed !== config.seed || provenance.metadata?.model !== config.model.id || provenance.metadata?.outputAdapter !== expectedOutputAdapter || provenance.metadata?.caseSelection !== prepared.selection.method || provenance.metadata?.caseSelectionSeed !== config.seed || provenance.metadata?.selectionStratified !== prepared.selection.stratified || provenance.metadata?.protocolSha256 !== config.analystProtocolSha256 || provenance.metadata?.implementationSha256 !== config.implementationSha256 || provenance.metadata?.dependencyLockSha256 !== config.dependencyLockSha256 || provenance.metadata?.populationRepresentativenessProven !== false) throw new Error("completed benchmark result provenance does not match the run manifest");
+	const expectedSummaries = config.runnerIds.map((runnerId) => summarizeAnalystBenchmarkRunner(runnerId, observations.filter((observation) => observation.runnerId === runnerId)));
+	if (canonicalJson(artifact.result.summaries) !== canonicalJson(expectedSummaries)) throw new Error("completed benchmark summaries do not match durable observations");
+	const expectedComparisons = [compareAnalystRunners(artifact.result, {
+		baselineRunnerId: "empty",
+		candidateRunnerId: "model",
+		seed: config.seed
+	})];
+	if (canonicalJson(artifact.comparisons) !== canonicalJson(expectedComparisons)) throw new Error("completed benchmark comparisons do not match durable observations");
+	if (config.dataset === "codetracebench") {
+		if (canonicalJson(artifact.codeTraceCalibration) !== canonicalJson(summarizeCodeTraceCalibration(artifact.result)) || artifact.agentRxCalibration !== void 0) throw new Error("completed CodeTraceBench calibration does not match durable observations");
+	} else if (canonicalJson(artifact.agentRxCalibration) !== canonicalJson(summarizeAgentRxCalibration(artifact.result, "f228165bfec60a801fd5fedd9d8ffe0f9de0c69d")) || artifact.codeTraceCalibration !== void 0) throw new Error("completed AgentRx calibration does not match durable observations");
+}
+function assertSameObservations(expected, actual) {
+	if (expected.length !== actual.length) throw new Error(`benchmark result has ${expected.length} observations but the durable log has ${actual.length}`);
+	const expectedByKey = new Map(expected.map((observation) => [observationKey(observation), canonicalJson(observation)]));
+	for (const observation of actual) {
+		const key = observationKey(observation);
+		if (expectedByKey.get(key) !== canonicalJson(observation)) throw new Error(`benchmark result does not match durable observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}'`);
+		expectedByKey.delete(key);
+	}
+	if (expectedByKey.size > 0) throw new Error("benchmark result is missing durable observations");
+}
+//#endregion
+//#region src/analyst/benchmark-report.ts
+function renderAnalystBenchmarkMarkdown(result, comparisons = []) {
+	const { provenance } = result;
+	const lines = [
+		"# Trace analyst benchmark",
+		"",
+		"## Run",
+		"",
+		"| Field | Value |",
+		"| --- | --- |",
+		`| Benchmark | ${escapeCell(provenance.id ?? "unspecified")} |`,
+		`| Dataset | ${escapeCell(provenance.dataset?.id ?? "unspecified")} |`,
+		`| Dataset revision | ${escapeCell(provenance.dataset?.revision ?? "unspecified")} |`,
+		`| Dataset split | ${escapeCell(provenance.dataset?.split ?? "unspecified")} |`,
+		`| Started | ${escapeCell(provenance.startedAt)} |`,
+		`| Ended | ${escapeCell(provenance.endedAt)} |`,
+		`| Cases | ${provenance.caseCount} |`,
+		`| Runners | ${escapeCell(provenance.runnerIds.join(", "))} |`,
+		`| Repetitions | ${provenance.repetitions} |`,
+		`| Maximum concurrency | ${provenance.maxConcurrency} |`,
+		`| Runner-order seed | ${provenance.runnerOrderSeed} |`,
+		`| Command | ${escapeCell(provenance.command ?? "uncaptured")} |`,
+		`| Environment | ${escapeCell(json(provenance.environment))} |`,
+		`| Metadata | ${escapeCell(json(provenance.metadata))} |`,
+		"",
+		"## Summary",
+		""
+	];
+	lines.push("| Runner | Runs | Failed | Issue-bearing | Trusted negatives | Unlabeled | Micro recall | Micro precision | Micro F1 | Macro recall | Macro precision | Macro F1 | Critical step | Citation coverage | Quote coverage | Label-location agreement | Citation resolution | Resolution unknown runs | Unresolved citations | Resolution errors | Trusted-negative false positives | Trusted-negative failures | Unlabeled prediction rate | Unlabeled failures | Prediction repeat | Prediction repeated cases | Matched-label repeat | Matched-label repeated cases | Latency min/mean/p50/p95/max ms | Locally timed runs | Runner-reported latency runs | Unknown latency | Calls | Input tokens | Output tokens | Reasoning tokens | Cached tokens | Cache-write tokens | Known cost USD | Unknown calls | Unknown input/output | Unknown reasoning | Unknown cached | Unknown cache-write | Unknown cost |", "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |");
+	for (const summary of result.summaries) lines.push(`| ${escapeCell(summary.runnerId)} | ${summary.completedRuns}/${summary.plannedRuns} | ${summary.failedRuns} | ${summary.issueBearingRuns} | ${summary.trustedNegativeRuns} | ${summary.unlabeledRuns} | ${optionalRate(summary.issueRecall)} | ${optionalRate(summary.findingPrecision)} | ${optionalRate(summary.f1)} | ${optionalRate(summary.macroIssueRecall)} | ${optionalRate(summary.macroFindingPrecision)} | ${optionalRate(summary.macroF1)} | ${optionalRate(summary.criticalStepAccuracy)} | ${optionalRate(summary.citationCoverage)} | ${optionalRate(summary.citationExcerptCoverage)} | ${optionalRate(summary.citationLabelAgreement)} | ${optionalRate(summary.citationResolution)} | ${summary.citationResolutionUnknownRuns} | ${summary.unresolvedCitations} | ${summary.citationResolutionErrors} | ${optionalRate(summary.trustedNegativeFalsePositiveRate)} | ${optionalRate(summary.trustedNegativeFailureRate)} | ${optionalRate(summary.unlabeledPredictionRate)} | ${optionalRate(summary.unlabeledFailureRate)} | ${optionalRate(summary.predictionAgreement)} | ${summary.predictionAgreementCases} | ${optionalRate(summary.matchedLabelAgreement)} | ${summary.matchedLabelAgreementCases} | ${latency(summary.latencyMs)} | ${summary.benchmarkClockLatencyRuns} | ${summary.runnerReportedLatencyRuns} | ${summary.latencyUnknownRuns} | ${summary.calls} | ${summary.inputTokens} | ${summary.outputTokens} | ${summary.reasoningTokens} | ${summary.cachedTokens} | ${summary.cacheWriteTokens} | ${summary.knownCostUsd.toFixed(6)} | ${summary.callsUnknownRuns} | ${summary.tokenUsageUnknownRuns} | ${summary.reasoningTokenUsageUnknownRuns} | ${summary.cachedTokenUsageUnknownRuns} | ${summary.cacheWriteTokenUsageUnknownRuns} | ${summary.costUnknownRuns} |`);
+	for (const comparison of comparisons) {
+		lines.push("", `## ${escapeCell(comparison.candidateRunnerId)} compared with ${escapeCell(comparison.baselineRunnerId)}`, "", "| Metric | Better direction | Paired cases | Independent clusters | Eligible observations | Paired observations | Missing baseline | Missing candidate | Missing asymmetry | Survivor-only | Baseline mean | Candidate mean | Delta | Interval | Minimum sample | Population inference | Limits |", "| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | ---: | ---: | ---: | --- | --- | --- | --- |");
+		for (const metric of comparison.metrics) lines.push(`| ${metric.metric} | ${metric.direction} | ${metric.pairedCases} | ${metric.pairedClusters} | ${metric.eligibleObservations} | ${metric.pairedObservations} | ${metric.baselineMissingObservations} | ${metric.candidateMissingObservations} | ${metric.asymmetricMissingObservations} | ${metric.survivorOnly ? "yes" : "no"} | ${optionalMetricNumber(metric.baselineMean)} | ${optionalMetricNumber(metric.candidateMean)} | ${optionalSigned(metric.meanDelta)} | ${interval(metric.intervalLow, metric.intervalHigh)} | ${metric.minimumSampleMet ? "yes" : "no"} | ${metric.populationInferenceEligible ? "yes" : "no"} | ${escapeCell(metric.inferenceLimitations.join(", ") || "none")} |`);
+	}
+	lines.push("", "## Runs", "", "| Runner | Case | Cluster | Label state | Tags | Case metadata | Runner metadata | Rep | Execution index | Completed | Recall | Precision | F1 | Critical step | Citation coverage | Quote coverage | Label-location agreement | Citation resolution | Prediction on label-empty case | Scored findings | Diagnostic findings | Unlabeled citations | Unresolved citations | Resolution errors | Latency ms | Latency source | Calls | Input tokens | Output tokens | Reasoning tokens | Cached tokens | Cache-write tokens | Cost USD | Known cost USD | Cost source | Error class | Error |", "| --- | --- | --- | --- | --- | --- | --- | ---: | ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- | --- |");
+	for (const observation of result.observations) {
+		const usage = observation.usage;
+		const cost = usage?.cost.kind === "uncaptured" ? null : usage?.cost.usd;
+		const positive = observation.labelState === "positive";
+		lines.push(`| ${escapeCell(observation.runnerId)} | ${escapeCell(observation.caseId)} | ${escapeCell(observation.clusterId)} | ${observation.labelState} | ${escapeCell(observation.caseTags.join(", "))} | ${escapeCell(json(observation.caseMetadata))} | ${escapeCell(json(observation.runnerMetadata))} | ${observation.repetition} | ${observation.executionIndex} | ${observation.error ? "no" : "yes"} | ${positive ? rate(observation.score.issueRecall) : "n/a"} | ${positive ? rate(observation.score.findingPrecision) : "n/a"} | ${positive ? rate(observation.score.f1) : "n/a"} | ${optionalRate(observation.score.criticalStepAccuracy)} | ${optionalRate(observation.score.citationCoverage)} | ${optionalRate(observation.score.citationExcerptCoverage)} | ${optionalRate(observation.score.citationLabelAgreement)} | ${optionalRate(observation.evidenceResolution?.validity ?? null)} | ${positive ? "n/a" : observation.score.predictionOnLabelEmptyCase ? "yes" : "no"} | ${observation.score.supportedFindingIndexes.length}/${observation.error ? 0 : observation.findings.length} | ${observation.error ? observation.findings.length : 0} | ${observation.score.unlabeledEvidence.length} | ${observation.evidenceResolution?.unresolvedEvidence.length ?? "unknown"} | ${observation.evidenceResolution?.errors.length ?? "unknown"} | ${optionalNumber(observation.latencyMs)} | ${observation.latencySource} | ${usage?.calls ?? "unknown"} | ${usage?.tokens?.input ?? "unknown"} | ${usage?.tokens?.output ?? "unknown"} | ${usage?.tokens?.reasoning ?? "unknown"} | ${usage?.tokens?.cached ?? "unknown"} | ${usage?.tokens?.cacheWrite ?? "unknown"} | ${cost === null || cost === void 0 ? "unknown" : cost.toFixed(6)} | ${usage?.knownCostUsd?.toFixed(6) ?? (cost === null || cost === void 0 ? "unknown" : cost.toFixed(6))} | ${usage?.cost.kind ?? "unknown"} | ${escapeCell(observation.error?.class ?? "")} | ${escapeCell(observation.error?.message ?? "")} |`);
+	}
+	return `${lines.join("\n")}\n`;
+}
+function rate(value) {
+	return `${(value * 100).toFixed(1)}%`;
+}
+function optionalRate(value) {
+	return value === null ? "n/a" : rate(value);
+}
+function number(value) {
+	return Number.isInteger(value) ? String(value) : value.toFixed(3);
+}
+function optionalNumber(value) {
+	return value === null ? "unknown" : number(value);
+}
+function optionalMetricNumber(value) {
+	return value === null ? "n/a" : number(value);
+}
+function signed(value) {
+	return `${value >= 0 ? "+" : ""}${number(value)}`;
+}
+function optionalSigned(value) {
+	return value === null ? "n/a" : signed(value);
+}
+function interval(low, high) {
+	return low === null || high === null ? "n/a" : `[${number(low)}, ${number(high)}]`;
+}
+function latency(value) {
+	if (value === null) return "unknown";
+	return [
+		value.min,
+		value.mean,
+		value.p50,
+		value.p95,
+		value.max
+	].map(number).join("/");
+}
+function json(value) {
+	return value === void 0 ? "uncaptured" : JSON.stringify(value);
+}
+function escapeCell(value) {
+	return value.replaceAll("|", "\\|").replaceAll("\n", " ");
+}
+//#endregion
+//#region src/analyst/benchmark-command.ts
+async function runAnalystBenchmarkCommand(argv, env = process.env, dependencies = {}) {
+	if (argv.includes("--help") || argv.includes("-h")) {
+		process.stdout.write(`${ANALYST_BENCHMARK_HELP}\n`);
+		return 0;
+	}
+	const config = parseCommandConfig(argv, env);
+	const outputLock = acquireSingleRunLock({ lockPath: await prepareOutputLockPath(config.outDir) });
+	try {
+		return await executeAnalystBenchmarkCommand(config, dependencies);
+	} finally {
+		outputLock.release();
+	}
+}
+async function executeAnalystBenchmarkCommand(config, dependencies) {
+	const paths = await openOutputDirectory(config.outDir, config.resume);
+	const prepared = await preparePublicAnalystBenchmark({
+		dataset: config.dataset,
+		labelsPath: config.labelsPath,
+		traceDir: config.traceDir,
+		artifactDir: config.artifactDir,
+		maxArtifactBytes: config.maxArtifactBytes,
+		limit: config.limit,
+		seed: config.seed
+	});
+	const identity = createRunIdentity(config, prepared);
+	const localReceipt = createLocalRunReceipt(config, paths);
+	const localIdentitySha256 = digestCanonical(localReceipt.local);
+	const identitySha256 = digestCanonical(identity);
+	const manifest = config.resume ? await readAndValidateResumeFiles(paths, identity, identitySha256, localIdentitySha256, localReceipt) : await initializeRunFiles(paths, identity, identitySha256, localIdentitySha256, localReceipt);
+	const progress = await readProgress(paths.observations, manifest.identitySha256, prepared.selectedCaseIds, config.repetitions);
+	const costLedger = createRunCostLedger({
+		storage: fsCampaignStorage(),
+		runDir: paths.directory,
+		costCeilingUsd: config.maxCostUsd
+	});
+	if (await regularFileExists(paths.result)) {
+		assertCostLedgerFinalizable(costLedger);
+		const artifact = await readAnalystBenchmarkArtifact(paths.result);
+		assertCompletedArtifactMatchesRun(artifact, manifest, progress.observations, prepared);
+		const markdown = renderArtifactMarkdown(artifact);
+		await writeExclusiveOrVerify(paths.report, markdown);
+		printSuccessSummary(artifact, paths);
+		return benchmarkExitCode(artifact.result);
+	}
+	if (await regularFileExists(paths.report)) throw new Error(`benchmark report exists without a completed result; refusing ambiguous resume: ${paths.report}`);
+	const createModelRunner = dependencies.createModelRunner ?? ((dataset, model) => createPublicBenchmarkModelRunner(dataset, model));
+	const runners = [emptyPublicBenchmarkRunner(), createModelRunner(config.dataset, {
+		...config.model,
+		costLedger,
+		durability: {
+			runIdentitySha256: manifest.identitySha256,
+			responseCacheDir: paths.modelResponses
+		}
+	})];
+	const appendObservation = createObservationAppender(paths.observations, manifest.identitySha256, progress);
+	const runAbort = new AbortController();
+	let result;
+	try {
+		result = await runAnalystBenchmark({
+			cases: prepared.cases,
+			runners,
+			repetitions: config.repetitions,
+			maxConcurrency: config.concurrency,
+			runnerOrderSeed: config.seed,
+			initialObservations: progress.observations,
+			signal: runAbort.signal,
+			onObservation: async (observation) => {
+				assertObservationAccountingComplete(observation, costLedger);
+				await appendObservation(observation);
+			},
+			resolveEvidence: traceStoreEvidenceResolver((input) => {
+				if (!input.traceStore) throw new Error("benchmark case has no trace store");
+				return input.traceStore;
+			}),
+			benchmark: {
+				id: `${config.dataset}-real-model-analyst`,
+				dataset: {
+					id: config.dataset === "agentrx" ? "microsoft/AgentRx" : "NJU-LINK/CodeTraceBench",
+					revision: config.revision,
+					split: config.split
+				},
+				environment: {
+					node: process.version,
+					platform: platform(),
+					arch: arch()
+				},
+				metadata: {
+					model: config.model.model,
+					outputAdapter: config.dataset === "agentrx" ? "agentrx-taxonomy-and-root-step" : "codetracebench-incorrect-step",
+					caseSelection: prepared.selection.method,
+					caseSelectionSeed: config.seed,
+					selectionStratified: prepared.selection.stratified,
+					protocolSha256: publicBenchmarkProtocolSha256(config.dataset),
+					implementationSha256: ANALYST_BENCHMARK_IMPLEMENTATION_SHA256,
+					dependencyLockSha256: ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256,
+					populationRepresentativenessProven: false
+				}
+			}
+		});
+	} catch (error) {
+		runAbort.abort(error);
+		if (!await costLedger.waitForIdle({ timeoutMs: Math.min(config.model.timeoutMs, 1e4) })) throw accountingError(costLedger, "provider calls remain unresolved after cancellation");
+		throw error;
+	}
+	assertCostLedgerFinalizable(costLedger);
+	result.provenance.startedAt = manifest.createdAt;
+	const persisted = await readProgress(paths.observations, manifest.identitySha256, prepared.selectedCaseIds, config.repetitions);
+	assertSameObservations(result.observations, persisted.observations);
+	const comparisons = [compareAnalystRunners(result, {
+		baselineRunnerId: "empty",
+		candidateRunnerId: "model",
+		seed: config.seed
+	})];
+	const codeTraceCalibration = config.dataset === "codetracebench" ? summarizeCodeTraceCalibration(result) : void 0;
+	const agentRxCalibration = config.dataset === "agentrx" ? summarizeAgentRxCalibration(result, AGENT_RX_UPSTREAM_REVISION) : void 0;
+	const artifact = {
+		kind: "agent-eval/analyst-benchmark-result",
+		runIdentitySha256: manifest.identitySha256,
+		inputs: {
+			dataset: config.dataset,
+			datasetRevision: config.revision,
+			datasetSplit: config.split,
+			labelsSha256: prepared.labelsSha256,
+			sourceRowCount: prepared.sourceRowCount,
+			traceFiles: prepared.traceFiles,
+			verificationArtifacts: prepared.verificationArtifacts,
+			verificationAvailability: summarizeVerificationAvailability(prepared.verificationArtifacts),
+			selection: {
+				limit: config.limit,
+				seed: config.seed,
+				selectedCaseIds: prepared.selectedCaseIds,
+				report: prepared.selection
+			},
+			execution: {
+				repetitions: config.repetitions,
+				concurrency: config.concurrency,
+				model: config.model.model,
+				maxOutputTokens: config.model.maxOutputTokens,
+				timeoutMs: config.model.timeoutMs,
+				maxCostUsd: config.maxCostUsd,
+				maxArtifactBytes: config.maxArtifactBytes,
+				analystProtocolSha256: publicBenchmarkProtocolSha256(config.dataset),
+				implementationSha256: ANALYST_BENCHMARK_IMPLEMENTATION_SHA256,
+				dependencyLockSha256: ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256
+			}
+		},
+		result,
+		comparisons,
+		...codeTraceCalibration ? { codeTraceCalibration } : {},
+		...agentRxCalibration ? { agentRxCalibration } : {}
+	};
+	const markdown = renderArtifactMarkdown(artifact);
+	await writeExclusiveOrVerify(paths.result, `${JSON.stringify(artifact, null, 2)}\n`);
+	await writeExclusiveOrVerify(paths.report, markdown);
+	printSuccessSummary(artifact, paths);
+	return benchmarkExitCode(result);
+}
+const NON_SCORABLE_COST_ERRORS = /* @__PURE__ */ new Set([
+	"CostAccountingIncompleteError",
+	"CostCallConflictError",
+	"CostCeilingReachedError",
+	"CostLedgerPersistenceError",
+	"CostReceiptCaptureError",
+	"CostReservationExceededError"
+]);
+function assertObservationAccountingComplete(observation, costLedger) {
+	if (observation.error && NON_SCORABLE_COST_ERRORS.has(observation.error.class)) throw new CostAccountingIncompleteError(`Analyst benchmark stopped before scoring: ${observation.error.message}`);
+	if (observation.runnerId !== "model") return;
+	const summary = costLedger.summary({
+		channel: "analyst",
+		tags: {
+			benchmarkCaseId: observation.caseId,
+			benchmarkRepetition: String(observation.repetition)
+		}
+	});
+	if (!summary.accountingComplete || summary.pendingCalls > 0 || summary.unresolvedCalls > 0) throw accountingError(costLedger, "the model observation has incomplete cost accounting", {
+		channel: "analyst",
+		tags: {
+			benchmarkCaseId: observation.caseId,
+			benchmarkRepetition: String(observation.repetition)
+		}
+	});
+}
+function assertCostLedgerFinalizable(costLedger) {
+	const summary = costLedger.summary();
+	if (!summary.accountingComplete || summary.pendingCalls > 0 || summary.unresolvedCalls > 0) throw accountingError(costLedger, "the run has pending or incomplete cost entries");
+}
+function accountingError(costLedger, reason, filter) {
+	const details = costLedger.summary(filter).incompleteReasons.slice(0, 3).join("; ");
+	return new CostAccountingIncompleteError(`Analyst benchmark cannot continue because ${reason}${details ? `: ${details}` : ""}`);
+}
+const ANALYST_BENCHMARK_HELP = `agent-eval analyst-benchmark
+
+Run a real-model trace analyst against public AgentRx or CodeTraceBench labels.
+
+Required:
+  --dataset agentrx|codetracebench
+  --labels <dataset.json|dataset.jsonl>
+  --trace-dir <one-trace-per-file OTLP JSONL directory>
+  --artifact-dir <extracted artifact root>  Required for CodeTraceBench
+  --out <new output directory>
+  --revision <full 40- or 64-character hex digest>
+  --split <dataset split>
+  --base-url <OpenAI-compatible /v1 URL>
+  --api-key-env <environment variable containing the bearer>
+  --model <provider model id>
+  --limit <positive case count>
+
+Controls:
+  --resume                         Continue an interrupted run in --out
+  --seed <integer>                 Case-selection and comparison seed. Default: 0
+  --concurrency <positive integer> Parallel benchmark jobs. Default: 1
+  --repetitions <positive integer> Runs per case and runner. Default: 1
+  --max-output-tokens <positive>   Model output limit per call. Default: 4096
+  --timeout-ms <positive>          Model analyst deadline per case. Default: 300000
+  --max-cost-usd <positive>        Run-wide spend limit. Default: 5
+  --max-artifact-bytes <positive>  Final evidence bytes per case. Default: ${DEFAULT_MAX_VERIFICATION_ARTIFACT_BYTES}
+
+Writes result.json with every observation, metric, usage field, error, comparison,
+input digest, artifact digest, case distribution, selected case id, and explicit
+unknown cost. Limited deterministic-hash subsets are marked non-representative.
+Completed observations are fsynced to observations.jsonl. Shareable output is in
+result.json and report.md. Machine-local paths, endpoint, and command are isolated
+in run.local.json.
+The key is read from the named environment variable and is never written.`;
+function parseCommandConfig(argv, env) {
+	const flags = parseFlags(argv);
+	assertKnownFlags(flags);
+	const dataset = requiredFlag(flags, "dataset");
+	if (dataset !== "agentrx" && dataset !== "codetracebench") throw new Error("--dataset must be 'agentrx' or 'codetracebench'");
+	const artifactDir = flags.get("artifact-dir")?.trim();
+	if (dataset === "codetracebench" && !artifactDir) throw new Error("--artifact-dir is required for CodeTraceBench");
+	const apiKeyEnv = requiredFlag(flags, "api-key-env");
+	if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(apiKeyEnv)) throw new Error("--api-key-env must be a valid environment variable name");
+	const apiKey = env[apiKeyEnv]?.trim();
+	if (!apiKey) throw new Error(`--api-key-env points to an empty or missing variable: ${apiKeyEnv}`);
+	return {
+		dataset,
+		labelsPath: requiredFlag(flags, "labels"),
+		traceDir: requiredFlag(flags, "trace-dir"),
+		...artifactDir ? { artifactDir } : {},
+		outDir: requiredFlag(flags, "out"),
+		revision: immutableRevision(requiredFlag(flags, "revision")),
+		split: requiredFlag(flags, "split"),
+		model: {
+			baseUrl: openAiCompatibleBaseUrl(requiredFlag(flags, "base-url")),
+			apiKey,
+			model: requiredFlag(flags, "model"),
+			maxOutputTokens: positiveFlag(flags, "max-output-tokens", 4096),
+			timeoutMs: positiveFlag(flags, "timeout-ms", 3e5)
+		},
+		limit: positiveFlag(flags, "limit"),
+		seed: integerFlag(flags, "seed", 0),
+		concurrency: positiveFlag(flags, "concurrency", 1),
+		repetitions: positiveFlag(flags, "repetitions", 1),
+		maxCostUsd: positiveFiniteFlag(flags, "max-cost-usd", 5),
+		maxArtifactBytes: positiveFlag(flags, "max-artifact-bytes", DEFAULT_MAX_VERIFICATION_ARTIFACT_BYTES),
+		apiKeyEnv,
+		command: `agent-eval analyst-benchmark ${argv.filter((argument) => argument !== "--resume").map(shellQuote).join(" ")}`,
+		resume: flags.has("resume")
+	};
+}
+function parseFlags(argv) {
+	const flags = /* @__PURE__ */ new Map();
+	for (let index = 0; index < argv.length; index += 1) {
+		const token = argv[index];
+		if (!token.startsWith("--")) throw new Error(`unexpected positional argument: ${token}`);
+		const raw = token.slice(2);
+		const equalsAt = raw.indexOf("=");
+		const name = equalsAt < 0 ? raw : raw.slice(0, equalsAt);
+		const inlineValue = equalsAt < 0 ? void 0 : raw.slice(equalsAt + 1);
+		if (!name || flags.has(name)) throw new Error(`duplicate or empty flag: --${name}`);
+		if (BOOLEAN_FLAGS.has(name)) {
+			if (inlineValue !== void 0) throw new Error(`--${name} does not accept a value`);
+			flags.set(name, "true");
+			continue;
+		}
+		const value = inlineValue ?? argv[++index];
+		if (!value || value.startsWith("--")) throw new Error(`--${name} requires a value`);
+		flags.set(name, value);
+	}
+	return flags;
+}
+const KNOWN_FLAGS = /* @__PURE__ */ new Set([
+	"resume",
+	"dataset",
+	"labels",
+	"trace-dir",
+	"artifact-dir",
+	"out",
+	"revision",
+	"split",
+	"base-url",
+	"api-key-env",
+	"model",
+	"limit",
+	"seed",
+	"concurrency",
+	"repetitions",
+	"max-output-tokens",
+	"timeout-ms",
+	"max-cost-usd",
+	"max-artifact-bytes"
+]);
+const BOOLEAN_FLAGS = /* @__PURE__ */ new Set(["resume"]);
+function assertKnownFlags(flags) {
+	for (const flag of flags.keys()) if (!KNOWN_FLAGS.has(flag)) throw new Error(`unknown analyst-benchmark flag: --${flag}`);
+}
+function requiredFlag(flags, name) {
+	const value = flags.get(name)?.trim();
+	if (!value) throw new Error(`--${name} is required`);
+	return value;
+}
+function positiveFlag(flags, name, defaultValue) {
+	const raw = flags.get(name);
+	if (raw === void 0 && defaultValue !== void 0) return defaultValue;
+	if (raw === void 0) throw new Error(`--${name} is required`);
+	const value = Number(raw);
+	if (!Number.isSafeInteger(value) || value < 1) throw new Error(`--${name} must be a positive safe integer`);
+	return value;
+}
+function integerFlag(flags, name, defaultValue) {
+	const raw = flags.get(name);
+	if (raw === void 0) return defaultValue;
+	const value = Number(raw);
+	if (!Number.isSafeInteger(value)) throw new Error(`--${name} must be a safe integer`);
+	return value;
+}
+function positiveFiniteFlag(flags, name, defaultValue) {
+	const raw = flags.get(name);
+	if (raw === void 0) return defaultValue;
+	const value = Number(raw);
+	if (!Number.isFinite(value) || value <= 0) throw new Error(`--${name} must be a positive finite number`);
+	return value;
+}
+function openAiCompatibleBaseUrl(value) {
+	let parsed;
+	try {
+		parsed = new URL(value);
+	} catch {
+		throw new Error("--base-url must be an absolute HTTP or HTTPS URL");
+	}
+	if (parsed.protocol !== "http:" && parsed.protocol !== "https:" || parsed.username || parsed.password || parsed.search || parsed.hash) throw new Error("--base-url must use HTTP or HTTPS without credentials, query, or fragment");
+	if (parsed.protocol === "http:" && !isLoopbackHost(parsed.hostname)) throw new Error("--base-url must use HTTPS unless the endpoint is on the local machine");
+	return value;
+}
+function isLoopbackHost(hostname) {
+	const normalized = hostname.toLowerCase();
+	return normalized === "localhost" || normalized === "[::1]" || normalized === "::1" || /^127(?:\.\d{1,3}){3}$/.test(normalized);
+}
+function immutableRevision(value) {
+	if (!/^(?:[a-fA-F0-9]{40}|[a-fA-F0-9]{64})$/.test(value)) throw new Error("--revision must be a full 40- or 64-character hexadecimal digest");
+	return value.toLowerCase();
+}
+function renderSelectionMarkdown(report) {
+	const rows = [
+		"class",
+		"agent",
+		"model",
+		"difficulty",
+		"solved"
+	].map((dimension) => {
+		const source = report.source[dimension];
+		const selected = report.selected[dimension];
+		return `| ${dimension} | ${distributionText(source.counts, source.missing, source.total)} | ${distributionText(selected.counts, selected.missing, selected.total)} |`;
+	});
+	return [
+		"## Case Selection",
+		"",
+		`Method: \`${report.method}\`; seed: \`${report.seed}\`; selected: ${report.selectedCount}/${report.sourceCount}.`,
+		report.representativeOfInput ? "This is a census of the supplied input." : "This deterministic hash subset is not stratified and must not be presented as representative.",
+		"",
+		"| Dimension | Supplied input | Selected cases |",
+		"| --- | --- | --- |",
+		...rows
+	].join("\n");
+}
+function distributionText(counts, missing, total) {
+	const values = Object.entries(counts).map(([value, count]) => `${value}=${count}`);
+	if (missing > 0) values.push(`missing=${missing}`);
+	return `${values.join(", ") || "none"} (n=${total})`;
+}
+function summarizeVerificationAvailability(manifests) {
+	return {
+		cases: manifests.length,
+		resultFilesPresent: manifests.filter((manifest) => manifest.status === "present").length,
+		resultFilesMissing: manifests.filter((manifest) => manifest.status === "missing").length,
+		outcomes: {
+			passed: manifests.filter((manifest) => manifest.outcome.status === "passed").length,
+			failed: manifests.filter((manifest) => manifest.outcome.status === "failed").length,
+			unavailable: manifests.filter((manifest) => manifest.outcome.status === "unavailable").length
+		}
+	};
+}
+function renderVerificationAvailability(summary) {
+	return [
+		"## Final Verification Availability",
+		"",
+		"| Cases | Result files present | Result files missing | Passed | Failed | Unavailable |",
+		"| ---: | ---: | ---: | ---: | ---: | ---: |",
+		`| ${summary.cases} | ${summary.resultFilesPresent} | ${summary.resultFilesMissing} | ${summary.outcomes.passed} | ${summary.outcomes.failed} | ${summary.outcomes.unavailable} |`
+	].join("\n");
+}
+function renderArtifactMarkdown(artifact) {
+	const calibrationMarkdown = artifact.codeTraceCalibration ? `\n\n${renderCodeTraceCalibrationMarkdown(artifact.codeTraceCalibration)}` : artifact.agentRxCalibration ? `\n\n${renderAgentRxCalibrationMarkdown(artifact.agentRxCalibration)}` : "";
+	const verificationMarkdown = artifact.inputs.dataset === "codetracebench" ? `\n\n${renderVerificationAvailability(artifact.inputs.verificationAvailability)}` : "";
+	return `${renderAnalystBenchmarkMarkdown(artifact.result, artifact.comparisons).trimEnd()}${calibrationMarkdown}${verificationMarkdown}\n\n${renderSelectionMarkdown(artifact.inputs.selection.report)}\n`;
+}
+function benchmarkExitCode(result) {
+	return result.summaries.find((summary) => summary.runnerId === "model")?.failedRuns ? 2 : 0;
+}
+function printSuccessSummary(artifact, paths) {
+	const failures = artifact.result.summaries.reduce((total, summary) => total + summary.failedRuns, 0);
+	const knownCostUsd = artifact.result.summaries.reduce((total, summary) => total + summary.knownCostUsd, 0);
+	const unknownCostRuns = artifact.result.summaries.reduce((total, summary) => total + summary.costUnknownRuns, 0);
+	process.stdout.write(`Analyst benchmark complete: cases=${artifact.result.provenance.caseCount} failures=${failures} known_cost_usd=${knownCostUsd.toFixed(6)} unknown_cost_runs=${unknownCostRuns}\nresult=${paths.result}\nreport=${paths.report}\n`);
+}
+function shellQuote(value) {
+	return /^[a-zA-Z0-9_./:@%+=,-]+$/.test(value) ? value : `'${value.replaceAll("'", "'\\''")}'`;
+}
+//#endregion
+export { analystBenchmarkDependencyLockDigest as A, codeTracerPredictionsToFindings as B, ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES as C, ANALYST_BENCHMARK_IMPLEMENTATION_DIGEST_ALGORITHM as D, ANALYST_BENCHMARK_EVIDENCE_PACKAGE_VERSION as E, ANALYST_BENCHMARK_OBSERVATIONS_FILE as F, normalizeBenchmarkLabel as G, agentRxPredictionsToFindings as H, AGENT_RX_UPSTREAM_REVISION as I, renderAgentRxCalibrationMarkdown as L, ANALYST_BENCHMARK_COST_LEDGER_FILE as M, ANALYST_BENCHMARK_LOCAL_RECEIPT_FILE as N, ANALYST_BENCHMARK_IMPLEMENTATION_FILES as O, ANALYST_BENCHMARK_MANIFEST_FILE as P, summarizeAgentRxCalibration as R, ANALYST_BENCHMARK_DEPENDENCY_LOCK_DIGEST_ALGORITHM as S, ANALYST_BENCHMARK_EVIDENCE_DEPENDENCY_LOCK_SHA256 as T, normalizeAgentRxCategory as U, agentRxBenchmarkCase as V, roundAgentRxStep as W, appendVerificationArtifactsToOtlp as _, renderCodeTraceCalibrationMarkdown as a, adaptPublicBenchmarkFindings as b, CODE_TRACE_BENCH_ANALYST_PROMPT as c, loadPublicBenchmarkRows as d, preparePublicAnalystBenchmark as f, DEFAULT_MAX_VERIFICATION_ARTIFACT_BYTES as g, selectPublicBenchmarkRows as h, readAnalystBenchmarkArtifact as i, analystBenchmarkImplementationDigest as j, ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 as k, createPublicBenchmarkModelRunner as l, publicBenchmarkSelectionReport as m, runAnalystBenchmarkCommand as n, summarizeCodeTraceCalibration as o, publicBenchmarkDistributions as p, renderAnalystBenchmarkMarkdown as r, compareAnalystRunners as s, ANALYST_BENCHMARK_HELP as t, publicBenchmarkProtocolSha256 as u, loadCodeTraceVerificationArtifacts as v, ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 as w, emptyPublicBenchmarkRunner as x, parseVerificationOutcome as y, codeTraceBenchCase as z };
+
+//# sourceMappingURL=benchmark-command-CMqVqReF.js.map