@tangle-network/agent-eval 0.140.1 → 0.141.0

package/dist/{benchmark-command-Bo_hkhQf.js → benchmark-command-D8xXNVVh.js}

@@ -2,14 +2,14 @@ import { c as ValidationError, t as AgentEvalError } from "./errors-D-LKuDhb.js"
 import { s as resolveModelPricing } from "./metrics-C9YY1OcL.js";
 import { a as CostLedgerPersistenceError, i as CostLedger, n as CostCallConflictError, o as CostReceiptCaptureError, r as CostCeilingReachedError, s as CostReservationExceededError, t as CostAccountingIncompleteError } from "./cost-ledger-DMFxsLKr.js";
 import { c as callLlmJson, f as maximumChargeForLlmRequest, l as costReceiptFromLlm, r as LlmResponseError, t as LlmCallError, u as costReceiptFromLlmError } from "./llm-client-B3WXSH5Y.js";
-import { O as evidenceRefsFromRawFinding, h as TRACE_ANALYSIS_LIMITS, i as runTraceAnalyst } from "./kind-factory-DB7nIs35.js";
+import { O as evidenceRefsFromRawFinding, h as TRACE_ANALYSIS_LIMITS, i as runTraceAnalyst } from "./kind-factory-Bvwe3pup.js";
 import { o as makeFinding, r as usageReceiptFromCostLedger } from "./usage-receipt-CgxMEBZq.js";
 import { i as hashCanonical, r as canonicalString } from "./canonical-D011XM8r.js";
-import { n as createRunCostLedger, r as fsCampaignStorage, t as acquireSingleRunLock } from "./single-run-lock-CiQThJxB.js";
-import { t as createDspyRlmTraceEngine } from "./dspy-rlm-engine-Bkak4nzo.js";
+import { n as createRunCostLedger, r as fsCampaignStorage, t as acquireSingleRunLock } from "./single-run-lock-BcvPQY57.js";
+import { t as createDspyRlmTraceEngine } from "./dspy-rlm-engine-Bw8Zz9CX.js";
 import { c as writeLedgerFileAtomically, s as withLedgerFileLock } from "./ledger-core-Dxz0Rkwa.js";
 import { E as pairedBootstrap } from "./statistics-ByxzSiOM.js";
-import { i as otlpTextToTraceAnalysisStore, r as createOtlpBufferTraceStore, t as DEFAULT_MAX_TRACE_FILE_BYTES } from "./store-otlp-vRByAR6h.js";
+import { i as otlpTextToTraceAnalysisStore, r as createOtlpBufferTraceStore, t as DEFAULT_MAX_TRACE_FILE_BYTES } from "./store-otlp-D4I90_vR.js";
 import { i as summarizeAnalystBenchmarkRunner, n as runAnalystBenchmark, r as traceStoreEvidenceResolver } from "./benchmark-CYtcIF2V.js";
 import { z } from "zod";
 import { constants, existsSync, lstatSync, readFileSync } from "node:fs";
@@ -1068,6 +1068,7 @@ const artifactSchema = z.strictObject({
 		execution: z.strictObject({
 			repetitions: positiveInteger$1,
 			concurrency: positiveInteger$1,
+			rlmSamples: positiveInteger$1.optional(),
 			model: nonEmptyString,
 			maxOutputTokens: positiveInteger$1,
 			timeoutMs: positiveInteger$1,
@@ -1172,7 +1173,7 @@ const ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES = Object.freeze([
 	"package.json",
 	"pnpm-lock.yaml"
 ]);
-const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = "701ea59184f3d227cf590ea3f16d24d65dbed2ad5644f8294801ceb66a30d4ef";
+const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = "106219b86a612a0352fabf44eb34bc1bbffd5c842ed97ae6132343af88f89c94";
 /** The published benchmark evidence was produced at this package version, by
 * the retired one-shot direct runner, before trace analysts moved to the
 * recursive DSPy RLM engine. Both evidence digests below are historical facts
@@ -1198,8 +1199,10 @@ const ANALYST_BENCHMARK_IMPLEMENTATION_FILES = Object.freeze([
 	"src/analyst/benchmark-dataset-utils.ts",
 	"src/analyst/benchmark-datasets.ts",
 	"src/analyst/benchmark-evidence-validation.ts",
+	"src/analyst/benchmark-instructions-override.ts",
 	"src/analyst/benchmark-public-adapters.ts",
 	"src/analyst/benchmark-public-calibration.ts",
+	"src/analyst/benchmark-public-consensus.ts",
 	"src/analyst/benchmark-public-data.ts",
 	"src/analyst/benchmark-public-errors.ts",
 	"src/analyst/benchmark-public-model.ts",
@@ -1266,7 +1269,7 @@ const ANALYST_BENCHMARK_IMPLEMENTATION_FILES = Object.freeze([
 	"src/trace/otlp-attributes.ts",
 	"src/trace/raw-provider-sink.ts"
 ]);
-const ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 = "56dd4c7ed19fc5855f99ad238464ad95cb1bea155ad58c49dab3eb0f6cbe7d6a";
+const ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 = "a3f5a8203f0a7c74736170c53100c50c4db1cf84381533002f0539ef1e7a4c6d";
 function analystBenchmarkImplementationDigest() {
 	return ANALYST_BENCHMARK_IMPLEMENTATION_SHA256;
 }
@@ -2104,37 +2107,44 @@ const TRACE_PROJECTION_ATTRIBUTE_BYTE_CAPS = [
 	64
 ];
 const CODE_TRACE_BENCH_ANALYST_PROMPT = `Analyze exactly one coding-agent trajectory and its attached final verification.
-Your task is the CodeTraceBench incorrect-step task: identify every wrong state-changing action, bad hypothesis that drives an action, and regression.
-An incorrect step remains incorrect when the agent later recovers or the final verification passes; a trajectory that ends in success still contains every mistake the agent made along the way.
-Incorrect steps occur in contiguous failure blocks: one mistake plus every consecutive following step that commits to, compounds, or acts on it.
+Your task is the CodeTraceBench incorrect-step task: identify every incorrect step, defined as a wrong state-changing intervention given the evidence — a mislocalized edit, a wrong hypothesis that drives an action, a regression, an irrelevant change, or an incorrect dependency or configuration choice.
+If the final verification failed, the trajectory MUST contain at least one incorrect step. Never return an empty findings array on a failing trajectory; trace backward until you find the root cause.
+Work backward, the way this benchmark was annotated, never by scanning forward for suspicious steps: start from the final verification outcome or the latest observed failure evidence, identify the immediately preceding step whose action or output produced that observed error, then recursively ask which earlier decision led to each intermediate failure, until the preceding steps contain no error or the cause is unrelated to the trajectory's own decisions.
+Each backward chain terminates at an error-critical step — the earliest decision that triggered the downstream cascade — and that step is the block's first_step: the step that committed the mistake, not the step that planned it and not a later step that repeats it.
+A block is a maximal contiguous sequence of strictly incorrect steps. A step belongs in the block ONLY if it introduces, propagates, or compounds the error. 
+Do NOT include steps that merely "act on", diagnose, or react to the error. A diagnostic command, a test run exposing the bug, or a correct exploratory read is a CORRECT step. 
+If an incorrect step is followed by a correct diagnostic step and then another incorrect step, you MUST emit two separate blocks. NEVER bridge correct steps by grouping them into a single block with incorrect steps. Over-blocking drastically hurts your precision.
+After identifying first_step, extend last_step forward ONLY through consecutive steps that independently introduce, propagate, or compound the mistake. A cascade of repeated failed attempts at the same wrong approach is one maximal block, provided EVERY step is independently incorrect.
+Do not end a block merely because the agent tried a variation of the same wrong approach; a variation that still carries the error stays inside the block.
+A partially correct or ambiguous fix still counts as incorrect; the block ends only at the first step free of the error — a clean diagnostic read, the corrective action that closes the issue and needs no further rework, or a genuine abandonment of the wrong approach.
+Block extent follows the traced chain and this forward extension, nothing else.
 Report each failure block as exactly one finding whose first_step is the block's first incorrect step and whose last_step is its last, covering every consecutive step between them.
-Set first_step to the first step that commits the mistake, not the step that planned it and not a later step that repeats it.
-Extend last_step one step at a time, and only while the next step independently satisfies the incorrect-step definition on its own action and its own following observation.
-Stop at the first step where the agent detects the problem, inspects it, or begins repairing it: a diagnostic probe, a test run that exposes the defect, or a repair action ends the block and is never inside it.
-A one-step block is a complete and correct answer.
-Every step inside a block is scored on its own: naming a correct step costs exactly as much as missing an incorrect one, and naming only the first step of a longer block forfeits every unnamed step.
-Report blocks separated by at least one correct step as separate findings, and never let two blocks overlap.
-Inspect the complete supplied trace data.
-Use the final-verification outcome as evidence about the final state, not as a rule for whether earlier steps were incorrect.
-For each candidate block, inspect every assistant action in it and its following observation.
-Admit a block only when you can point at the specific evidence it produced: name as consequence_step the step number whose action or observation shows the damage — a failing command, a wrong file state, a repeated failure, or rework the agent had to do because of this block. That step is the block's own last step when its observation already shows the damage, and a later step otherwise.
-When you cannot name that later step number from the trace you were given, drop the block; a plausible story about why a step looks wrong is not evidence that it was.
-Judge that consequence from the trajectory itself: a passing final verification is not evidence that a block caused nothing, and a failing final verification is not evidence that any particular block caused it.
+Every step inside a block is scored on its own: naming a correct step costs exactly as much as missing an incorrect one, and naming only the first step of a longer block forfeits every unnamed step. Because of this, carefully verify every step between first_step and last_step. Only include steps that introduce, propagate, or compound the error.
+Report blocks separated by at least one correct step as separate findings, and never let two blocks overlap. If there are multiple separate failure cascades, emit a separate finding for each one.
+Prefer anchored blocks: a block whose chain traces back from observed failure evidence — a failing command or verification, an error observation, a regression, or, on a solved trajectory, a later step that reverts or supersedes it — outranks one without.
+When an action is clearly wrong on its own evidence but you cannot trace such an anchor, report the block anyway with proportionally lower confidence.
+A solved trajectory still carries every mistake made along the way: inspect its final patching and verification stages for a state-changing action that a later step reverted, superseded, or corrected — a wrong edit just before the final fix is incorrect even when every test ends green.
+Before emitting a candidate block, check its boundaries.
+Neighbor check: ask whether the accusation fits one step earlier (the decision rather than its consequence) or one step later (the next step still acts on or reworks the same error) better than where you placed it, and move the boundary when it does; a boundary off by one step scores zero at that step.
+Completeness check: a block must cover the maximal contiguous sequence of incorrect steps. If an agent fails at step 10, tries to fix it at 11, fails, and tries again at 12 and 13, all four steps are incorrect and must be included in the block. Never truncate a cascade. If you miss the later steps of a cascade, your recall drops to zero for them.
+Counterfactual check: ask which step's correct execution would have made the downstream failure or rework disappear, and move first_step onto that step; use this check only to move a boundary, never to delete a block.
+Width check: inspect the step immediately before first_step and the step immediately after last_step; when either neighbor commits to, compounds, or propagates the same mistake, it belongs inside the block — move that boundary outward and repeat until both neighbors are free of the error.
+For each block you keep, name as consequence_step the step number whose action or observation shows the damage — a failing command, a wrong file state, a repeated failure, or rework the agent had to do because of this block; that step is the block's own last step when its observation already shows the damage, and a later step otherwise.
+When you cannot name that step number from the trace you were given, drop the block; a plausible story about why a step looks wrong is not evidence that it was.
+A passing final verification is not evidence that a block caused nothing, and a failing final verification is not evidence that any particular block caused it.
 For every block, decide whether the agent escaped the failure.
-Mark escape_status "escaped" only when you can name the single later step that fully reversed the block, the agent needed no other step to recover, and nothing after that step revisits the same file, command, or hypothesis; write that step number in the rationale.
+Mark escape_status "escaped" only when you can name the single later step that fully reversed the block, the agent needed no other step to recover, and nothing after that step revisits the same file, command, or hypothesis; write that step number in the rationale. If the agent required multiple steps to fix the issue, or if you are unsure, it is not escaped.
 Mark escape_status "unescaped" in every other case, including whenever you are unsure.
-A passing final verification never makes a block escaped: the agent may have made the mistake and repaired it over several steps, and those steps are still incorrect.
-Label a failed command when the assistant caused it through a wrong action or unsupported hypothesis.
-Label the later corrective action only when that action is itself wrong.
-Do not label a diagnostic probe merely because it exposes an earlier defect.
-Do not label a redundant but correct read or search; CodeTraceBench scores unuseful steps separately, and this run scores incorrect steps only.
+A passing final verification never makes a block escaped.
+Do not label a diagnostic probe or test run merely because its output exposes an earlier defect.
+Do not label a redundant but correct read or search; CodeTraceBench scores unuseful steps separately, and this run scores incorrect steps only. Never include a correct step in a block just to bridge two incorrect steps; instead, emit two separate blocks.
 Do not label a step solely because final verification failed.
-When final verification is unavailable, use only directly observed trajectory evidence.
+When final verification is unavailable, trace backward from the latest failure evidence inside the trajectory itself.
 Every step in a reported block MUST be the positive integer n from an existing assistant LLM span named step-<n>.
 Never select an EVALUATOR, TOOL, CHAIN, final-verification, benchmark-verification, or message-<n> span.
 Before emitting a finding, inspect every covered span's attributes.content and describe only the actions shown there.
-Report at most 16 blocks and at most 12 steps in one block; when more candidates than that exist, report the ones you can support with the clearest downstream evidence.
-When the trajectory has no incorrect steps, return an empty findings array.`;
+Report at most 16 blocks and at most 12 steps in one block; when more candidates than that exist, report the ones whose chains carry the clearest downstream evidence.
+When the trajectory has no incorrect steps — its final verification passed and the final-stage sweep found no reverted, superseded, or corrected action — return an empty findings array.`;
 const AGENT_RX_PROMPT = `Analyze exactly one failed agent trajectory.
 Find the first unrecoverable critical failure, not every later symptom.
 Inspect the complete supplied trace data.
@@ -2175,8 +2185,8 @@ The excerpt must quote the assistant action exactly.`;
 const CODE_TRACE_RLM_CONTRACT = `Use the trace tools rather than asking for the whole trajectory in the prompt.
 Keep retrieved trace objects in Python variables.
 Never print an entire trace, full source file, or more than 12000 characters in one iteration.
-Build a compact table of assistant step ids, actions, following observations, and final verification.
-Inspect suspicious steps with viewSpans or searchSpan instead of repeatedly printing the table.
+Read the final verification and the latest failure evidence first, then build a compact table of assistant step ids, actions, and following observations.
+Trace backward from that evidence with viewSpans or searchSpan, confirming each candidate step's own action content, instead of repeatedly printing the table.
 This runner emits no JSON fields, so the block is encoded in the finding's subject.
 Only findings_json is scored; your prose answer is ignored, so every incorrect block you identify must appear as a finding, never only in the answer.
 Emit exactly one finding per contiguous failure block.
@@ -2235,774 +2245,1314 @@ function publicBenchmarkProtocolSha256(dataset) {
 	}));
 }
 //#endregion
-//#region src/analyst/benchmark-public-adapters.ts
-function emptyPublicBenchmarkRunner() {
+//#region src/analyst/benchmark-instructions-override.ts
+/** Build an override from instruction text. Blank text is a caller error. */
+function analystInstructionsOverrideFromText(text) {
+	if (typeof text !== "string" || !text.trim()) throw new Error("analyst instructions override must contain non-empty instruction text");
 	return {
-		id: "empty",
-		analyze() {
-			return {
-				findings: [],
-				usage: {
-					calls: 0,
-					tokens: {
-						input: 0,
-						output: 0
-					},
-					cost: {
-						kind: "observed",
-						usd: 0
-					}
-				},
-				metadata: { baseline: "emit-no-findings" }
-			};
-		}
-	};
-}
-async function adaptPublicBenchmarkFindings(options) {
-	if (options.dataset === "agentrx") return {
-		findings: adaptAgentRxFindings(options.trajectoryId, options.findings, options.analystId),
-		diagnostics: void 0
+		text,
+		sha256: sha256Digest(text)
 	};
-	return adaptCodeTraceFindings(options.trajectoryId, options.findings, options.analystId, options.store, options.signal);
-}
-function adaptAgentRxFindings(trajectoryId, findings, analystId) {
-	if (findings.length === 0) return [];
-	if (findings.length !== 1) throw new Error(`AgentRx model analyst must emit zero or one root cause, received ${findings.length}`);
-	const source = findings[0];
-	if (!source.subject) throw new Error("AgentRx model analyst finding is missing its failure-category subject");
-	const steps = exactFindingSteps(trajectoryId, source);
-	if (steps.length !== 1) throw new Error(`AgentRx model analyst must cite exactly one root-cause step, received ${steps.length}`);
-	const [adapted] = agentRxPredictionsToFindings(trajectoryId, [{
-		failure_case: source.subject,
-		step_number: steps[0],
-		description: source.rationale ?? source.claim
-	}], {
-		analystId,
-		producedAt: source.produced_at,
-		confidence: source.confidence
-	});
-	if (!adapted) throw new Error("AgentRx output adapter produced no root-cause finding");
-	return [{
-		...adapted,
-		metadata: {
-			...adapted.metadata,
-			sourceFindingId: source.finding_id
-		}
-	}];
 }
-const CODE_TRACE_BLOCK_SUBJECT = /^incorrect-steps-(\d+)-(\d+)-(escaped|unescaped)-consequence-(\d+)$/;
-async function adaptCodeTraceFindings(trajectoryId, findings, analystId, store, signal) {
-	const clean = findings.filter((finding) => finding.subject === "clean");
-	if (clean.length > 0) {
-		if (findings.length !== 1) throw new Error("CodeTraceBench model analyst mixed a clean verdict with incorrect steps");
-		exactFindingSteps(trajectoryId, clean[0]);
-		return {
-			findings: [],
-			diagnostics: emptyCodeTraceBlockDiagnostics()
-		};
-	}
-	const blocks = [];
-	const rejectedFindings = [];
-	for (const source of findings) try {
-		await validateCodeTraceFindingEvidence({
-			trajectoryId,
-			findings: [source],
-			store,
-			...signal ? { signal } : {}
-		});
-		blocks.push(codeTraceBlockFromFinding(trajectoryId, source));
+/** Read override instructions from a file. Any read failure is fatal. */
+function readAnalystInstructionsOverride(path) {
+	let text;
+	try {
+		text = readFileSync(path, "utf8");
 	} catch (error) {
-		rejectedFindings.push(`${source.finding_id}: ${error instanceof Error ? error.message : String(error)}`);
+		throw new Error(`cannot read --instructions-file '${path}': ${error instanceof Error ? error.message : String(error)}`);
 	}
-	const expanded = await expandCodeTraceFailureBlocks({
-		trajectoryId,
-		blocks,
-		store,
-		analystId,
-		...findings[0] ? { producedAt: findings[0].produced_at } : {},
-		...signal ? { signal } : {}
-	});
-	return {
-		findings: expanded.findings,
-		diagnostics: {
-			...expanded.diagnostics,
-			rejectedFindings
-		}
-	};
-}
-function codeTraceBlockFromFinding(trajectoryId, source) {
-	const parsed = CODE_TRACE_BLOCK_SUBJECT.exec(source.subject ?? "");
-	if (!parsed) throw new Error(`CodeTraceBench model finding '${source.finding_id}' must set subject to incorrect-steps-<first>-<last>-<escaped|unescaped>-consequence-<step>, received '${source.subject ?? ""}'`);
-	const firstStep = Number(parsed[1]);
-	const lastStep = Number(parsed[2]);
-	const consequenceStep = Number(parsed[4]);
-	const cited = exactFindingSteps(trajectoryId, source);
-	for (const step of cited) if (step < firstStep || step > lastStep) throw new Error(`CodeTraceBench model finding '${source.finding_id}' cites step ${step} outside its block ${firstStep}-${lastStep}`);
-	return {
-		firstStep,
-		lastStep,
-		consequenceStep,
-		escapeStatus: parsed[3],
-		severity: source.severity,
-		claim: source.claim,
-		confidence: source.confidence,
-		...source.rationale === void 0 ? {} : { rationale: source.rationale },
-		...source.recommended_action === void 0 ? {} : { recommendedAction: source.recommended_action },
-		metadata: { sourceFindingId: source.finding_id }
-	};
+	if (!text.trim()) throw new Error(`--instructions-file '${path}' is empty; refusing to run without instructions`);
+	return analystInstructionsOverrideFromText(text);
 }
 /**
-* Expand contiguous failure blocks into one scored finding per member step.
+* Protocol digest of the run as executed.
 *
-* The official scorer matches on area plus the exact step evidence URI, so
-* blocks never reach it: every runner reports blocks, and this function turns
-* them into the per-step findings the benchmark defines.
+* Without an override this is exactly `publicBenchmarkProtocolSha256(dataset)`,
+* so stock runs stay byte-identical to runs recorded before the override
+* existed. With an override the digest binds the stock protocol digest (which
+* covers both shipped prompts, including the abstention fallback's direct
+* prompt) to the exact override text, so the recorded digest always hashes the
+* instructions that actually ran.
 */
-async function expandCodeTraceFailureBlocks(options) {
-	const diagnostics = emptyCodeTraceBlockDiagnostics();
-	diagnostics.reportedBlocks = options.blocks.length;
-	if (options.blocks.length === 0) return {
-		findings: [],
-		diagnostics
-	};
-	assertCodeTraceBlockShape(options.blocks);
-	const boundarySteps = options.blocks.flatMap((block) => [block.firstStep, block.lastStep]);
-	const derivedSteps = options.blocks.flatMap((block) => [block.consequenceStep, ...interiorSteps(block)]);
-	const evidenceByStep = await resolveAssistantStepEvidence({
-		trajectoryId: options.trajectoryId,
-		steps: boundarySteps,
-		optionalSteps: derivedSteps,
-		store: options.store,
-		...options.signal ? { signal: options.signal } : {}
-	});
-	const byStep = /* @__PURE__ */ new Map();
-	for (const block of options.blocks) {
-		if (!evidenceByStep.has(block.consequenceStep)) {
-			diagnostics.blocksWithoutConsequenceEvidence.push(block);
-			continue;
+function effectiveAnalystProtocolSha256(dataset, override) {
+	const stock = publicBenchmarkProtocolSha256(dataset);
+	if (!override) return stock;
+	return sha256Digest(JSON.stringify({
+		kind: "analyst-instructions-override-protocol",
+		dataset,
+		stockProtocolSha256: stock,
+		rlmInstructionsSha256: override.sha256
+	}));
+}
+//#endregion
+//#region src/analyst/benchmark-command-persistence.ts
+const ANALYST_BENCHMARK_INITIALIZATION_COMPLETE_FILE = "initialization-complete.json";
+async function openOutputDirectory(outDir, resume) {
+	const directory = resolve(outDir);
+	if (resume) {
+		let outputStat;
+		try {
+			outputStat = await lstat(directory);
+		} catch (error) {
+			if (isNodeError(error, "ENOENT")) throw new Error(`cannot resume missing benchmark output directory: ${directory}`);
+			throw error;
 		}
-		if (block.escapeStatus === "escaped") diagnostics.escapedBlocks += 1;
-		for (let step = block.firstStep; step <= block.lastStep; step += 1) {
-			if (!evidenceByStep.has(step)) {
-				diagnostics.unresolvedBlockInteriorSteps.push(step);
-				continue;
-			}
-			if (byStep.has(step)) {
-				diagnostics.overlappingBlockSteps.push(step);
-				continue;
-			}
-			byStep.set(step, block);
+		if (!outputStat.isDirectory() || outputStat.isSymbolicLink()) throw new Error(`benchmark output must be a real directory: ${directory}`);
+	} else {
+		await mkdir(dirname(directory), { recursive: true });
+		try {
+			await mkdir(directory);
+		} catch (error) {
+			if (isNodeError(error, "EEXIST")) throw new Error(`refusing to use existing benchmark output directory: ${directory}`);
+			throw error;
 		}
+		await syncDirectory(dirname(directory));
 	}
 	return {
-		findings: [...byStep].sort(([left], [right]) => left - right).map(([step, block]) => makeFinding({
-			analyst_id: options.analystId,
-			area: "incorrect",
-			subject: `incorrect-step-${step}`,
-			claim: `Step ${step} is incorrect. ${block.claim}`,
-			rationale: block.rationale,
-			severity: block.severity,
-			confidence: block.confidence,
-			evidence_refs: [evidenceByStep.get(step)],
-			recommended_action: block.recommendedAction,
-			metadata: {
-				...block.metadata,
-				block_first_step: block.firstStep,
-				block_last_step: block.lastStep,
-				block_consequence_step: block.consequenceStep,
-				escape_status: block.escapeStatus
-			},
-			...options.producedAt === void 0 ? {} : { produced_at: options.producedAt },
-			id_basis: `incorrect-step-${step}`
-		})),
-		diagnostics
+		directory,
+		initializationComplete: resolve(directory, ANALYST_BENCHMARK_INITIALIZATION_COMPLETE_FILE),
+		manifest: resolve(directory, ANALYST_BENCHMARK_MANIFEST_FILE),
+		observations: resolve(directory, ANALYST_BENCHMARK_OBSERVATIONS_FILE),
+		costLedger: resolve(directory, ANALYST_BENCHMARK_COST_LEDGER_FILE),
+		modelResponses: resolve(directory, "model-responses"),
+		localReceipt: resolve(directory, ANALYST_BENCHMARK_LOCAL_RECEIPT_FILE),
+		result: resolve(directory, "result.json"),
+		report: resolve(directory, "report.md")
 	};
 }
-function assertCodeTraceBlockShape(blocks) {
-	if (blocks.length > 16) throw new Error(`model reported ${blocks.length} failure blocks; the maximum is 16`);
-	for (const block of blocks) {
-		if (block.lastStep < block.firstStep) throw new Error(`failure block last_step ${block.lastStep} precedes first_step ${block.firstStep}`);
-		const length = block.lastStep - block.firstStep + 1;
-		if (length > 12) throw new Error(`failure block spans ${length} steps; the maximum is 12`);
-		if (block.consequenceStep < block.firstStep) throw new Error(`failure block consequence_step ${block.consequenceStep} precedes first_step ${block.firstStep}`);
-	}
-}
-function interiorSteps(block) {
-	const steps = [];
-	for (let step = block.firstStep + 1; step < block.lastStep; step += 1) steps.push(step);
-	return steps;
+async function prepareOutputLockPath(outDir) {
+	const directory = resolve(outDir);
+	await mkdir(dirname(directory), { recursive: true });
+	return `${directory}.lock`;
 }
-function emptyCodeTraceBlockDiagnostics() {
+function createRunIdentity(config, prepared) {
+	const caseDefinitions = prepared.cases.map((testCase) => ({
+		id: testCase.id,
+		clusterId: testCase.clusterId,
+		labelState: testCase.labelState,
+		expectedIssues: testCase.expectedIssues,
+		labeledEvidence: testCase.labeledEvidence ?? [],
+		tags: testCase.tags ?? [],
+		metadata: testCase.metadata ?? {}
+	}));
 	return {
-		reportedBlocks: 0,
-		escapedBlocks: 0,
-		blocksWithoutConsequenceEvidence: [],
-		unresolvedBlockInteriorSteps: [],
-		overlappingBlockSteps: []
+		config: {
+			dataset: config.dataset,
+			datasetRevision: config.revision,
+			datasetSplit: config.split,
+			model: {
+				id: config.model.model,
+				maxOutputTokens: config.model.maxOutputTokens,
+				timeoutMs: config.model.timeoutMs
+			},
+			limit: config.limit,
+			seed: config.seed,
+			concurrency: config.concurrency,
+			repetitions: config.repetitions,
+			rlmSamples: config.rlmSamples,
+			maxCostUsd: config.maxCostUsd,
+			maxArtifactBytes: config.maxArtifactBytes,
+			analystProtocolSha256: effectiveAnalystProtocolSha256(config.dataset, config.model.instructionsOverride),
+			...config.model.instructionsOverride ? { instructionsOverrideSha256: config.model.instructionsOverride.sha256 } : {},
+			implementationSha256: ANALYST_BENCHMARK_IMPLEMENTATION_SHA256,
+			dependencyLockSha256: ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256,
+			runnerIds: ["empty", config.analyst]
+		},
+		inputs: {
+			labelsSha256: prepared.labelsSha256,
+			sourceRowCount: prepared.sourceRowCount,
+			selectedCaseIds: [...prepared.selectedCaseIds],
+			traceFiles: prepared.traceFiles.map((traceFile) => ({ ...traceFile })),
+			verificationArtifactsSha256: digestCanonical(prepared.verificationArtifacts),
+			caseDefinitionsSha256: digestCanonical(caseDefinitions)
+		}
 	};
 }
-function exactFindingSteps(trajectoryId, finding) {
-	if (finding.evidence_refs.length === 0) throw new Error(`model finding '${finding.finding_id}' has no step evidence`);
-	const steps = finding.evidence_refs.map((evidence) => {
-		const parsed = codeTraceStepFromEvidence(evidence.uri);
-		if (!parsed || parsed.traceId !== trajectoryId) throw new Error(`model finding '${finding.finding_id}' cites non-case evidence '${evidence.uri}'`);
-		return parsed.step;
-	});
-	return [...new Set(steps)];
-}
-//#endregion
-//#region src/analyst/benchmark-public-types.ts
-function requiredString(value, field) {
-	const trimmed = value.trim();
-	if (!trimmed) throw new TypeError(`${field} must be a non-empty string`);
-	return trimmed;
-}
-function positiveSafeInteger(value, field) {
-	if (!Number.isSafeInteger(value) || value < 1) throw new RangeError(`${field} must be a positive safe integer`);
-	return value;
+function createLocalRunReceipt(config, paths) {
+	return {
+		kind: "agent-eval/analyst-benchmark-local-run",
+		local: {
+			labelsPath: resolve(config.labelsPath),
+			traceDir: resolve(config.traceDir),
+			...config.artifactDir ? { artifactDir: resolve(config.artifactDir) } : {},
+			outputDir: paths.directory,
+			baseUrl: config.model.baseUrl,
+			apiKeyEnvironment: config.apiKeyEnv
+		},
+		command: config.command,
+		environment: {
+			node: process.version,
+			platform: platform(),
+			arch: arch()
+		},
+		files: {
+			manifest: paths.manifest,
+			observations: paths.observations,
+			costLedger: paths.costLedger,
+			modelResponses: paths.modelResponses,
+			result: paths.result,
+			report: paths.report
+		}
+	};
 }
-function safeInteger(value, field) {
-	if (!Number.isSafeInteger(value)) throw new RangeError(`${field} must be a safe integer`);
-	return value;
+async function initializeRunFiles(paths, identity, identitySha256, localIdentitySha256, localReceiptInput) {
+	const existingManifest = await readOptionalRegularFile(paths.manifest, "benchmark run manifest");
+	const manifest = existingManifest ? await readAndValidateManifestContent(paths.manifest, existingManifest, identity, identitySha256, localIdentitySha256) : {
+		kind: "agent-eval/analyst-benchmark-run",
+		createdAt: (/* @__PURE__ */ new Date()).toISOString(),
+		identitySha256,
+		localIdentitySha256,
+		identity
+	};
+	const localReceipt = {
+		...localReceiptInput,
+		runIdentitySha256: identitySha256,
+		localIdentitySha256
+	};
+	const manifestContent = `${JSON.stringify(manifest, null, 2)}\n`;
+	const localReceiptContent = `${JSON.stringify(localReceipt, null, 2)}\n`;
+	const initializationCompleteContent = renderInitializationComplete(manifest);
+	await assertAbsentOrExact(paths.observations, "", "benchmark observation log");
+	await assertAbsentOrExact(paths.localReceipt, localReceiptContent, "benchmark local run receipt");
+	await assertAbsentOrExact(paths.manifest, manifestContent, "benchmark run manifest");
+	for (const path of [
+		paths.costLedger,
+		paths.modelResponses,
+		paths.result,
+		paths.report
+	]) if (await regularFileExists(path)) throw new Error(`benchmark initialization marker is missing but later run artifact exists: ${path}`);
+	if (await regularFileExists(paths.initializationComplete)) throw new Error(`benchmark initialization marker already exists during partial initialization: ${paths.initializationComplete}`);
+	await writeExclusiveOrVerify(paths.observations, "");
+	await writeExclusiveOrVerify(paths.localReceipt, localReceiptContent);
+	await writeExclusiveOrVerify(paths.manifest, manifestContent);
+	await writeExclusiveOrVerify(paths.initializationComplete, initializationCompleteContent);
+	return manifest;
 }
-function isRecord(value) {
-	return typeof value === "object" && value !== null && !Array.isArray(value);
+async function readAndValidateResumeFiles(paths, currentIdentity, currentIdentitySha256, currentLocalIdentitySha256, localReceiptInput) {
+	if (!await regularFileExists(paths.initializationComplete)) return initializeRunFiles(paths, currentIdentity, currentIdentitySha256, currentLocalIdentitySha256, localReceiptInput);
+	const manifest = await readAndValidateManifest(paths.manifest, currentIdentity, currentIdentitySha256, currentLocalIdentitySha256);
+	const localReceiptContent = await readRegularFile(paths.localReceipt, "benchmark local run receipt");
+	const value = parseJson(localReceiptContent, paths.localReceipt);
+	if (!isRecord$1(value)) throw new TypeError(`benchmark local run receipt must be an object: ${paths.localReceipt}`);
+	assertExactKeys(value, [
+		"kind",
+		"runIdentitySha256",
+		"localIdentitySha256",
+		"local",
+		"command",
+		"environment",
+		"files"
+	], "benchmark local run receipt");
+	if (value.kind !== "agent-eval/analyst-benchmark-local-run" || value.runIdentitySha256 !== currentIdentitySha256 || value.localIdentitySha256 !== currentLocalIdentitySha256 || !isRecord$1(value.local)) throw new Error("benchmark local run receipt does not match the requested resume");
+	const expectedLocalReceipt = {
+		...localReceiptInput,
+		runIdentitySha256: currentIdentitySha256,
+		localIdentitySha256: currentLocalIdentitySha256
+	};
+	if (digestCanonical(value.local) !== currentLocalIdentitySha256 || canonicalJson(value.local) !== canonicalJson(localReceiptInput.local)) throw new Error("benchmark local paths or endpoint do not match the requested resume");
+	if (localReceiptContent !== `${JSON.stringify(expectedLocalReceipt, null, 2)}\n`) throw new Error(`benchmark local run receipt does not exactly match: ${paths.localReceipt}`);
+	if (await readRegularFile(paths.initializationComplete, "benchmark initialization marker") !== renderInitializationComplete(manifest)) throw new Error(`benchmark initialization marker does not match the run manifest: ${paths.initializationComplete}`);
+	return manifest;
 }
-//#endregion
-//#region src/analyst/benchmark-public-data.ts
-const DEFAULT_MAX_PUBLIC_BENCHMARK_LABEL_BYTES = 256 * 1024 * 1024;
-const INPUT_OPEN_FLAGS = constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0);
-async function loadPublicBenchmarkRows(path) {
-	return parsePublicBenchmarkRows((await readImmutableInputSnapshot(resolve(path), DEFAULT_MAX_PUBLIC_BENCHMARK_LABEL_BYTES)).text, path);
+async function readAndValidateManifest(path, currentIdentity, currentIdentitySha256, currentLocalIdentitySha256) {
+	const content = await readRegularFile(path, "benchmark run manifest");
+	const manifest = await readAndValidateManifestContent(path, content, currentIdentity, currentIdentitySha256, currentLocalIdentitySha256);
+	if (content !== `${JSON.stringify(manifest, null, 2)}\n`) throw new Error(`benchmark run manifest does not exactly match: ${path}`);
+	return manifest;
 }
-function parsePublicBenchmarkRows(text, path) {
-	const trimmed = text.trim();
-	if (!trimmed) throw new Error(`public analyst benchmark dataset is empty: ${path}`);
-	let parsed;
-	try {
-		parsed = JSON.parse(trimmed);
-	} catch {
-		return parseJsonl(trimmed, path);
-	}
-	if (Array.isArray(parsed)) return records(parsed, path);
-	if (isRecord(parsed) && Array.isArray(parsed.data)) return records(parsed.data, `${path}.data`);
-	if (isRecord(parsed) && Array.isArray(parsed.cases)) return records(parsed.cases, `${path}.cases`);
-	if (isRecord(parsed)) return [parsed];
-	throw new TypeError(`public analyst benchmark dataset must contain JSON objects: ${path}`);
+async function readAndValidateManifestContent(path, content, currentIdentity, currentIdentitySha256, currentLocalIdentitySha256) {
+	const value = parseJson(content, path);
+	if (!isRecord$1(value)) throw new TypeError(`benchmark run manifest must be an object: ${path}`);
+	assertExactKeys(value, [
+		"kind",
+		"createdAt",
+		"identitySha256",
+		"localIdentitySha256",
+		"identity"
+	], "benchmark run manifest");
+	if (value.kind !== "agent-eval/analyst-benchmark-run") throw new TypeError(`unsupported benchmark run manifest: ${path}`);
+	if (typeof value.createdAt !== "string" || !Number.isFinite(Date.parse(value.createdAt))) throw new TypeError(`benchmark run manifest has an invalid createdAt: ${path}`);
+	if (!isSha256(value.identitySha256) || !isSha256(value.localIdentitySha256) || !isRecord$1(value.identity)) throw new TypeError(`benchmark run manifest has an invalid identity: ${path}`);
+	if (digestCanonical(value.identity) !== value.identitySha256) throw new Error(`benchmark run manifest identity digest does not match its contents: ${path}`);
+	if (currentIdentitySha256 !== value.identitySha256 || currentLocalIdentitySha256 !== value.localIdentitySha256 || canonicalJson(currentIdentity) !== canonicalJson(value.identity)) throw new Error(`benchmark resume configuration or inputs do not match ${ANALYST_BENCHMARK_MANIFEST_FILE}`);
+	return {
+		kind: "agent-eval/analyst-benchmark-run",
+		createdAt: value.createdAt,
+		identitySha256: currentIdentitySha256,
+		localIdentitySha256: currentLocalIdentitySha256,
+		identity: currentIdentity
+	};
 }
-function selectPublicBenchmarkRows(dataset, rows, options) {
-	positiveSafeInteger(options.limit, "limit");
-	safeInteger(options.seed, "seed");
-	if (rows.length === 0) throw new Error("public analyst benchmark dataset has no rows");
-	const byId = /* @__PURE__ */ new Map();
-	for (const row of rows) {
-		const id = publicBenchmarkRowId(dataset, row);
-		if (byId.has(id)) throw new Error(`public analyst benchmark dataset repeats trajectory id '${id}'`);
-		byId.set(id, row);
-	}
-	return [...byId].sort(([left], [right]) => selectionKey(options.seed, left).localeCompare(selectionKey(options.seed, right)) || left.localeCompare(right)).slice(0, Math.min(options.limit, byId.size)).map(([, row]) => row);
+function renderInitializationComplete(manifest) {
+	return `${JSON.stringify({
+		kind: "agent-eval/analyst-benchmark-initialization-complete",
+		runIdentitySha256: manifest.identitySha256,
+		localIdentitySha256: manifest.localIdentitySha256,
+		createdAt: manifest.createdAt
+	}, null, 2)}\n`;
 }
-function publicBenchmarkDistributions(dataset, rows) {
-	const values = {
-		class: [],
-		agent: [],
-		model: [],
-		difficulty: [],
-		solved: []
+async function assertAbsentOrExact(path, expected, label) {
+	const existing = await readOptionalRegularFile(path, label);
+	if (existing !== void 0 && existing !== expected) throw new Error(`${label} does not exactly match interrupted initialization: ${path}`);
+}
+async function readOptionalRegularFile(path, label) {
+	if (!await regularFileExists(path)) return void 0;
+	return readRegularFile(path, label);
+}
+function createObservationAppender(path, runIdentitySha256, progress) {
+	let writes = Promise.resolve();
+	const seen = new Set(progress.observations.map(observationKey));
+	return (observation) => {
+		const write = writes.then(async () => {
+			assertAnalystBenchmarkObservation(observation, "benchmark observation");
+			const key = observationKey(observation);
+			if (seen.has(key)) throw new Error(`refusing duplicate benchmark observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}'`);
+			const rowWithoutDigest = {
+				sequence: progress.nextSequence,
+				runIdentitySha256,
+				previousRowSha256: progress.previousRowSha256,
+				observation
+			};
+			const row = {
+				...rowWithoutDigest,
+				rowSha256: digestCanonical(rowWithoutDigest)
+			};
+			await appendDurable(path, `${JSON.stringify(row)}\n`);
+			progress.nextSequence += 1;
+			progress.previousRowSha256 = row.rowSha256;
+			progress.observations.push(observation);
+			seen.add(key);
+		});
+		writes = write;
+		return write;
 	};
-	for (const row of rows) {
-		const benchmarkCase = dataset === "agentrx" ? agentRxBenchmarkCase(row, void 0) : codeTraceBenchCase(row, void 0);
-		values.class.push(dataset === "codetracebench" ? benchmarkCase.expectedIssues.length > 0 ? "positive" : row.solved === true ? "trusted-negative" : row.solved === false ? "unlabeled-failure" : "unlabeled-unknown" : benchmarkCase.expectedIssues[0]?.areas?.[0]);
-		values.agent.push(scalarDistributionValue(row.agent) ?? (dataset === "agentrx" ? rootAgent(row) : void 0));
-		values.model.push(scalarDistributionValue(row.model));
-		values.difficulty.push(scalarDistributionValue(row.difficulty));
-		values.solved.push(scalarDistributionValue(row.solved));
+}
+async function readProgress(path, runIdentitySha256, caseIds, repetitions, analystRunnerId) {
+	const rawLines = (await readRegularFile(path, "benchmark observation log")).split("\n");
+	if (rawLines.at(-1) === "") rawLines.pop();
+	const observations = [];
+	const seen = /* @__PURE__ */ new Set();
+	const executionIndexes = /* @__PURE__ */ new Set();
+	let previousRowSha256 = null;
+	const allowedCases = new Set(caseIds);
+	const plannedObservationCount = caseIds.length * 2 * repetitions;
+	for (const [index, line] of rawLines.entries()) {
+		if (!line.trim()) throw new Error(`benchmark observation log contains an empty row at line ${index + 1}`);
+		const parsed = parseJson(line, `${path}:${index + 1}`);
+		if (!isRecord$1(parsed)) throw new TypeError(`benchmark observation row ${index + 1} must be an object`);
+		assertExactKeys(parsed, [
+			"sequence",
+			"runIdentitySha256",
+			"previousRowSha256",
+			"observation",
+			"rowSha256"
+		], `benchmark observation row ${index + 1}`);
+		assertAnalystBenchmarkObservation(parsed.observation, `benchmark observation row ${index + 1}.observation`);
+		const observation = parsed.observation;
+		const key = observationKey(observation);
+		if (seen.has(key)) throw new Error(`duplicate benchmark observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}' at line ${index + 1}`);
+		if (parsed.sequence !== index) throw new Error(`benchmark observation row ${index + 1} has sequence ${String(parsed.sequence)}; expected ${index}`);
+		if (parsed.runIdentitySha256 !== runIdentitySha256) throw new Error(`benchmark observation row ${index + 1} belongs to another run`);
+		if (parsed.previousRowSha256 !== previousRowSha256) throw new Error(`benchmark observation row ${index + 1} breaks the digest chain`);
+		if (!isSha256(parsed.rowSha256)) throw new TypeError(`benchmark observation row ${index + 1} has an invalid digest`);
+		if (digestCanonical({
+			sequence: parsed.sequence,
+			runIdentitySha256: parsed.runIdentitySha256,
+			previousRowSha256: parsed.previousRowSha256,
+			observation
+		}) !== parsed.rowSha256) throw new Error(`benchmark observation row ${index + 1} digest does not match its contents`);
+		if (!allowedCases.has(observation.caseId) || observation.runnerId !== "empty" && observation.runnerId !== analystRunnerId || observation.repetition >= repetitions || observation.executionIndex >= plannedObservationCount) throw new Error(`benchmark observation row ${index + 1} does not match a planned case, runner, and repetition`);
+		if (executionIndexes.has(observation.executionIndex)) throw new Error(`duplicate benchmark executionIndex ${observation.executionIndex} at line ${index + 1}`);
+		observations.push(observation);
+		seen.add(key);
+		executionIndexes.add(observation.executionIndex);
+		previousRowSha256 = parsed.rowSha256;
 	}
 	return {
-		class: valueDistribution(values.class),
-		agent: valueDistribution(values.agent),
-		model: valueDistribution(values.model),
-		difficulty: valueDistribution(values.difficulty),
-		solved: valueDistribution(values.solved)
+		observations,
+		nextSequence: observations.length,
+		previousRowSha256
 	};
 }
-function publicBenchmarkSelectionReport(dataset, source, selected, seed) {
-	const census = source.length === selected.length;
-	return {
-		method: census ? "census" : "deterministic-hash",
-		seed,
-		sourceCount: source.length,
-		selectedCount: selected.length,
-		stratified: false,
-		representativeOfInput: census,
-		source: publicBenchmarkDistributions(dataset, source),
-		selected: publicBenchmarkDistributions(dataset, selected)
-	};
-}
-async function preparePublicAnalystBenchmark(options) {
-	const labelsPath = resolve(options.labelsPath);
-	const traceRoot = resolve(options.traceDir);
-	const artifactRoot = options.artifactDir ? resolve(options.artifactDir) : void 0;
-	const labelSnapshot = await readImmutableInputSnapshot(labelsPath, DEFAULT_MAX_PUBLIC_BENCHMARK_LABEL_BYTES);
-	const rows = parsePublicBenchmarkRows(labelSnapshot.text, labelsPath);
-	const selected = selectPublicBenchmarkRows(options.dataset, rows, {
-		limit: options.limit,
-		seed: options.seed
-	});
-	const stores = await indexSelectedSingleTraceFiles(traceRoot, new Set(selected.map((row) => publicBenchmarkRowId(options.dataset, row))));
-	const resolver = traceStoreEvidenceResolver((input) => {
-		if (!input.traceStore) throw new Error("prepared benchmark case has no trace store");
-		return input.traceStore;
-	});
-	const traceFiles = [];
-	const verificationArtifacts = [];
-	const cases = [];
-	for (const row of selected) {
-		const trajectoryId = publicBenchmarkRowId(options.dataset, row);
-		const indexed = stores.get(trajectoryId);
-		if (!indexed) throw new Error(`public analyst benchmark trace directory has no single-trace OTLP JSONL for '${trajectoryId}'`);
-		let modelVisibleOtlp = indexed.text;
-		let traceStore = indexed.store;
-		let artifactDir;
-		let verificationManifest;
-		if (options.dataset === "codetracebench") {
-			if (!options.artifactDir?.trim()) throw new Error("--artifact-dir is required for CodeTraceBench so final verification evidence is not omitted");
-			const artifacts = await loadCodeTraceVerificationArtifacts({
-				artifactDir: options.artifactDir,
-				row,
-				maxBytes: options.maxArtifactBytes ?? 8388608
-			});
-			for (const artifact of artifacts.files) assertNoBenchmarkLabelsInArtifact({
-				traceId: trajectoryId,
-				relativePath: artifact.relativePath,
-				content: artifact.content
-			});
-			verificationManifest = shareableVerificationManifest(artifacts.manifest, artifactRoot ?? resolve(options.artifactDir));
-			const collisions = await indexed.store.hasSpans({
-				trace_id: trajectoryId,
-				span_ids: [...artifacts.manifest.files.map((file) => file.spanId), artifacts.manifest.outcomeSpanId]
-			});
-			if (collisions.length > 0) throw new Error(`CodeTraceBench '${trajectoryId}' trace already contains benchmark verification span '${collisions[0]}'`);
-			modelVisibleOtlp = appendVerificationArtifactsToOtlp(indexed.text, trajectoryId, artifacts, indexed.latestTimestamp);
-			traceStore = otlpTextToTraceAnalysisStore(modelVisibleOtlp);
-			artifactDir = artifacts.manifest.status === "present" ? artifacts.manifest.caseDirectory : void 0;
-			verificationArtifacts.push(verificationManifest);
-		}
-		const labelLeakScan = assertNoBenchmarkLabelsInTrace({
-			traceId: trajectoryId,
-			otlpText: modelVisibleOtlp
-		});
-		const input = {
-			traceStore,
-			artifactDir
-		};
-		const benchmarkCase = options.dataset === "agentrx" ? agentRxBenchmarkCase(row, input, { stepCount: indexed.stepCount }) : codeTraceBenchCase(row, input);
-		for (const evidence of benchmarkCase.labeledEvidence ?? []) if (!await resolver({
-			caseId: benchmarkCase.id,
-			caseInput: input,
-			evidence: {
-				kind: evidence.kind ?? "span",
-				uri: evidence.uri
-			}
-		})) throw new Error(`${benchmarkCase.id}: missing labeled span ${spanIdFromEvidence(evidence.uri) ?? evidence.uri} in ${indexed.path}`);
-		cases.push({
-			...benchmarkCase,
-			metadata: {
-				...benchmarkCase.metadata,
-				traceFileRelativePath: slashRelative(traceRoot, indexed.path),
-				traceFileSha256: indexed.sha256,
-				labelLeakScan,
-				...verificationManifest ? { verificationArtifacts: verificationManifest } : {}
-			}
-		});
-		traceFiles.push({
-			traceId: trajectoryId,
-			relativePath: slashRelative(traceRoot, indexed.path),
-			sha256: indexed.sha256
-		});
+async function writeExclusiveOrVerify(path, content) {
+	try {
+		await writeExclusive(path, content);
+	} catch (error) {
+		if (!isNodeError(error, "EEXIST")) throw error;
+		if (await readRegularFile(path, "existing benchmark artifact") !== content) throw new Error(`refusing to replace existing benchmark artifact: ${path}`);
 	}
-	return {
-		cases,
-		sourceRowCount: rows.length,
-		selectedCaseIds: cases.map((testCase) => testCase.id),
-		labelsSha256: labelSnapshot.sha256,
-		traceFiles,
-		verificationArtifacts,
-		selection: publicBenchmarkSelectionReport(options.dataset, rows, selected, options.seed)
-	};
 }
-async function indexSelectedSingleTraceFiles(traceDir, selectedTraceIds) {
-	if (selectedTraceIds.size === 0) throw new Error("public analyst benchmark selected no trace ids");
-	const files = (await readdir(traceDir, { withFileTypes: true })).filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")).map((entry) => resolve(traceDir, entry.name)).sort();
-	if (files.length === 0) throw new Error(`public analyst benchmark trace directory has no JSONL files: ${traceDir}`);
-	const indexed = /* @__PURE__ */ new Map();
-	for (const path of files) {
-		const snapshot = await readImmutableInputSnapshot(path, DEFAULT_MAX_TRACE_FILE_BYTES);
-		const store = createOtlpBufferTraceStore(snapshot.bytes);
-		const overview = await store.getOverview();
-		if (overview.total_traces !== 1 || overview.sample_trace_ids.length !== 1) throw new Error(`public analyst benchmark trace file must contain exactly one trace: ${path} contains ${overview.total_traces}`);
-		if (!overview.time_range) throw new Error(`public analyst benchmark trace file has no valid timestamps: ${path}`);
-		const traceId = overview.sample_trace_ids[0];
-		if (!selectedTraceIds.has(traceId)) continue;
-		if (indexed.has(traceId)) throw new Error(`public analyst benchmark trace id '${traceId}' appears in multiple files`);
-		indexed.set(traceId, {
-			path,
-			sha256: snapshot.sha256,
-			store,
-			latestTimestamp: overview.time_range.latest,
-			text: snapshot.text,
-			stepCount: traceStepCount(snapshot.text, path)
-		});
+async function regularFileExists(path) {
+	try {
+		const fileStat = await lstat(path);
+		if (!fileStat.isFile() || fileStat.isSymbolicLink()) throw new Error(`benchmark artifact path must be a real file: ${path}`);
+		return true;
+	} catch (error) {
+		if (isNodeError(error, "ENOENT")) return false;
+		throw error;
 	}
-	return indexed;
 }
-async function readImmutableInputSnapshot(path, maxBytes) {
-	if (!Number.isSafeInteger(maxBytes) || maxBytes < 1) throw new RangeError("benchmark input maxBytes must be a positive safe integer");
-	const handle = await open(path, INPUT_OPEN_FLAGS);
+async function appendDurable(path, content) {
+	const handle = await open(path, constants.O_APPEND | constants.O_WRONLY | constants.O_NOFOLLOW);
 	try {
-		return await readImmutableInputHandle(handle, path, maxBytes);
+		await handle.writeFile(content, "utf8");
+		await handle.sync();
 	} finally {
 		await handle.close();
 	}
 }
-async function readImmutableInputHandle(handle, path, maxBytes) {
-	const before = await handle.stat({ bigint: true });
-	if (!before.isFile()) throw new TypeError(`public analyst benchmark input must be a regular file: ${path}`);
-	if (before.size > BigInt(maxBytes)) throw new RangeError(`public analyst benchmark input exceeds ${maxBytes} bytes: ${path} has ${before.size}`);
-	const size = Number(before.size);
-	const bytes = Buffer.allocUnsafe(size);
-	let offset = 0;
-	while (offset < size) {
-		const result = await handle.read(bytes, offset, size - offset, offset);
-		if (result.bytesRead === 0) throw new Error(`public analyst benchmark input changed while being read: ${path}`);
-		offset += result.bytesRead;
+async function writeExclusive(path, content) {
+	const temporary = `${path}.tmp-${process.pid}-${randomUUID()}`;
+	let handle;
+	try {
+		handle = await open(temporary, "wx");
+		await handle.writeFile(content, "utf8");
+		await handle.sync();
+		await handle.close();
+		handle = void 0;
+		await link(temporary, path);
+		await syncDirectory(dirname(path));
+	} finally {
+		await handle?.close().catch(() => void 0);
+		await unlink(temporary).catch(() => void 0);
 	}
-	const overflow = Buffer.allocUnsafe(1);
-	const extra = await handle.read(overflow, 0, 1, size);
-	const after = await handle.stat({ bigint: true });
-	if (extra.bytesRead !== 0 || before.dev !== after.dev || before.ino !== after.ino || before.size !== after.size || before.mtimeNs !== after.mtimeNs || before.ctimeNs !== after.ctimeNs) throw new Error(`public analyst benchmark input changed while being read: ${path}`);
-	let text;
+}
+async function readRegularFile(path, label) {
+	let fileStat;
 	try {
-		text = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
+		fileStat = await lstat(path);
 	} catch (error) {
-		throw new TypeError(`public analyst benchmark input is not valid UTF-8: ${path}: ${error instanceof Error ? error.message : String(error)}`);
+		if (isNodeError(error, "ENOENT")) throw new Error(`${label} is missing: ${path}`);
+		throw error;
 	}
-	return Object.freeze({
-		bytes,
-		sha256: sha256Digest(bytes),
-		text
-	});
+	if (!fileStat.isFile() || fileStat.isSymbolicLink()) throw new Error(`${label} must be a real file: ${path}`);
+	return readFile(path, "utf8");
 }
-function traceStepCount(text, path) {
-	const steps = parseJsonl(text, path).map((row) => row.span_id).filter((spanId) => typeof spanId === "string").map((spanId) => /^step-(\d+)$/.exec(spanId)?.[1]).filter((step) => step !== void 0).map(Number).filter((step) => Number.isSafeInteger(step) && step > 0);
-	if (steps.length === 0) throw new Error(`public analyst benchmark trace has no step-<n> spans: ${path}`);
-	return Math.max(...steps);
+async function syncDirectory(path) {
+	const directory = await open(path, "r");
+	try {
+		await directory.sync();
+	} finally {
+		await directory.close();
+	}
 }
-function shareableVerificationManifest(manifest, artifactRoot) {
+function isNodeError(error, code) {
+	return error instanceof Error && "code" in error && error.code === code;
+}
+//#endregion
+//#region src/analyst/benchmark-comparison.ts
+function compareAnalystRunners(result, options) {
+	const confidence = options.confidence ?? .95;
+	const resamples = options.resamples ?? 2e3;
+	assertComparisonControls(confidence, resamples);
+	const runnerIds = new Set(result.summaries.map((summary) => summary.runnerId));
+	if (!runnerIds.has(options.baselineRunnerId)) throw new TypeError(`unknown baseline analyst runner '${options.baselineRunnerId}'`);
+	if (!runnerIds.has(options.candidateRunnerId)) throw new TypeError(`unknown candidate analyst runner '${options.candidateRunnerId}'`);
+	if (options.baselineRunnerId === options.candidateRunnerId) throw new TypeError("baseline and candidate analyst runners must be different");
+	const baseline = observationsByCase(result.observations, options.baselineRunnerId);
+	const candidate = observationsByCase(result.observations, options.candidateRunnerId);
+	const populationRepresentativenessProven = result.provenance.metadata?.populationRepresentativenessProven === true;
+	const metrics = METRICS.map((metric) => compareMetric({
+		metric,
+		baseline,
+		candidate,
+		confidence,
+		resamples,
+		seed: options.seed,
+		populationRepresentativenessProven
+	}));
 	return {
-		...manifest,
-		caseDirectory: slashRelative(artifactRoot, manifest.caseDirectory),
-		caseDirectoriesSearched: manifest.caseDirectoriesSearched.map((path) => slashRelative(artifactRoot, path)),
-		files: manifest.files.map((file) => ({
-			...file,
-			path: slashRelative(artifactRoot, file.path)
-		}))
+		baselineRunnerId: options.baselineRunnerId,
+		candidateRunnerId: options.candidateRunnerId,
+		metrics
 	};
 }
-function slashRelative(root, path) {
-	const value = relative(root, path);
-	if (!value || value === ".." || value.startsWith(`..${sep}`)) {
-		if (!value) return ".";
-		throw new Error(`benchmark artifact path escapes its declared root: ${path}`);
-	}
-	return value.replaceAll("\\", "/");
-}
-function parseJsonl(text, path) {
-	const rows = [];
-	for (const [index, line] of text.split(/\r?\n/).entries()) {
-		const trimmed = line.trim();
-		if (!trimmed) continue;
-		let parsed;
-		try {
-			parsed = JSON.parse(trimmed);
-		} catch (error) {
-			throw new Error(`${path}:${index + 1}: invalid JSON: ${error instanceof Error ? error.message : String(error)}`);
-		}
-		if (!isRecord(parsed)) throw new TypeError(`${path}:${index + 1}: dataset row must be a JSON object`);
-		rows.push(parsed);
-	}
-	if (rows.length === 0) throw new Error(`public analyst benchmark dataset is empty: ${path}`);
-	return rows;
-}
-function records(values, path) {
-	return values.map((value, index) => {
-		if (!isRecord(value)) throw new TypeError(`${path}[${index}] must be a JSON object`);
-		return value;
+function compareMetric(options) {
+	const pairedCases = [];
+	let eligibleObservations = 0;
+	let pairedObservations = 0;
+	let baselineMissingObservations = 0;
+	let candidateMissingObservations = 0;
+	let asymmetricMissingObservations = 0;
+	const caseIds = /* @__PURE__ */ new Set([...options.baseline.keys(), ...options.candidate.keys()]);
+	for (const caseId of caseIds) {
+		const baselineByRepetition = new Map((options.baseline.get(caseId) ?? []).map((observation) => [observation.repetition, observation]));
+		const candidateByRepetition = new Map((options.candidate.get(caseId) ?? []).map((observation) => [observation.repetition, observation]));
+		const caseBefore = [];
+		const caseAfter = [];
+		let clusterId;
+		const repetitions = /* @__PURE__ */ new Set([...baselineByRepetition.keys(), ...candidateByRepetition.keys()]);
+		for (const repetition of repetitions) {
+			const baselineObservation = baselineByRepetition.get(repetition);
+			const candidateObservation = candidateByRepetition.get(repetition);
+			const identity = baselineObservation ?? candidateObservation;
+			if (!identity || !metricApplies(identity, options.metric)) continue;
+			if (baselineObservation && candidateObservation) assertSameCaseIdentity(baselineObservation, candidateObservation);
+			eligibleObservations += 1;
+			clusterId = identity.clusterId;
+			const baselineValue = baselineObservation ? metricValue(baselineObservation, options.metric) : null;
+			const candidateValue = candidateObservation ? metricValue(candidateObservation, options.metric) : null;
+			const baselineMissing = baselineValue === null;
+			const candidateMissing = candidateValue === null;
+			if (baselineMissing) baselineMissingObservations += 1;
+			if (candidateMissing) candidateMissingObservations += 1;
+			if (baselineMissing !== candidateMissing) asymmetricMissingObservations += 1;
+			if (baselineMissing || candidateMissing) continue;
+			caseBefore.push(baselineValue);
+			caseAfter.push(candidateValue);
+			pairedObservations += 1;
+		}
+		if (caseBefore.length === 0 || !clusterId) continue;
+		pairedCases.push({
+			clusterId,
+			baseline: mean$1(caseBefore),
+			candidate: mean$1(caseAfter)
+		});
+	}
+	const byCluster = /* @__PURE__ */ new Map();
+	for (const pairedCase of pairedCases) {
+		const rows = byCluster.get(pairedCase.clusterId) ?? [];
+		rows.push(pairedCase);
+		byCluster.set(pairedCase.clusterId, rows);
+	}
+	const before = [...byCluster.values()].map((rows) => mean$1(rows.map((row) => row.baseline)));
+	const after = [...byCluster.values()].map((rows) => mean$1(rows.map((row) => row.candidate)));
+	const interval = before.length === 0 ? null : pairedBootstrap(before, after, {
+		confidence: options.confidence,
+		resamples: options.resamples,
+		statistic: "mean",
+		seed: options.seed
 	});
+	const survivorOnly = pairedObservations < eligibleObservations;
+	const limitations = [];
+	if (!interval?.gateEligible) limitations.push("fewer-than-20-independent-clusters");
+	if (!options.populationRepresentativenessProven) limitations.push("population-representativeness-not-proven");
+	if (survivorOnly) limitations.push("missing-observations");
+	const comparison = {
+		metric: options.metric,
+		direction: LOWER_IS_BETTER.has(options.metric) ? "lower" : "higher",
+		pairedCases: pairedCases.length,
+		pairedClusters: before.length,
+		eligibleObservations,
+		pairedObservations,
+		baselineMissingObservations,
+		candidateMissingObservations,
+		asymmetricMissingObservations,
+		survivorOnly,
+		baselineMean: before.length === 0 ? null : mean$1(before),
+		candidateMean: after.length === 0 ? null : mean$1(after),
+		meanDelta: interval?.mean ?? null,
+		intervalLow: interval?.low ?? null,
+		intervalHigh: interval?.high ?? null,
+		confidence: options.confidence,
+		resamples: options.resamples,
+		minimumSampleMet: interval?.gateEligible ?? false,
+		populationInferenceEligible: limitations.length === 0,
+		inferenceLimitations: limitations
+	};
+	assertValidComparison(comparison);
+	return comparison;
 }
-function publicBenchmarkRowId(dataset, row) {
-	const value = dataset === "agentrx" ? row.trajectory_id : row.traj_id;
-	if (typeof value !== "string" && typeof value !== "number" || !String(value).trim()) throw new TypeError(`${dataset} dataset row requires a non-empty ${dataset === "agentrx" ? "trajectory_id" : "traj_id"}`);
-	return String(value);
+const METRICS = [
+	"completion",
+	"issueRecall",
+	"findingPrecision",
+	"f1",
+	"criticalStepAccuracy",
+	"citationCoverage",
+	"citationExcerptCoverage",
+	"citationLabelAgreement",
+	"citationResolution",
+	"trustedNegativeAccuracy",
+	"latencyMs",
+	"calls",
+	"inputTokens",
+	"outputTokens",
+	"reasoningTokens",
+	"cachedTokens",
+	"cacheWriteTokens",
+	"costUsd"
+];
+const LOWER_IS_BETTER = /* @__PURE__ */ new Set([
+	"latencyMs",
+	"calls",
+	"inputTokens",
+	"outputTokens",
+	"reasoningTokens",
+	"cachedTokens",
+	"cacheWriteTokens",
+	"costUsd"
+]);
+function observationsByCase(observations, runnerId) {
+	const byCase = /* @__PURE__ */ new Map();
+	for (const observation of observations) {
+		if (observation.runnerId !== runnerId) continue;
+		const rows = byCase.get(observation.caseId) ?? [];
+		rows.push(observation);
+		byCase.set(observation.caseId, rows);
+	}
+	return byCase;
 }
-function spanIdFromEvidence(uri) {
-	const match = /\/span\/([^/]+)$/.exec(uri);
-	return match?.[1] ? decodeURIComponent(match[1]) : null;
+function assertSameCaseIdentity(baseline, candidate) {
+	if (baseline.clusterId !== candidate.clusterId || baseline.labelState !== candidate.labelState) throw new Error(`analyst comparison case identity differs for '${baseline.caseId}' repetition ${baseline.repetition}`);
 }
-function selectionKey(seed, id) {
-	return sha256Digest(`${seed}\u0000${id}`);
+function metricApplies(observation, metric) {
+	if (metric === "trustedNegativeAccuracy") return observation.labelState === "trusted-negative";
+	if (metric === "issueRecall" || metric === "findingPrecision" || metric === "f1") return observation.labelState === "positive";
+	if (metric === "criticalStepAccuracy") return observation.labelState === "positive" && observation.score.criticalStepAccuracy !== null;
+	return true;
 }
-function valueDistribution(values) {
-	const counts = /* @__PURE__ */ new Map();
-	let missing = 0;
-	for (const value of values) {
-		if (value === void 0) {
-			missing += 1;
-			continue;
-		}
-		counts.set(value, (counts.get(value) ?? 0) + 1);
+function metricValue(observation, metric) {
+	if (metric === "completion") return observation.error ? 0 : 1;
+	if (metric === "latencyMs") return observation.latencyMs;
+	if (metric === "trustedNegativeAccuracy") {
+		if (observation.error) return 0;
+		return observation.score.predictionOnLabelEmptyCase ? 0 : 1;
 	}
-	return {
-		total: values.length,
-		missing,
-		counts: Object.fromEntries([...counts].sort(([left], [right]) => left.localeCompare(right)))
-	};
+	if (observation.error && (metric === "issueRecall" || metric === "findingPrecision" || metric === "f1" || metric === "criticalStepAccuracy")) return 0;
+	if (observation.error && (metric === "citationCoverage" || metric === "citationExcerptCoverage" || metric === "citationLabelAgreement" || metric === "citationResolution")) return null;
+	if (metric === "issueRecall") return observation.score.issueRecall;
+	if (metric === "findingPrecision") return observation.score.findingPrecision;
+	if (metric === "f1") return observation.score.f1;
+	if (metric === "criticalStepAccuracy") return observation.score.criticalStepAccuracy;
+	if (metric === "citationCoverage") return observation.score.citationCoverage;
+	if (metric === "citationExcerptCoverage") return observation.score.citationExcerptCoverage;
+	if (metric === "citationLabelAgreement") return observation.score.citationLabelAgreement;
+	if (metric === "citationResolution") return observation.evidenceResolution?.validity ?? null;
+	if (metric === "calls") return observation.usage?.calls ?? null;
+	if (metric === "inputTokens") return observation.usage?.tokens?.input ?? null;
+	if (metric === "outputTokens") return observation.usage?.tokens?.output ?? null;
+	if (metric === "reasoningTokens") return observation.usage?.tokens?.reasoning ?? null;
+	if (metric === "cachedTokens") return observation.usage?.tokens?.cached ?? null;
+	if (metric === "cacheWriteTokens") return observation.usage?.tokens?.cacheWrite ?? null;
+	if (observation.usage?.cost.kind === "uncaptured") return null;
+	return observation.usage?.cost.usd ?? null;
 }
-function scalarDistributionValue(value) {
-	if (typeof value === "string") return value.trim() || void 0;
-	if (typeof value === "number" || typeof value === "boolean") return String(value);
+function mean$1(values) {
+	return values.reduce((sum, value) => sum + value, 0) / values.length;
 }
-function rootAgent(row) {
-	const rootCauseId = row.root_cause_failure_id ?? row.root_cause?.failure_id;
-	return scalarDistributionValue(row.failures.find((failure) => String(failure.failure_id) === String(rootCauseId))?.failed_agent);
+function assertComparisonControls(confidence, resamples) {
+	if (!Number.isSafeInteger(resamples) || resamples <= 0 || resamples > 1e6) throw new Error(`compareAnalystRunners: resamples must be a positive safe integer no greater than 1000000, got ${String(resamples)}`);
+	if (!Number.isFinite(confidence) || confidence <= 0 || confidence >= 1) throw new Error(`compareAnalystRunners: confidence must be a finite number in (0,1), got ${String(confidence)}`);
+}
+function assertValidComparison(comparison) {
+	if ([
+		"pairedCases",
+		"pairedClusters",
+		"eligibleObservations",
+		"pairedObservations",
+		"baselineMissingObservations",
+		"candidateMissingObservations",
+		"asymmetricMissingObservations",
+		"confidence",
+		"resamples"
+	].some((field) => !Number.isFinite(comparison[field])) || [
+		"baselineMean",
+		"candidateMean",
+		"meanDelta",
+		"intervalLow",
+		"intervalHigh"
+	].some((field) => comparison[field] !== null && !Number.isFinite(comparison[field]))) throw new Error(`compareAnalystRunners: ${comparison.metric} produced non-finite comparison output`);
+	if (comparison.intervalLow !== null && comparison.intervalHigh !== null && comparison.intervalLow > comparison.intervalHigh) throw new Error(`compareAnalystRunners: ${comparison.metric} produced an invalid confidence interval`);
 }
 //#endregion
-//#region src/analyst/benchmark-public-errors.ts
-function publicBenchmarkError(error, secrets = []) {
-	if (error instanceof LlmCallError) return {
-		class: "LlmCallError",
-		code: error.code,
-		status: error.status,
-		message: `Provider request failed with HTTP ${error.status}.`
-	};
-	if (error instanceof LlmResponseError) return {
-		class: "LlmResponseError",
-		code: error.code,
-		message: "Provider response did not satisfy the structured output contract."
-	};
-	if (error instanceof z.ZodError) return {
-		class: "ModelOutputValidationError",
-		message: "Provider response did not match the benchmark output schema."
-	};
-	if (error instanceof SyntaxError) return {
-		class: "ModelOutputParseError",
-		message: "Provider response was not valid JSON."
-	};
-	if (error instanceof CostCeilingReachedError || error instanceof CostAccountingIncompleteError || error instanceof CostReservationExceededError) return {
-		class: error.constructor.name,
-		code: error.code,
-		message: redactSensitiveText(error.message, secrets)
-	};
-	if (error instanceof Error && error.name === "AbortError") return {
-		class: "ProviderTimeoutError",
-		message: "Provider request timed out."
-	};
-	if (error instanceof Error && /(?:assistant steps?|finding evidence|selected missing|selected unavailable|no readable spans|requires a trace store)/i.test(error.message)) return {
-		class: "BenchmarkEvidenceError",
-		message: redactSensitiveText(error.message, secrets)
-	};
-	if (error instanceof AgentEvalError) return {
-		class: error.constructor.name,
-		code: error.code,
-		message: redactSensitiveText(error.message, secrets)
-	};
-	if (error instanceof Error) return {
-		class: error.constructor.name || "Error",
-		message: redactSensitiveText(error.message, secrets)
-	};
-	return {
-		class: "Error",
-		message: "Benchmark analyst execution failed."
+//#region src/analyst/benchmark-public-calibration.ts
+function summarizeCodeTraceCalibration(result) {
+	return {
+		protocol: "labeled-positive-and-solved-negative",
+		rationale: "Uses rows with incorrect-step labels as positives and solved label-empty rows as trusted negatives. Failed label-empty rows remain in the published result but are not treated as clean controls.",
+		runners: result.provenance.runnerIds.map((runnerId) => summarizeRunner(runnerId, result.observations.filter((observation) => observation.runnerId === runnerId)))
 	};
 }
-function redactSensitiveText(value, secrets) {
-	let redacted = value;
-	for (const secret of secrets) if (secret) redacted = redacted.replaceAll(secret, "[REDACTED]");
-	redacted = redacted.replace(/\bBearer\s+[^\s"',;]+/gi, "Bearer [REDACTED]").replace(/\b(api[_-]?key|access[_-]?token|refresh[_-]?token|password|secret)\b\s*[:=]\s*[^\s"',;]+/gi, "$1=[REDACTED]");
-	if (redacted.length <= 500) return redacted;
-	const head = redacted.slice(0, 180);
-	const marker = `...[${redacted.length - 460} chars omitted]...`;
-	return `${head}${marker}${redacted.slice(-(500 - head.length - marker.length))}`;
+function renderCodeTraceCalibrationMarkdown(summary) {
+	return [
+		"## CodeTraceBench Calibrated View",
+		"",
+		summary.rationale,
+		"",
+		"| Runner | Completed/selected | Failed | Positive runs | Trusted negative runs | Unlabeled runs | Failed label-empty | Unknown label-empty | Matched/expected steps | Predicted steps | Precision | Recall | F1 | Official all-row F1 | Official rows | Trusted-negative false positives | Trusted-negative failures | Unlabeled predictions | Unlabeled failures |",
+		"| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
+		...summary.runners.map((runner) => `| ${escapeCell$1(runner.runnerId)} | ${runner.completedRuns}/${runner.selectedRuns} | ${runner.failedRuns} | ${runner.positiveRuns} | ${runner.trustedNegativeRuns} | ${runner.unlabeledRuns} | ${runner.failedLabelEmptyRuns} | ${runner.unknownLabelEmptyRuns} | ${runner.matchedIncorrectSteps}/${runner.expectedIncorrectSteps} | ${runner.predictedIncorrectSteps} | ${rate$1(runner.precision)} | ${rate$1(runner.recall)} | ${rate$1(runner.f1)} | ${rate$1(runner.officialAllRowF1)} | ${runner.officialAllRowRuns} | ${rate$1(runner.trustedNegativeFalsePositiveRate)} | ${rate$1(runner.trustedNegativeFailureRate)} | ${rate$1(runner.unlabeledPredictionRate)} | ${rate$1(runner.unlabeledFailureRate)} |`)
+	].join("\n");
 }
-//#endregion
-//#region src/analyst/benchmark-response-cache.ts
-const SHA256 = /^[a-f0-9]{64}$/;
-const CostReceiptInputSchema = z.object({
-	model: z.string().min(1),
-	inputTokens: z.number().int().nonnegative(),
-	outputTokens: z.number().int().nonnegative(),
-	reasoningTokens: z.number().int().nonnegative().optional(),
-	cachedTokens: z.number().int().nonnegative().optional(),
-	cacheWriteTokens: z.number().int().nonnegative().optional(),
-	customTokenPricing: z.object({
-		inputUsdPerMillion: z.number().nonnegative(),
-		cachedInputUsdPerMillion: z.number().nonnegative().optional(),
-		cacheWriteUsdPerMillion: z.number().nonnegative().optional(),
-		outputUsdPerMillion: z.number().nonnegative()
-	}).strict().optional(),
-	actualCostUsd: z.number().nonnegative().optional(),
-	estimatedCostUsd: z.number().nonnegative().optional(),
-	costUnknown: z.boolean().optional(),
-	usageUnknown: z.boolean().optional()
-}).strict();
-const BenchmarkErrorSchema = z.object({
-	class: z.string().min(1),
-	message: z.string(),
-	code: z.string().min(1).optional(),
-	status: z.number().int().min(100).max(599).optional()
-}).strict();
-const ResponseMetadataSchema = z.object({
-	providerModel: z.string().min(1),
-	providerDurationMs: z.number().nonnegative(),
-	finishReason: z.string().nullable(),
-	producedAt: z.string().datetime()
-}).strict();
-const CacheIdentityShape = {
-	kind: z.literal("agent-eval/public-benchmark-model-response"),
-	callId: z.string().min(1),
-	runIdentitySha256: z.string().regex(SHA256),
-	caseId: z.string().min(1),
-	repetition: z.number().int().nonnegative()
-};
-const SuccessCacheEntryWithoutDigestSchema = z.object({
-	...CacheIdentityShape,
-	status: z.literal("succeeded"),
-	response: z.json(),
-	metadata: ResponseMetadataSchema,
-	receipt: CostReceiptInputSchema
-}).strict();
-const FailureCacheEntryWithoutDigestSchema = z.object({
-	...CacheIdentityShape,
-	status: z.literal("failed"),
-	error: BenchmarkErrorSchema,
-	receipt: CostReceiptInputSchema
-}).strict();
-const CacheEntryWithoutDigestSchema = z.discriminatedUnion("status", [SuccessCacheEntryWithoutDigestSchema, FailureCacheEntryWithoutDigestSchema]);
-const CacheEntrySchema = z.discriminatedUnion("status", [SuccessCacheEntryWithoutDigestSchema.extend({ entrySha256: z.string().regex(SHA256) }), FailureCacheEntryWithoutDigestSchema.extend({ entrySha256: z.string().regex(SHA256) })]);
-function publicBenchmarkCallId(identity) {
-	assertCacheIdentity(identity);
-	return `analyst-benchmark-${hashCanonical({
-		runIdentitySha256: identity.runIdentitySha256,
-		caseId: identity.caseId,
-		repetition: identity.repetition
-	}).slice(7)}`;
+function summarizeRunner(runnerId, observations) {
+	const positive = observations.filter((observation) => observation.labelState === "positive");
+	const trustedNegative = observations.filter((observation) => observation.labelState === "trusted-negative");
+	const excluded = observations.filter((observation) => observation.labelState === "unlabeled");
+	const selected = [...positive, ...trustedNegative];
+	const expected = sum(positive.map((observation) => observation.score.expectedIssueCount));
+	const predicted = sum(selected.map((observation) => observation.error ? 0 : observation.findings.length));
+	const matched = sum(positive.map((observation) => observation.score.matchedIssueIds.length));
+	const precision = predicted === 0 ? expected > 0 ? 0 : null : matched / predicted;
+	const recall = ratio(matched, expected);
+	const completedTrustedNegative = trustedNegative.filter((observation) => !observation.error);
+	const completedExcluded = excluded.filter((observation) => !observation.error);
+	const officialRows = observations.map(officialCodeTraceF1);
+	return {
+		runnerId,
+		selectedRuns: selected.length,
+		positiveRuns: positive.length,
+		trustedNegativeRuns: trustedNegative.length,
+		unlabeledRuns: excluded.length,
+		failedLabelEmptyRuns: excluded.filter((observation) => observation.caseMetadata?.solved === false).length,
+		unknownLabelEmptyRuns: excluded.filter((observation) => observation.caseMetadata?.solved !== false).length,
+		completedRuns: selected.filter((observation) => !observation.error).length,
+		failedRuns: selected.filter((observation) => observation.error).length,
+		expectedIncorrectSteps: expected,
+		predictedIncorrectSteps: predicted,
+		matchedIncorrectSteps: matched,
+		officialAllRowF1: mean(officialRows),
+		officialAllRowRuns: officialRows.length,
+		precision,
+		recall,
+		f1: harmonicMean(precision, recall),
+		trustedNegativeFalsePositiveRate: ratio(completedTrustedNegative.filter((observation) => observation.score.predictionOnLabelEmptyCase).length, completedTrustedNegative.length),
+		trustedNegativeFailureRate: ratio(trustedNegative.filter((observation) => observation.error).length, trustedNegative.length),
+		unlabeledPredictionRate: ratio(completedExcluded.filter((observation) => observation.findings.length > 0).length, completedExcluded.length),
+		unlabeledFailureRate: ratio(excluded.filter((observation) => Boolean(observation.error)).length, excluded.length)
+	};
 }
-function readPublicBenchmarkResponseCache(cacheDirectory, identity) {
-	const callId = publicBenchmarkCallId(identity);
-	const path = responseCachePath(cacheDirectory, callId);
-	if (!existsSync(path)) return void 0;
-	const metadata = lstatSync(path);
-	if (!metadata.isFile() || metadata.isSymbolicLink()) throw new ValidationError(`benchmark response cache must be a real file: ${path}`);
-	let parsed;
-	try {
-		parsed = JSON.parse(readFileSync(path, "utf8"));
-	} catch (error) {
-		throw new ValidationError(`benchmark response cache contains invalid JSON: ${path}`, { cause: error });
+function officialCodeTraceF1(observation) {
+	const trajectoryId = observation.caseMetadata?.trajectoryId;
+	if (typeof trajectoryId !== "string" || !trajectoryId.trim()) throw new TypeError(`${observation.caseId}: CodeTraceBench trajectoryId metadata is missing`);
+	const expected = new Set([...observation.score.matchedIssueIds, ...observation.score.missedIssueIds].map((issueId) => {
+		const match = /^incorrect:(\d+)$/.exec(issueId);
+		if (!match) throw new TypeError(`${observation.caseId}: invalid incorrect-step label '${issueId}'`);
+		return Number(match[1]);
+	}));
+	const predicted = /* @__PURE__ */ new Set();
+	if (!observation.error) for (const finding of observation.findings) {
+		if (finding.area !== "incorrect") continue;
+		for (const evidence of finding.evidence_refs) {
+			const location = codeTraceStepFromEvidence(evidence.uri);
+			if (!location || location.traceId !== trajectoryId) throw new TypeError(`${observation.caseId}: invalid CodeTraceBench prediction evidence '${evidence.uri}'`);
+			predicted.add(location.step);
+		}
 	}
-	const entry = parseCacheEntry(parsed, path);
-	if (entry.callId !== callId || entry.runIdentitySha256 !== identity.runIdentitySha256 || entry.caseId !== identity.caseId || entry.repetition !== identity.repetition) throw new ValidationError(`benchmark response cache identity does not match: ${path}`);
-	return entry;
+	let matched = 0;
+	for (const step of predicted) if (expected.has(step)) matched += 1;
+	return harmonicMean(predicted.size === 0 ? 0 : matched / predicted.size, expected.size === 0 ? 0 : matched / expected.size) ?? 0;
 }
-function writePublicBenchmarkResponseCache(cacheDirectory, entry) {
-	const expectedCallId = publicBenchmarkCallId(entry);
-	if (entry.callId !== expectedCallId) throw new ValidationError("benchmark response cache callId does not match its identity");
-	const validated = JSON.parse(JSON.stringify(CacheEntryWithoutDigestSchema.parse(entry)));
-	const complete = {
-		...validated,
-		entrySha256: hashCanonical(validated).slice(7)
-	};
-	const path = responseCachePath(cacheDirectory, complete.callId);
-	const content = `${canonicalString(complete)}\n`;
-	withLedgerFileLock(path, fileContext(), () => {
-		if (existsSync(path)) {
-			const existing = readPublicBenchmarkResponseCache(cacheDirectory, complete);
-			if (!existing || canonicalString(existing) !== canonicalString(complete)) throw new ValidationError(`benchmark response cache conflicts with existing file: ${path}`);
-			return;
-		}
-		writeLedgerFileAtomically(path, content, fileContext());
-	});
-	return complete;
+function sum(values) {
+	return values.reduce((total, value) => total + value, 0);
 }
-function parseCacheEntry(value, path) {
-	let parsed;
-	try {
-		parsed = CacheEntrySchema.parse(value);
-	} catch (error) {
-		throw new ValidationError(`benchmark response cache has an invalid shape: ${path}`, { cause: error });
-	}
-	const { entrySha256, ...withoutDigest } = parsed;
-	if (entrySha256 !== hashCanonical(withoutDigest).slice(7)) throw new ValidationError(`benchmark response cache digest does not match: ${path}`);
-	return parsed;
+function ratio(numerator, denominator) {
+	return denominator === 0 ? null : numerator / denominator;
 }
-function responseCachePath(cacheDirectory, callId) {
-	const directory = nodePath.resolve(cacheDirectory);
-	const path = nodePath.resolve(directory, `${hashCanonical(callId).slice(7)}.json`);
-	if (!isPathInsideDirectory(directory, path, nodePath)) throw new ValidationError("benchmark response cache path escapes its directory");
-	return path;
+function mean(values) {
+	return values.length === 0 ? null : sum(values) / values.length;
 }
-function isPathInsideDirectory(directory, candidate, pathOperations = nodePath) {
-	const relative = pathOperations.relative(directory, candidate);
-	return relative !== "" && relative !== ".." && !relative.startsWith(`..${pathOperations.sep}`) && !pathOperations.isAbsolute(relative);
+function harmonicMean(left, right) {
+	if (left === null || right === null) return null;
+	return left + right === 0 ? 0 : 2 * left * right / (left + right);
 }
-function assertCacheIdentity(identity) {
-	if (!SHA256.test(identity.runIdentitySha256)) throw new ValidationError("benchmark response cache requires a SHA-256 run identity");
-	if (!identity.caseId.trim()) throw new ValidationError("benchmark response cache requires a case id");
-	if (!Number.isSafeInteger(identity.repetition) || identity.repetition < 0) throw new ValidationError("benchmark response cache repetition must be non-negative");
+function rate$1(value) {
+	return value === null ? "n/a" : value.toFixed(3);
 }
-function fileContext() {
-	return {
-		subject: "benchmark response cache",
-		integrityError: (message, options) => new ValidationError(message, options)
-	};
+function escapeCell$1(value) {
+	return value.replaceAll("|", "\\|").replaceAll("\n", " ");
 }
 //#endregion
-//#region src/analyst/benchmark-public-model.ts
-/** One-shot JSON baseline. This is not a recursive trace analyst. */
-function createPublicBenchmarkDirectRunner(dataset, config) {
-	const model = requiredString(config.model, "model");
-	const baseUrl = requiredString(config.baseUrl, "baseUrl");
-	const apiKey = requiredString(config.apiKey, "apiKey");
-	const maxOutputTokens = positiveSafeInteger(config.maxOutputTokens, "maxOutputTokens");
-	const timeoutMs = positiveSafeInteger(config.timeoutMs, "timeoutMs");
-	const costLedger = config.costLedger ?? new CostLedger();
-	const durability = config.durability ? {
-		runIdentitySha256: requiredString(config.durability.runIdentitySha256, "durability.runIdentitySha256"),
-		responseCacheDir: requiredString(config.durability.responseCacheDir, "durability.responseCacheDir")
-	} : void 0;
-	const actor = dataset === "agentrx" ? "agentrx-root-cause-localizer" : "codetracebench-step-localizer";
-	const outputAdapter = dataset === "agentrx" ? "agentrx-taxonomy-and-root-step" : "codetracebench-incorrect-block";
-	const llmOptions = {
-		baseUrl,
-		apiKey,
-		maximumAttempts: 1,
-		jsonSchemaTransport: "json-object",
-		jsonPayloadMode: "exact",
-		thinking: "disabled",
-		...config.fetchImpl ? { fetch: config.fetchImpl } : {}
-	};
-	return {
-		id: "direct",
-		async analyze(input, context) {
-			const trajectoryId = trajectoryIdFromCaseId$1(dataset, context.caseId);
-			const costTags = {
-				analystId: actor,
-				benchmarkCaseId: context.caseId,
-				benchmarkRepetition: String(context.repetition)
-			};
-			let rawPredictions = [];
-			let rejectedBlocks = [];
-			let modelFindings = [];
-			let providerModel = model;
+//#region src/analyst/benchmark-command-result.ts
+async function readAnalystBenchmarkArtifact(path) {
+	const value = parseJson(await readRegularFile(path, "analyst benchmark result"), path);
+	assertAnalystBenchmarkArtifact(value, "analyst benchmark result");
+	return value;
+}
+function assertCompletedArtifactMatchesRun(artifact, manifest, observations, prepared) {
+	if (artifact.runIdentitySha256 !== manifest.identitySha256) throw new Error("completed benchmark result belongs to another run");
+	assertSameObservations(artifact.result.observations, observations);
+	const expectedCount = manifest.identity.inputs.selectedCaseIds.length * manifest.identity.config.runnerIds.length * manifest.identity.config.repetitions;
+	if (observations.length !== expectedCount) throw new Error(`completed benchmark result has ${observations.length} observations; expected ${expectedCount}`);
+	const { config, inputs } = manifest.identity;
+	const verificationAvailability = {
+		cases: prepared.verificationArtifacts.length,
+		resultFilesPresent: prepared.verificationArtifacts.filter((artifact) => artifact.status === "present").length,
+		resultFilesMissing: prepared.verificationArtifacts.filter((artifact) => artifact.status === "missing").length,
+		outcomes: {
+			passed: prepared.verificationArtifacts.filter((artifact) => artifact.outcome.status === "passed").length,
+			failed: prepared.verificationArtifacts.filter((artifact) => artifact.outcome.status === "failed").length,
+			unavailable: prepared.verificationArtifacts.filter((artifact) => artifact.outcome.status === "unavailable").length
+		}
+	};
+	const expectedInputs = {
+		dataset: config.dataset,
+		datasetRevision: config.datasetRevision,
+		datasetSplit: config.datasetSplit,
+		labelsSha256: inputs.labelsSha256,
+		sourceRowCount: inputs.sourceRowCount,
+		traceFiles: inputs.traceFiles.map((traceFile) => ({ ...traceFile })),
+		verificationArtifacts: prepared.verificationArtifacts,
+		verificationAvailability,
+		selection: {
+			limit: config.limit,
+			seed: config.seed,
+			selectedCaseIds: [...inputs.selectedCaseIds],
+			report: prepared.selection
+		},
+		execution: {
+			repetitions: config.repetitions,
+			concurrency: config.concurrency,
+			...config.rlmSamples === void 0 ? {} : { rlmSamples: config.rlmSamples },
+			model: config.model.id,
+			maxOutputTokens: config.model.maxOutputTokens,
+			timeoutMs: config.model.timeoutMs,
+			maxCostUsd: config.maxCostUsd,
+			maxArtifactBytes: config.maxArtifactBytes,
+			analystProtocolSha256: config.analystProtocolSha256,
+			...config.instructionsOverrideSha256 === void 0 ? {} : { instructionsOverrideSha256: config.instructionsOverrideSha256 },
+			implementationSha256: config.implementationSha256,
+			dependencyLockSha256: config.dependencyLockSha256
+		}
+	};
+	if (canonicalJson(artifact.inputs) !== canonicalJson(expectedInputs)) throw new Error("completed benchmark result inputs do not match the run manifest");
+	const provenance = artifact.result.provenance;
+	const expectedDatasetId = config.dataset === "agentrx" ? "microsoft/AgentRx" : "NJU-LINK/CodeTraceBench";
+	const expectedOutputAdapter = config.dataset === "agentrx" ? "agentrx-taxonomy-and-root-step" : "codetracebench-incorrect-block";
+	if (provenance.id !== `${config.dataset}-real-model-analyst` || provenance.startedAt !== manifest.createdAt || !Number.isFinite(Date.parse(provenance.endedAt)) || Date.parse(provenance.endedAt) < Date.parse(provenance.startedAt) || canonicalJson(provenance.dataset) !== canonicalJson({
+		id: expectedDatasetId,
+		revision: config.datasetRevision,
+		split: config.datasetSplit
+	}) || provenance.caseCount !== inputs.selectedCaseIds.length || canonicalJson(provenance.runnerIds) !== canonicalJson(config.runnerIds) || provenance.repetitions !== config.repetitions || provenance.maxConcurrency !== Math.min(config.concurrency, expectedCount) || provenance.runnerOrderSeed !== config.seed || provenance.metadata?.model !== config.model.id || provenance.metadata?.rlmSamples !== config.rlmSamples || provenance.metadata?.outputAdapter !== expectedOutputAdapter || provenance.metadata?.caseSelection !== prepared.selection.method || provenance.metadata?.caseSelectionSeed !== config.seed || provenance.metadata?.selectionStratified !== prepared.selection.stratified || provenance.metadata?.protocolSha256 !== config.analystProtocolSha256 || provenance.metadata?.implementationSha256 !== config.implementationSha256 || provenance.metadata?.dependencyLockSha256 !== config.dependencyLockSha256 || provenance.metadata?.populationRepresentativenessProven !== false) throw new Error("completed benchmark result provenance does not match the run manifest");
+	const expectedSummaries = config.runnerIds.map((runnerId) => summarizeAnalystBenchmarkRunner(runnerId, observations.filter((observation) => observation.runnerId === runnerId)));
+	if (canonicalJson(artifact.result.summaries) !== canonicalJson(expectedSummaries)) throw new Error("completed benchmark summaries do not match durable observations");
+	const expectedComparisons = [compareAnalystRunners(artifact.result, {
+		baselineRunnerId: "empty",
+		candidateRunnerId: config.runnerIds[1],
+		seed: config.seed
+	})];
+	if (canonicalJson(artifact.comparisons) !== canonicalJson(expectedComparisons)) throw new Error("completed benchmark comparisons do not match durable observations");
+	if (config.dataset === "codetracebench") {
+		if (canonicalJson(artifact.codeTraceCalibration) !== canonicalJson(summarizeCodeTraceCalibration(artifact.result)) || artifact.agentRxCalibration !== void 0) throw new Error("completed CodeTraceBench calibration does not match durable observations");
+	} else if (canonicalJson(artifact.agentRxCalibration) !== canonicalJson(summarizeAgentRxCalibration(artifact.result, "f228165bfec60a801fd5fedd9d8ffe0f9de0c69d")) || artifact.codeTraceCalibration !== void 0) throw new Error("completed AgentRx calibration does not match durable observations");
+}
+function assertSameObservations(expected, actual) {
+	if (expected.length !== actual.length) throw new Error(`benchmark result has ${expected.length} observations but the durable log has ${actual.length}`);
+	const expectedByKey = new Map(expected.map((observation) => [observationKey(observation), canonicalJson(observation)]));
+	for (const observation of actual) {
+		const key = observationKey(observation);
+		if (expectedByKey.get(key) !== canonicalJson(observation)) throw new Error(`benchmark result does not match durable observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}'`);
+		expectedByKey.delete(key);
+	}
+	if (expectedByKey.size > 0) throw new Error("benchmark result is missing durable observations");
+}
+//#endregion
+//#region src/analyst/benchmark-public-adapters.ts
+function emptyPublicBenchmarkRunner() {
+	return {
+		id: "empty",
+		analyze() {
+			return {
+				findings: [],
+				usage: {
+					calls: 0,
+					tokens: {
+						input: 0,
+						output: 0
+					},
+					cost: {
+						kind: "observed",
+						usd: 0
+					}
+				},
+				metadata: { baseline: "emit-no-findings" }
+			};
+		}
+	};
+}
+async function adaptPublicBenchmarkFindings(options) {
+	if (options.dataset === "agentrx") return {
+		findings: adaptAgentRxFindings(options.trajectoryId, options.findings, options.analystId),
+		diagnostics: void 0
+	};
+	return adaptCodeTraceFindings(options.trajectoryId, options.findings, options.analystId, options.store, options.signal);
+}
+function adaptAgentRxFindings(trajectoryId, findings, analystId) {
+	if (findings.length === 0) return [];
+	if (findings.length !== 1) throw new Error(`AgentRx model analyst must emit zero or one root cause, received ${findings.length}`);
+	const source = findings[0];
+	if (!source.subject) throw new Error("AgentRx model analyst finding is missing its failure-category subject");
+	const steps = exactFindingSteps(trajectoryId, source);
+	if (steps.length !== 1) throw new Error(`AgentRx model analyst must cite exactly one root-cause step, received ${steps.length}`);
+	const [adapted] = agentRxPredictionsToFindings(trajectoryId, [{
+		failure_case: source.subject,
+		step_number: steps[0],
+		description: source.rationale ?? source.claim
+	}], {
+		analystId,
+		producedAt: source.produced_at,
+		confidence: source.confidence
+	});
+	if (!adapted) throw new Error("AgentRx output adapter produced no root-cause finding");
+	return [{
+		...adapted,
+		metadata: {
+			...adapted.metadata,
+			sourceFindingId: source.finding_id
+		}
+	}];
+}
+const CODE_TRACE_BLOCK_SUBJECT = /^incorrect-steps-(\d+)-(\d+)-(escaped|unescaped)-consequence-(\d+)$/;
+async function adaptCodeTraceFindings(trajectoryId, findings, analystId, store, signal) {
+	const clean = findings.filter((finding) => finding.subject === "clean");
+	if (clean.length > 0) {
+		if (findings.length !== 1) throw new Error("CodeTraceBench model analyst mixed a clean verdict with incorrect steps");
+		exactFindingSteps(trajectoryId, clean[0]);
+		return {
+			findings: [],
+			diagnostics: emptyCodeTraceBlockDiagnostics(),
+			stepBlocks: []
+		};
+	}
+	const blocks = [];
+	const rejectedFindings = [];
+	const trimmedCitations = [];
+	for (const source of findings) try {
+		await validateCodeTraceFindingEvidence({
+			trajectoryId,
+			findings: [source],
+			store,
+			...signal ? { signal } : {}
+		});
+		const converted = codeTraceBlockFromFinding(trajectoryId, source);
+		trimmedCitations.push(...converted.trimmedCitations);
+		blocks.push(converted.block);
+	} catch (error) {
+		rejectedFindings.push(`${source.finding_id}: ${error instanceof Error ? error.message : String(error)}`);
+	}
+	const expanded = await expandCodeTraceFailureBlocks({
+		trajectoryId,
+		blocks,
+		store,
+		analystId,
+		...findings[0] ? { producedAt: findings[0].produced_at } : {},
+		...signal ? { signal } : {}
+	});
+	return {
+		findings: expanded.findings,
+		diagnostics: {
+			...expanded.diagnostics,
+			rejectedFindings,
+			trimmedCitations
+		},
+		stepBlocks: expanded.stepBlocks
+	};
+}
+/**
+* Block coordinates recoverable from a subject in the block grammar, in the
+* metadata field names the expanded findings carry. Returns undefined when the
+* subject does not parse — nothing is invented for a malformed subject.
+*/
+function codeTraceBlockMetadataFromSubject(subject) {
+	const parsed = CODE_TRACE_BLOCK_SUBJECT.exec(subject ?? "");
+	if (!parsed) return void 0;
+	return {
+		block_first_step: Number(parsed[1]),
+		block_last_step: Number(parsed[2]),
+		block_consequence_step: Number(parsed[4]),
+		escape_status: parsed[3]
+	};
+}
+function codeTraceBlockFromFinding(trajectoryId, source) {
+	const parsed = CODE_TRACE_BLOCK_SUBJECT.exec(source.subject ?? "");
+	if (!parsed) throw new Error(`CodeTraceBench model finding '${source.finding_id}' must set subject to incorrect-steps-<first>-<last>-<escaped|unescaped>-consequence-<step>, received '${source.subject ?? ""}'`);
+	const firstStep = Number(parsed[1]);
+	const lastStep = Number(parsed[2]);
+	const consequenceStep = Number(parsed[4]);
+	const cited = exactFindingSteps(trajectoryId, source);
+	const outOfRange = cited.filter((step) => step < firstStep || step > lastStep);
+	if (outOfRange.length === cited.length) throw new Error(`CodeTraceBench model finding '${source.finding_id}' cites step ${outOfRange.join(", ")} outside its block ${firstStep}-${lastStep} and no citation falls inside the block`);
+	const trimmedCitations = outOfRange.map((step) => `${source.finding_id}: trimmed citation step ${step} outside block ${firstStep}-${lastStep}`);
+	return {
+		block: {
+			firstStep,
+			lastStep,
+			consequenceStep,
+			escapeStatus: parsed[3],
+			severity: source.severity,
+			claim: source.claim,
+			confidence: source.confidence,
+			...source.rationale === void 0 ? {} : { rationale: source.rationale },
+			...source.recommended_action === void 0 ? {} : { recommendedAction: source.recommended_action },
+			metadata: { sourceFindingId: source.finding_id }
+		},
+		trimmedCitations
+	};
+}
+/**
+* Expand contiguous failure blocks into one scored finding per member step.
+*
+* The official scorer matches on area plus the exact step evidence URI, so
+* blocks never reach it: every runner reports blocks, and this function turns
+* them into the per-step findings the benchmark defines.
+*/
+async function expandCodeTraceFailureBlocks(options) {
+	const diagnostics = emptyCodeTraceBlockDiagnostics();
+	diagnostics.reportedBlocks = options.blocks.length;
+	if (options.blocks.length === 0) return {
+		findings: [],
+		diagnostics,
+		stepBlocks: []
+	};
+	const blocks = acceptCodeTraceBlockShape(options.blocks, diagnostics);
+	if (blocks.length === 0) return {
+		findings: [],
+		diagnostics,
+		stepBlocks: []
+	};
+	const boundarySteps = blocks.flatMap((block) => [block.firstStep, block.lastStep]);
+	const derivedSteps = blocks.flatMap((block) => [block.consequenceStep, ...interiorSteps(block)]);
+	const evidenceByStep = await resolveAssistantStepEvidence({
+		trajectoryId: options.trajectoryId,
+		steps: boundarySteps,
+		optionalSteps: derivedSteps,
+		store: options.store,
+		...options.signal ? { signal: options.signal } : {}
+	});
+	const byStep = /* @__PURE__ */ new Map();
+	for (const block of blocks) {
+		if (!evidenceByStep.has(block.consequenceStep)) {
+			diagnostics.blocksWithoutConsequenceEvidence.push(block);
+			continue;
+		}
+		if (block.escapeStatus === "escaped") diagnostics.escapedBlocks += 1;
+		for (let step = block.firstStep; step <= block.lastStep; step += 1) {
+			if (!evidenceByStep.has(step)) {
+				diagnostics.unresolvedBlockInteriorSteps.push(step);
+				continue;
+			}
+			if (byStep.has(step)) {
+				diagnostics.overlappingBlockSteps.push(step);
+				continue;
+			}
+			byStep.set(step, block);
+		}
+	}
+	const stepBlocks = [...byStep].sort(([left], [right]) => left - right).map(([step, block]) => ({
+		step,
+		block
+	}));
+	return {
+		findings: stepBlocks.map(({ step, block }) => makeFinding({
+			analyst_id: options.analystId,
+			area: "incorrect",
+			subject: `incorrect-step-${step}`,
+			claim: `Step ${step} is incorrect. ${block.claim}`,
+			rationale: block.rationale,
+			severity: block.severity,
+			confidence: block.confidence,
+			evidence_refs: [evidenceByStep.get(step)],
+			recommended_action: block.recommendedAction,
+			metadata: {
+				...block.metadata,
+				block_first_step: block.firstStep,
+				block_last_step: block.lastStep,
+				block_consequence_step: block.consequenceStep,
+				escape_status: block.escapeStatus
+			},
+			...options.producedAt === void 0 ? {} : { produced_at: options.producedAt },
+			id_basis: `incorrect-step-${step}`
+		})),
+		diagnostics,
+		stepBlocks
+	};
+}
+/**
+* Enforce the protocol's per-block and per-case limits without voiding the
+* case: an offending block is dropped and named in `diagnostics.droppedBlocks`
+* while every valid sibling survives. A case whose blocks are ALL invalid ends
+* empty and carries the diagnostic for each drop. Shape is checked before the
+* count, so a malformed block never consumes one of the accepted slots.
+*/
+function acceptCodeTraceBlockShape(blocks, diagnostics) {
+	const accepted = [];
+	for (const block of blocks) {
+		const reason = codeTraceBlockShapeViolation(block) ?? (accepted.length >= 16 ? `model reported ${blocks.length} failure blocks; the maximum is 16` : void 0);
+		if (reason) {
+			diagnostics.droppedBlocks.push(`block ${block.firstStep}-${block.lastStep} (consequence ${block.consequenceStep}): ${reason}`);
+			continue;
+		}
+		accepted.push(block);
+	}
+	return accepted;
+}
+function codeTraceBlockShapeViolation(block) {
+	if (block.lastStep < block.firstStep) return `failure block last_step ${block.lastStep} precedes first_step ${block.firstStep}`;
+	const length = block.lastStep - block.firstStep + 1;
+	if (length > 12) return `failure block spans ${length} steps; the maximum is 12`;
+	if (block.consequenceStep < block.firstStep) return `failure block consequence_step ${block.consequenceStep} precedes first_step ${block.firstStep}`;
+}
+function interiorSteps(block) {
+	const steps = [];
+	for (let step = block.firstStep + 1; step < block.lastStep; step += 1) steps.push(step);
+	return steps;
+}
+function emptyCodeTraceBlockDiagnostics() {
+	return {
+		reportedBlocks: 0,
+		escapedBlocks: 0,
+		blocksWithoutConsequenceEvidence: [],
+		unresolvedBlockInteriorSteps: [],
+		overlappingBlockSteps: [],
+		droppedBlocks: []
+	};
+}
+function exactFindingSteps(trajectoryId, finding) {
+	if (finding.evidence_refs.length === 0) throw new Error(`model finding '${finding.finding_id}' has no step evidence`);
+	const steps = finding.evidence_refs.map((evidence) => {
+		const parsed = codeTraceStepFromEvidence(evidence.uri);
+		if (!parsed || parsed.traceId !== trajectoryId) throw new Error(`model finding '${finding.finding_id}' cites non-case evidence '${evidence.uri}'`);
+		return parsed.step;
+	});
+	return [...new Set(steps)];
+}
+//#endregion
+//#region src/analyst/benchmark-public-errors.ts
+function publicBenchmarkError(error, secrets = []) {
+	if (error instanceof LlmCallError) return {
+		class: "LlmCallError",
+		code: error.code,
+		status: error.status,
+		message: `Provider request failed with HTTP ${error.status}.`
+	};
+	if (error instanceof LlmResponseError) return {
+		class: "LlmResponseError",
+		code: error.code,
+		message: "Provider response did not satisfy the structured output contract."
+	};
+	if (error instanceof z.ZodError) return {
+		class: "ModelOutputValidationError",
+		message: "Provider response did not match the benchmark output schema."
+	};
+	if (error instanceof SyntaxError) return {
+		class: "ModelOutputParseError",
+		message: "Provider response was not valid JSON."
+	};
+	if (error instanceof CostCeilingReachedError || error instanceof CostAccountingIncompleteError || error instanceof CostReservationExceededError) return {
+		class: error.constructor.name,
+		code: error.code,
+		message: redactSensitiveText(error.message, secrets)
+	};
+	if (error instanceof Error && error.name === "AbortError") return {
+		class: "ProviderTimeoutError",
+		message: "Provider request timed out."
+	};
+	if (error instanceof Error && /(?:assistant steps?|finding evidence|selected missing|selected unavailable|no readable spans|requires a trace store)/i.test(error.message)) return {
+		class: "BenchmarkEvidenceError",
+		message: redactSensitiveText(error.message, secrets)
+	};
+	if (error instanceof AgentEvalError) return {
+		class: error.constructor.name,
+		code: error.code,
+		message: redactSensitiveText(error.message, secrets)
+	};
+	if (error instanceof Error) return {
+		class: error.constructor.name || "Error",
+		message: redactSensitiveText(error.message, secrets)
+	};
+	return {
+		class: "Error",
+		message: "Benchmark analyst execution failed."
+	};
+}
+function redactSensitiveText(value, secrets) {
+	let redacted = value;
+	for (const secret of secrets) if (secret) redacted = redacted.replaceAll(secret, "[REDACTED]");
+	redacted = redacted.replace(/\bBearer\s+[^\s"',;]+/gi, "Bearer [REDACTED]").replace(/\b(api[_-]?key|access[_-]?token|refresh[_-]?token|password|secret)\b\s*[:=]\s*[^\s"',;]+/gi, "$1=[REDACTED]");
+	if (redacted.length <= 500) return redacted;
+	const head = redacted.slice(0, 180);
+	const marker = `...[${redacted.length - 460} chars omitted]...`;
+	return `${head}${marker}${redacted.slice(-(500 - head.length - marker.length))}`;
+}
+//#endregion
+//#region src/analyst/benchmark-public-types.ts
+function requiredString(value, field) {
+	const trimmed = value.trim();
+	if (!trimmed) throw new TypeError(`${field} must be a non-empty string`);
+	return trimmed;
+}
+function positiveSafeInteger(value, field) {
+	if (!Number.isSafeInteger(value) || value < 1) throw new RangeError(`${field} must be a positive safe integer`);
+	return value;
+}
+function safeInteger(value, field) {
+	if (!Number.isSafeInteger(value)) throw new RangeError(`${field} must be a safe integer`);
+	return value;
+}
+function isRecord(value) {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+//#endregion
+//#region src/analyst/benchmark-response-cache.ts
+const SHA256 = /^[a-f0-9]{64}$/;
+const CostReceiptInputSchema = z.object({
+	model: z.string().min(1),
+	inputTokens: z.number().int().nonnegative(),
+	outputTokens: z.number().int().nonnegative(),
+	reasoningTokens: z.number().int().nonnegative().optional(),
+	cachedTokens: z.number().int().nonnegative().optional(),
+	cacheWriteTokens: z.number().int().nonnegative().optional(),
+	customTokenPricing: z.object({
+		inputUsdPerMillion: z.number().nonnegative(),
+		cachedInputUsdPerMillion: z.number().nonnegative().optional(),
+		cacheWriteUsdPerMillion: z.number().nonnegative().optional(),
+		outputUsdPerMillion: z.number().nonnegative()
+	}).strict().optional(),
+	actualCostUsd: z.number().nonnegative().optional(),
+	estimatedCostUsd: z.number().nonnegative().optional(),
+	costUnknown: z.boolean().optional(),
+	usageUnknown: z.boolean().optional()
+}).strict();
+const BenchmarkErrorSchema = z.object({
+	class: z.string().min(1),
+	message: z.string(),
+	code: z.string().min(1).optional(),
+	status: z.number().int().min(100).max(599).optional()
+}).strict();
+const ResponseMetadataSchema = z.object({
+	providerModel: z.string().min(1),
+	providerDurationMs: z.number().nonnegative(),
+	finishReason: z.string().nullable(),
+	producedAt: z.string().datetime()
+}).strict();
+const CacheIdentityShape = {
+	kind: z.literal("agent-eval/public-benchmark-model-response"),
+	callId: z.string().min(1),
+	runIdentitySha256: z.string().regex(SHA256),
+	caseId: z.string().min(1),
+	repetition: z.number().int().nonnegative()
+};
+const SuccessCacheEntryWithoutDigestSchema = z.object({
+	...CacheIdentityShape,
+	status: z.literal("succeeded"),
+	response: z.json(),
+	metadata: ResponseMetadataSchema,
+	receipt: CostReceiptInputSchema
+}).strict();
+const FailureCacheEntryWithoutDigestSchema = z.object({
+	...CacheIdentityShape,
+	status: z.literal("failed"),
+	error: BenchmarkErrorSchema,
+	receipt: CostReceiptInputSchema
+}).strict();
+const CacheEntryWithoutDigestSchema = z.discriminatedUnion("status", [SuccessCacheEntryWithoutDigestSchema, FailureCacheEntryWithoutDigestSchema]);
+const CacheEntrySchema = z.discriminatedUnion("status", [SuccessCacheEntryWithoutDigestSchema.extend({ entrySha256: z.string().regex(SHA256) }), FailureCacheEntryWithoutDigestSchema.extend({ entrySha256: z.string().regex(SHA256) })]);
+function publicBenchmarkCallId(identity) {
+	assertCacheIdentity(identity);
+	return `analyst-benchmark-${hashCanonical({
+		runIdentitySha256: identity.runIdentitySha256,
+		caseId: identity.caseId,
+		repetition: identity.repetition
+	}).slice(7)}`;
+}
+function readPublicBenchmarkResponseCache(cacheDirectory, identity) {
+	const callId = publicBenchmarkCallId(identity);
+	const path = responseCachePath(cacheDirectory, callId);
+	if (!existsSync(path)) return void 0;
+	const metadata = lstatSync(path);
+	if (!metadata.isFile() || metadata.isSymbolicLink()) throw new ValidationError(`benchmark response cache must be a real file: ${path}`);
+	let parsed;
+	try {
+		parsed = JSON.parse(readFileSync(path, "utf8"));
+	} catch (error) {
+		throw new ValidationError(`benchmark response cache contains invalid JSON: ${path}`, { cause: error });
+	}
+	const entry = parseCacheEntry(parsed, path);
+	if (entry.callId !== callId || entry.runIdentitySha256 !== identity.runIdentitySha256 || entry.caseId !== identity.caseId || entry.repetition !== identity.repetition) throw new ValidationError(`benchmark response cache identity does not match: ${path}`);
+	return entry;
+}
+function writePublicBenchmarkResponseCache(cacheDirectory, entry) {
+	const expectedCallId = publicBenchmarkCallId(entry);
+	if (entry.callId !== expectedCallId) throw new ValidationError("benchmark response cache callId does not match its identity");
+	const validated = JSON.parse(JSON.stringify(CacheEntryWithoutDigestSchema.parse(entry)));
+	const complete = {
+		...validated,
+		entrySha256: hashCanonical(validated).slice(7)
+	};
+	const path = responseCachePath(cacheDirectory, complete.callId);
+	const content = `${canonicalString(complete)}\n`;
+	withLedgerFileLock(path, fileContext(), () => {
+		if (existsSync(path)) {
+			const existing = readPublicBenchmarkResponseCache(cacheDirectory, complete);
+			if (!existing || canonicalString(existing) !== canonicalString(complete)) throw new ValidationError(`benchmark response cache conflicts with existing file: ${path}`);
+			return;
+		}
+		writeLedgerFileAtomically(path, content, fileContext());
+	});
+	return complete;
+}
+function parseCacheEntry(value, path) {
+	let parsed;
+	try {
+		parsed = CacheEntrySchema.parse(value);
+	} catch (error) {
+		throw new ValidationError(`benchmark response cache has an invalid shape: ${path}`, { cause: error });
+	}
+	const { entrySha256, ...withoutDigest } = parsed;
+	if (entrySha256 !== hashCanonical(withoutDigest).slice(7)) throw new ValidationError(`benchmark response cache digest does not match: ${path}`);
+	return parsed;
+}
+function responseCachePath(cacheDirectory, callId) {
+	const directory = nodePath.resolve(cacheDirectory);
+	const path = nodePath.resolve(directory, `${hashCanonical(callId).slice(7)}.json`);
+	if (!isPathInsideDirectory(directory, path, nodePath)) throw new ValidationError("benchmark response cache path escapes its directory");
+	return path;
+}
+function isPathInsideDirectory(directory, candidate, pathOperations = nodePath) {
+	const relative = pathOperations.relative(directory, candidate);
+	return relative !== "" && relative !== ".." && !relative.startsWith(`..${pathOperations.sep}`) && !pathOperations.isAbsolute(relative);
+}
+function assertCacheIdentity(identity) {
+	if (!SHA256.test(identity.runIdentitySha256)) throw new ValidationError("benchmark response cache requires a SHA-256 run identity");
+	if (!identity.caseId.trim()) throw new ValidationError("benchmark response cache requires a case id");
+	if (!Number.isSafeInteger(identity.repetition) || identity.repetition < 0) throw new ValidationError("benchmark response cache repetition must be non-negative");
+}
+function fileContext() {
+	return {
+		subject: "benchmark response cache",
+		integrityError: (message, options) => new ValidationError(message, options)
+	};
+}
+//#endregion
+//#region src/analyst/benchmark-public-model.ts
+/** One-shot JSON baseline. This is not a recursive trace analyst. */
+function createPublicBenchmarkDirectRunner(dataset, config) {
+	if (config.instructionsOverride) throw new Error("the direct runner executes only the stock protocol; an instructions override requires the dspy-rlm runner");
+	const model = requiredString(config.model, "model");
+	const baseUrl = requiredString(config.baseUrl, "baseUrl");
+	const apiKey = requiredString(config.apiKey, "apiKey");
+	const maxOutputTokens = positiveSafeInteger(config.maxOutputTokens, "maxOutputTokens");
+	const timeoutMs = positiveSafeInteger(config.timeoutMs, "timeoutMs");
+	const costLedger = config.costLedger ?? new CostLedger();
+	const durability = config.durability ? {
+		runIdentitySha256: requiredString(config.durability.runIdentitySha256, "durability.runIdentitySha256"),
+		responseCacheDir: requiredString(config.durability.responseCacheDir, "durability.responseCacheDir")
+	} : void 0;
+	const actor = dataset === "agentrx" ? "agentrx-root-cause-localizer" : "codetracebench-step-localizer";
+	const outputAdapter = dataset === "agentrx" ? "agentrx-taxonomy-and-root-step" : "codetracebench-incorrect-block";
+	const llmOptions = {
+		baseUrl,
+		apiKey,
+		maximumAttempts: 1,
+		jsonSchemaTransport: "json-object",
+		jsonPayloadMode: "exact",
+		thinking: "disabled",
+		...config.fetchImpl ? { fetch: config.fetchImpl } : {}
+	};
+	return {
+		id: "direct",
+		async analyze(input, context) {
+			const trajectoryId = trajectoryIdFromCaseId$1(dataset, context.caseId);
+			const costTags = {
+				analystId: actor,
+				benchmarkCaseId: context.caseId,
+				benchmarkRepetition: String(context.repetition)
+			};
+			let rawPredictions = [];
+			let rejectedBlocks = [];
+			let modelFindings = [];
+			let providerModel = model;
 			let producedAt;
 			let modelMetadata = {
 				analysisMode: "direct-baseline",
@@ -3129,1176 +3679,1021 @@ function createPublicBenchmarkDirectRunner(dataset, config) {
 						responseSource: "provider",
 						report: response.report,
 						providerModel: paid.value.result.model,
-						providerDurationMs: paid.value.result.durationMs,
-						finishReason: paid.value.result.finishReason ?? null,
-						cost: costReceiptMetadata(paid.receipt)
-					};
-				}
-				const converted = await publicBenchmarkPredictionsToFindings({
-					dataset,
-					trajectoryId,
-					predictions: rawPredictions,
-					store: input.traceStore,
-					analystId: "direct",
-					providerModel,
-					producedAt: requiredString(producedAt ?? "", "finding producedAt"),
-					...context.signal ? { signal: context.signal } : {}
-				});
-				modelFindings = converted.findings;
-				if (converted.diagnostics) modelMetadata = {
-					...modelMetadata,
-					blockDiagnostics: {
-						...converted.diagnostics,
-						rejectedBlocks
-					}
-				};
-				if (dataset === "codetracebench") await validateCodeTraceFindingEvidence({
-					trajectoryId,
-					findings: modelFindings,
-					store: input.traceStore,
-					...context.signal ? { signal: context.signal } : {}
-				});
-				return {
-					findings: modelFindings,
-					usage: usageReceiptFromCostLedger(costLedger, {
-						channel: "analyst",
-						tags: costTags
-					}),
-					metadata: modelMetadata
-				};
-			} catch (error) {
-				if (context.signal?.aborted) throw error;
-				if (isPaidCallControlError$1(error)) throw error;
-				return {
-					findings: [],
-					usage: usageReceiptFromCostLedger(costLedger, {
-						channel: "analyst",
-						tags: costTags
-					}),
-					error: publicBenchmarkError(error, [apiKey]),
-					metadata: {
-						...modelMetadata,
-						rawPredictions,
-						acceptedFindings: modelFindings
-					}
-				};
-			}
-		}
-	};
-}
-function settleCachedResponse(costLedger, cached) {
-	const settled = costLedger.list().find((receipt) => receipt.callId === cached.callId);
-	const pending = costLedger.listPending?.().find((record) => record.callId === cached.callId);
-	if (settled && pending) throw new CostCallConflictError(`benchmark response '${cached.callId}' is both pending and settled`, { callId: cached.callId });
-	const receipt = pending ? costLedger.reconcile(cached.callId, cached.receipt, { ...cached.status === "failed" ? { failed: true } : {} }) : settled;
-	if (!receipt) throw new CostCallConflictError(`benchmark response cache '${cached.callId}' has no matching cost record`, { callId: cached.callId });
-	assertCacheReceiptMatches(cached, receipt);
-	return receipt;
-}
-function assertNoSettledResponseWithoutCache(costLedger, callId) {
-	if (!callId) return;
-	if (costLedger.list().some((receipt) => receipt.callId === callId)) throw new CostCallConflictError(`settled benchmark call '${callId}' has no durable response cache`, { callId });
-}
-function assertCacheReceiptMatches(cached, receipt) {
-	const expected = cached.receipt;
-	if (receipt.callId !== cached.callId || receipt.model !== expected.model || receipt.inputTokens !== expected.inputTokens || receipt.outputTokens !== expected.outputTokens || (receipt.reasoningTokens ?? 0) !== (expected.reasoningTokens ?? 0) || (receipt.cachedTokens ?? 0) !== (expected.cachedTokens ?? 0) || (receipt.cacheWriteTokens ?? 0) !== (expected.cacheWriteTokens ?? 0) || expected.actualCostUsd !== void 0 && receipt.actualCostUsd !== expected.actualCostUsd || expected.estimatedCostUsd !== void 0 && receipt.estimatedCostUsd !== expected.estimatedCostUsd || expected.costUnknown === true && !receipt.costUnknown || expected.usageUnknown === true && !receipt.usageUnknown || cached.status === "succeeded" && receipt.error !== void 0 || cached.status === "failed" && receipt.error === void 0) throw new CostCallConflictError(`benchmark response cache receipt does not match cost record '${cached.callId}'`, {
-		callId: cached.callId,
-		receipt
-	});
-}
-function isPaidCallControlError$1(error) {
-	return error instanceof CostAccountingIncompleteError || error instanceof CostCallConflictError || error instanceof CostCeilingReachedError || error instanceof CostLedgerPersistenceError || error instanceof CostReceiptCaptureError || error instanceof CostReservationExceededError;
-}
-function receiptForProviderFailure(error, completedResult, model) {
-	if (completedResult) return costReceiptFromLlm(completedResult);
-	if (error instanceof Error) {
-		const captured = costReceiptFromLlmError(error);
-		if (captured) return captured;
-	}
-	return {
-		model,
-		inputTokens: 0,
-		outputTokens: 0,
-		costUnknown: true,
-		usageUnknown: true
-	};
-}
-function costReceiptMetadata(receipt) {
-	if (receipt.actualCostUsd !== void 0) return {
-		source: "provider",
-		actualCostUsd: receipt.actualCostUsd
-	};
-	if (receipt.estimatedCostUsd !== void 0) return {
-		source: "external-estimate",
-		estimatedCostUsd: receipt.estimatedCostUsd
-	};
-	if (receipt.pricing) return {
-		source: "agent-eval-model-pricing",
-		estimatedCostUsd: receipt.costUsd,
-		ratesPerThousandTokens: receipt.pricing
-	};
-	return {
-		source: "unknown",
-		estimatedCostUsd: null
-	};
-}
-const ModelSeveritySchema = z.enum([
-	"critical",
-	"high",
-	"medium",
-	"low",
-	"info"
-]);
-const AgentRxPredictionSchema = z.object({
-	step: z.number().int().positive(),
-	severity: ModelSeveritySchema,
-	claim: z.string().min(1),
-	confidence: z.number().min(0).max(1),
-	rationale: z.string().min(1).optional(),
-	recommended_action: z.string().min(1).optional()
-}).strict();
-const CodeTraceBlockPredictionSchema = z.object({
-	first_step: z.number().int().positive(),
-	last_step: z.number().int().positive(),
-	consequence_step: z.number().int().positive(),
-	escape_status: z.enum(["escaped", "unescaped"]),
-	severity: ModelSeveritySchema,
-	claim: z.string().min(1),
-	confidence: z.number().min(0).max(1),
-	rationale: z.string().min(1).optional(),
-	recommended_action: z.string().min(1).optional()
-}).strict().superRefine((block, ctx) => {
-	if (block.last_step < block.first_step) {
-		ctx.addIssue({
-			code: z.ZodIssueCode.custom,
-			message: `failure block last_step ${block.last_step} precedes first_step ${block.first_step}`
-		});
-		return;
-	}
-	const length = block.last_step - block.first_step + 1;
-	if (length > 12) ctx.addIssue({
-		code: z.ZodIssueCode.custom,
-		message: `failure block spans ${length} steps; the maximum is 12`
-	});
-	if (block.consequence_step < block.first_step) ctx.addIssue({
-		code: z.ZodIssueCode.custom,
-		message: `failure block consequence_step ${block.consequence_step} precedes first_step ${block.first_step}`
-	});
-});
-const AgentRxCategorySchema = z.enum([
-	"instruction-plan-adherence-failure",
-	"invention-of-new-information",
-	"invalid-invocation",
-	"misinterpretation-of-tool-output-handoff-failure",
-	"intent-plan-misalignment",
-	"underspecified-user-intent",
-	"intent-not-supported",
-	"guardrails-triggered",
-	"system-failure",
-	"inconclusive"
-]);
-const CodeTraceModelResponseEnvelopeSchema = z.object({
-	report: z.string().min(1).max(4e3),
-	findings: z.array(z.unknown()).max(16)
-}).strict();
-const AgentRxModelResponseSchema = z.object({
-	report: z.string().min(1).max(4e3),
-	findings: z.array(AgentRxPredictionSchema.extend({ category: AgentRxCategorySchema }).strict()).max(1)
-}).strict();
-function parsePublicBenchmarkModelResponse(dataset, value) {
-	if (dataset === "agentrx") return {
-		...AgentRxModelResponseSchema.parse(value),
-		rejectedBlocks: []
-	};
-	const envelope = CodeTraceModelResponseEnvelopeSchema.parse(value);
-	const findings = [];
-	const rejectedBlocks = [];
-	for (const [index, block] of envelope.findings.entries()) {
-		const parsed = CodeTraceBlockPredictionSchema.safeParse(block);
-		if (parsed.success) {
-			findings.push(parsed.data);
-			continue;
-		}
-		rejectedBlocks.push(`block ${index}: ${parsed.error.issues.map((issue) => `${issue.path.join(".") || "<root>"} ${issue.message}`).join("; ")}`);
-	}
-	if (findings.length === 0 && envelope.findings.length > 0) throw new ValidationError(`every reported failure block was malformed: ${rejectedBlocks.join(" | ")}`);
-	return {
-		report: envelope.report,
-		findings,
-		rejectedBlocks
-	};
-}
-async function publicBenchmarkPredictionsToFindings(options) {
-	if (options.predictions.length === 0 && options.dataset === "agentrx") return {
-		findings: [],
-		diagnostics: void 0
-	};
-	if (options.dataset === "agentrx") {
-		const prediction = options.predictions[0];
-		if (!("step" in prediction)) throw new Error("AgentRx model output must name a single root-cause step");
-		if (!prediction.category) throw new Error("AgentRx model output is missing its failure category");
-		const evidenceByStep = await resolveAssistantStepEvidence({
-			trajectoryId: options.trajectoryId,
-			steps: [prediction.step],
-			store: options.store,
-			...options.signal ? { signal: options.signal } : {}
-		});
-		const [finding] = agentRxPredictionsToFindings(options.trajectoryId, [{
-			failure_case: prediction.category,
-			step_number: prediction.step,
-			description: prediction.rationale ?? prediction.claim
-		}], {
-			analystId: options.analystId,
-			producedAt: options.producedAt,
-			confidence: prediction.confidence
-		});
-		if (!finding) throw new Error("AgentRx output adapter produced no root-cause finding");
-		return {
-			findings: [{
-				...finding,
-				evidence_refs: [evidenceByStep.get(prediction.step)],
-				metadata: {
-					...finding.metadata,
-					model: options.providerModel
+						providerDurationMs: paid.value.result.durationMs,
+						finishReason: paid.value.result.finishReason ?? null,
+						cost: costReceiptMetadata(paid.receipt)
+					};
 				}
-			}],
-			diagnostics: void 0
-		};
-	}
-	const blocks = options.predictions.map((prediction) => {
-		if (!("first_step" in prediction)) throw new Error("CodeTraceBench model output must report first_step/last_step failure blocks");
-		return {
-			firstStep: prediction.first_step,
-			lastStep: prediction.last_step,
-			consequenceStep: prediction.consequence_step,
-			escapeStatus: prediction.escape_status,
-			severity: prediction.severity,
-			claim: prediction.claim,
-			confidence: prediction.confidence,
-			...prediction.rationale === void 0 ? {} : { rationale: prediction.rationale },
-			...prediction.recommended_action === void 0 ? {} : { recommendedAction: prediction.recommended_action },
-			metadata: {
-				analysis_mode: "direct-baseline",
-				model: options.providerModel
-			}
-		};
-	});
-	return expandCodeTraceFailureBlocks({
-		trajectoryId: options.trajectoryId,
-		blocks,
-		store: options.store,
-		analystId: options.analystId,
-		producedAt: options.producedAt,
-		...options.signal ? { signal: options.signal } : {}
-	});
-}
-async function prepareSingleTraceContext(store, context) {
-	const storeContext = context.signal ? { signal: context.signal } : void 0;
-	const overview = await store.getOverview(void 0, storeContext);
-	if (overview.total_traces !== 1 || overview.sample_trace_ids.length !== 1) throw new Error(`public model benchmark requires exactly one trace, received ${overview.total_traces}`);
-	const traceId = overview.sample_trace_ids[0];
-	for (const perAttributeByteCap of TRACE_PROJECTION_ATTRIBUTE_BYTE_CAPS) {
-		const viewed = await store.viewTrace({
-			trace_id: traceId,
-			per_attribute_byte_cap: perAttributeByteCap
-		}, storeContext);
-		if (!viewed.spans) continue;
-		return JSON.stringify({
-			trace_id: traceId,
-			per_attribute_byte_cap: perAttributeByteCap,
-			spans: viewed.spans
-		});
-	}
-}
-function trajectoryIdFromCaseId$1(dataset, caseId) {
-	const prefix = dataset === "agentrx" ? "agentrx:" : "codetrace:";
-	if (!caseId.startsWith(prefix) || caseId.length === prefix.length) throw new Error(`unexpected ${dataset} benchmark case id '${caseId}'`);
-	return caseId.slice(prefix.length);
-}
-//#endregion
-//#region src/analyst/benchmark-public-rlm.ts
-/** Public benchmark candidate that runs the actual recursive trace analyst. */
-function createPublicBenchmarkRlmRunner(dataset, config) {
-	const costLedger = config.costLedger ?? new CostLedger();
-	const limits = {
-		maxIterations: config.dspyRlm?.maxIterations ?? 14,
-		maxLlmCalls: config.dspyRlm?.maxLlmCalls ?? 8,
-		maxToolCalls: config.dspyRlm?.maxToolCalls ?? 80,
-		maxOutputChars: config.dspyRlm?.maxOutputChars ?? 8e3
-	};
-	const pricing = config.pricing ?? pricingForModel(config.model);
-	const engine = createDspyRlmTraceEngine({
-		baseUrl: config.baseUrl,
-		apiKey: config.apiKey,
-		model: config.model,
-		maxOutputTokens: config.maxOutputTokens,
-		timeoutMs: config.timeoutMs,
-		maxCostUsd: config.maxCostUsdPerAnalysis ?? 1,
-		pricing,
-		...config.dspyRlm?.runner ? { runner: config.dspyRlm.runner } : {}
-	});
-	const definition = publicBenchmarkDefinition(dataset, limits);
-	return {
-		id: "dspy-rlm",
-		async analyze(input, context) {
-			const trajectoryId = trajectoryIdFromCaseId(dataset, context.caseId);
-			const tags = {
-				benchmarkCaseId: context.caseId,
-				benchmarkRepetition: String(context.repetition)
-			};
-			let usage;
-			let rawFindings = [];
-			try {
-				if (!input.traceStore) throw new Error(`${dataset} DSPy RLM runner requires a trace store`);
-				const completed = await runTraceAnalyst({
-					definition,
-					engine,
+				const converted = await publicBenchmarkPredictionsToFindings({
+					dataset,
+					trajectoryId,
+					predictions: rawPredictions,
 					store: input.traceStore,
-					context: {
-						runId: context.caseId,
-						correlationId: `${context.caseId}:${context.repetition}`,
-						costLedger,
-						costPhase: "analyst.public-benchmark.dspy-rlm",
-						tags,
-						recordUsage: (receipt) => {
-							usage = receipt;
-						},
-						signal: context.signal
-					}
+					analystId: "direct",
+					providerModel,
+					producedAt: requiredString(producedAt ?? "", "finding producedAt"),
+					...context.signal ? { signal: context.signal } : {}
 				});
-				const producedAt = (/* @__PURE__ */ new Date()).toISOString();
-				rawFindings = completed.findings.map((finding) => makeFinding({
-					analyst_id: "dspy-rlm",
-					area: dataset === "agentrx" ? "root-cause" : "incorrect",
-					subject: finding.subject,
-					claim: finding.claim,
-					rationale: finding.rationale,
-					severity: finding.severity,
-					confidence: finding.confidence,
-					evidence_refs: evidenceRefsFromRawFinding(finding),
-					recommended_action: finding.recommended_action,
-					metadata: {
-						analysis_mode: "recursive",
-						engine: "dspy-rlm",
-						model: config.model
-					},
-					produced_at: producedAt
-				}));
-				const adapted = await adaptPublicBenchmarkFindings({
-					dataset,
+				modelFindings = converted.findings;
+				if (converted.diagnostics) modelMetadata = {
+					...modelMetadata,
+					blockDiagnostics: {
+						...converted.diagnostics,
+						rejectedBlocks
+					}
+				};
+				if (dataset === "codetracebench") await validateCodeTraceFindingEvidence({
 					trajectoryId,
-					findings: rawFindings,
-					analystId: "dspy-rlm",
+					findings: modelFindings,
 					store: input.traceStore,
 					...context.signal ? { signal: context.signal } : {}
 				});
 				return {
-					findings: adapted.findings,
-					usage,
-					metadata: {
-						analysisMode: "recursive",
-						engine: "dspy-rlm",
-						protocolSha256: publicBenchmarkProtocolSha256(dataset),
-						...adapted.diagnostics ? { blockDiagnostics: adapted.diagnostics } : {},
-						answer: completed.answer,
-						trajectory: completed.trajectory,
-						modelCalls: completed.modelCalls,
-						toolCalls: completed.toolCalls,
-						runtime: completed.runtime
-					}
+					findings: modelFindings,
+					usage: usageReceiptFromCostLedger(costLedger, {
+						channel: "analyst",
+						tags: costTags
+					}),
+					metadata: modelMetadata
 				};
 			} catch (error) {
 				if (context.signal?.aborted) throw error;
-				if (isPaidCallControlError(error)) throw error;
+				if (isPaidCallControlError$1(error)) throw error;
 				return {
 					findings: [],
-					usage,
-					error: publicBenchmarkError(error, [config.apiKey]),
+					usage: usageReceiptFromCostLedger(costLedger, {
+						channel: "analyst",
+						tags: costTags
+					}),
+					error: publicBenchmarkError(error, [apiKey]),
 					metadata: {
-						analysisMode: "recursive",
-						engine: "dspy-rlm",
-						rawFindings
+						...modelMetadata,
+						rawPredictions,
+						acceptedFindings: modelFindings
 					}
 				};
 			}
 		}
 	};
 }
-function publicBenchmarkDefinition(dataset, limits) {
-	return {
-		id: dataset === "agentrx" ? "agentrx-dspy-rlm" : "codetracebench-dspy-rlm",
-		description: dataset === "agentrx" ? "Localizes the first unrecoverable root-cause step." : "Localizes every incorrect state-changing assistant step.",
-		area: dataset === "agentrx" ? "root-cause" : "incorrect",
-		version: "1.0.0",
-		question: dataset === "agentrx" ? "What is the first unrecoverable root cause in this failed trajectory?" : "Which assistant steps are incorrect under the CodeTraceBench definition?",
-		instructions: publicBenchmarkRlmInstructions(dataset),
-		toolGroup: "singleTrace",
-		limits
-	};
+function settleCachedResponse(costLedger, cached) {
+	const settled = costLedger.list().find((receipt) => receipt.callId === cached.callId);
+	const pending = costLedger.listPending?.().find((record) => record.callId === cached.callId);
+	if (settled && pending) throw new CostCallConflictError(`benchmark response '${cached.callId}' is both pending and settled`, { callId: cached.callId });
+	const receipt = pending ? costLedger.reconcile(cached.callId, cached.receipt, { ...cached.status === "failed" ? { failed: true } : {} }) : settled;
+	if (!receipt) throw new CostCallConflictError(`benchmark response cache '${cached.callId}' has no matching cost record`, { callId: cached.callId });
+	assertCacheReceiptMatches(cached, receipt);
+	return receipt;
 }
-function pricingForModel(model) {
-	const pricing = resolveModelPricing(model);
-	if (!pricing) throw new Error(`no pricing is configured for '${model}'; provide PublicAnalystBenchmarkModelConfig.pricing`);
-	return {
-		inputUsdPerMillion: pricing.input * 1e3,
-		outputUsdPerMillion: pricing.output * 1e3
-	};
+function assertNoSettledResponseWithoutCache(costLedger, callId) {
+	if (!callId) return;
+	if (costLedger.list().some((receipt) => receipt.callId === callId)) throw new CostCallConflictError(`settled benchmark call '${callId}' has no durable response cache`, { callId });
 }
-function trajectoryIdFromCaseId(dataset, caseId) {
-	const prefix = dataset === "agentrx" ? "agentrx:" : "codetrace:";
-	if (!caseId.startsWith(prefix) || caseId.length === prefix.length) throw new Error(`unexpected ${dataset} benchmark case id '${caseId}'`);
-	return caseId.slice(prefix.length);
+function assertCacheReceiptMatches(cached, receipt) {
+	const expected = cached.receipt;
+	if (receipt.callId !== cached.callId || receipt.model !== expected.model || receipt.inputTokens !== expected.inputTokens || receipt.outputTokens !== expected.outputTokens || (receipt.reasoningTokens ?? 0) !== (expected.reasoningTokens ?? 0) || (receipt.cachedTokens ?? 0) !== (expected.cachedTokens ?? 0) || (receipt.cacheWriteTokens ?? 0) !== (expected.cacheWriteTokens ?? 0) || expected.actualCostUsd !== void 0 && receipt.actualCostUsd !== expected.actualCostUsd || expected.estimatedCostUsd !== void 0 && receipt.estimatedCostUsd !== expected.estimatedCostUsd || expected.costUnknown === true && !receipt.costUnknown || expected.usageUnknown === true && !receipt.usageUnknown || cached.status === "succeeded" && receipt.error !== void 0 || cached.status === "failed" && receipt.error === void 0) throw new CostCallConflictError(`benchmark response cache receipt does not match cost record '${cached.callId}'`, {
+		callId: cached.callId,
+		receipt
+	});
 }
-function isPaidCallControlError(error) {
+function isPaidCallControlError$1(error) {
 	return error instanceof CostAccountingIncompleteError || error instanceof CostCallConflictError || error instanceof CostCeilingReachedError || error instanceof CostLedgerPersistenceError || error instanceof CostReceiptCaptureError || error instanceof CostReservationExceededError;
 }
-//#endregion
-//#region src/analyst/benchmark-command-persistence.ts
-const ANALYST_BENCHMARK_INITIALIZATION_COMPLETE_FILE = "initialization-complete.json";
-async function openOutputDirectory(outDir, resume) {
-	const directory = resolve(outDir);
-	if (resume) {
-		let outputStat;
-		try {
-			outputStat = await lstat(directory);
-		} catch (error) {
-			if (isNodeError(error, "ENOENT")) throw new Error(`cannot resume missing benchmark output directory: ${directory}`);
-			throw error;
-		}
-		if (!outputStat.isDirectory() || outputStat.isSymbolicLink()) throw new Error(`benchmark output must be a real directory: ${directory}`);
-	} else {
-		await mkdir(dirname(directory), { recursive: true });
-		try {
-			await mkdir(directory);
-		} catch (error) {
-			if (isNodeError(error, "EEXIST")) throw new Error(`refusing to use existing benchmark output directory: ${directory}`);
-			throw error;
-		}
-		await syncDirectory(dirname(directory));
-	}
-	return {
-		directory,
-		initializationComplete: resolve(directory, ANALYST_BENCHMARK_INITIALIZATION_COMPLETE_FILE),
-		manifest: resolve(directory, ANALYST_BENCHMARK_MANIFEST_FILE),
-		observations: resolve(directory, ANALYST_BENCHMARK_OBSERVATIONS_FILE),
-		costLedger: resolve(directory, ANALYST_BENCHMARK_COST_LEDGER_FILE),
-		modelResponses: resolve(directory, "model-responses"),
-		localReceipt: resolve(directory, ANALYST_BENCHMARK_LOCAL_RECEIPT_FILE),
-		result: resolve(directory, "result.json"),
-		report: resolve(directory, "report.md")
-	};
-}
-async function prepareOutputLockPath(outDir) {
-	const directory = resolve(outDir);
-	await mkdir(dirname(directory), { recursive: true });
-	return `${directory}.lock`;
-}
-function createRunIdentity(config, prepared) {
-	const caseDefinitions = prepared.cases.map((testCase) => ({
-		id: testCase.id,
-		clusterId: testCase.clusterId,
-		labelState: testCase.labelState,
-		expectedIssues: testCase.expectedIssues,
-		labeledEvidence: testCase.labeledEvidence ?? [],
-		tags: testCase.tags ?? [],
-		metadata: testCase.metadata ?? {}
-	}));
+function receiptForProviderFailure(error, completedResult, model) {
+	if (completedResult) return costReceiptFromLlm(completedResult);
+	if (error instanceof Error) {
+		const captured = costReceiptFromLlmError(error);
+		if (captured) return captured;
+	}
 	return {
-		config: {
-			dataset: config.dataset,
-			datasetRevision: config.revision,
-			datasetSplit: config.split,
-			model: {
-				id: config.model.model,
-				maxOutputTokens: config.model.maxOutputTokens,
-				timeoutMs: config.model.timeoutMs
-			},
-			limit: config.limit,
-			seed: config.seed,
-			concurrency: config.concurrency,
-			repetitions: config.repetitions,
-			maxCostUsd: config.maxCostUsd,
-			maxArtifactBytes: config.maxArtifactBytes,
-			analystProtocolSha256: publicBenchmarkProtocolSha256(config.dataset),
-			implementationSha256: ANALYST_BENCHMARK_IMPLEMENTATION_SHA256,
-			dependencyLockSha256: ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256,
-			runnerIds: ["empty", config.analyst]
-		},
-		inputs: {
-			labelsSha256: prepared.labelsSha256,
-			sourceRowCount: prepared.sourceRowCount,
-			selectedCaseIds: [...prepared.selectedCaseIds],
-			traceFiles: prepared.traceFiles.map((traceFile) => ({ ...traceFile })),
-			verificationArtifactsSha256: digestCanonical(prepared.verificationArtifacts),
-			caseDefinitionsSha256: digestCanonical(caseDefinitions)
-		}
+		model,
+		inputTokens: 0,
+		outputTokens: 0,
+		costUnknown: true,
+		usageUnknown: true
 	};
 }
-function createLocalRunReceipt(config, paths) {
+function costReceiptMetadata(receipt) {
+	if (receipt.actualCostUsd !== void 0) return {
+		source: "provider",
+		actualCostUsd: receipt.actualCostUsd
+	};
+	if (receipt.estimatedCostUsd !== void 0) return {
+		source: "external-estimate",
+		estimatedCostUsd: receipt.estimatedCostUsd
+	};
+	if (receipt.pricing) return {
+		source: "agent-eval-model-pricing",
+		estimatedCostUsd: receipt.costUsd,
+		ratesPerThousandTokens: receipt.pricing
+	};
 	return {
-		kind: "agent-eval/analyst-benchmark-local-run",
-		local: {
-			labelsPath: resolve(config.labelsPath),
-			traceDir: resolve(config.traceDir),
-			...config.artifactDir ? { artifactDir: resolve(config.artifactDir) } : {},
-			outputDir: paths.directory,
-			baseUrl: config.model.baseUrl,
-			apiKeyEnvironment: config.apiKeyEnv
-		},
-		command: config.command,
-		environment: {
-			node: process.version,
-			platform: platform(),
-			arch: arch()
-		},
-		files: {
-			manifest: paths.manifest,
-			observations: paths.observations,
-			costLedger: paths.costLedger,
-			modelResponses: paths.modelResponses,
-			result: paths.result,
-			report: paths.report
-		}
+		source: "unknown",
+		estimatedCostUsd: null
 	};
 }
-async function initializeRunFiles(paths, identity, identitySha256, localIdentitySha256, localReceiptInput) {
-	const existingManifest = await readOptionalRegularFile(paths.manifest, "benchmark run manifest");
-	const manifest = existingManifest ? await readAndValidateManifestContent(paths.manifest, existingManifest, identity, identitySha256, localIdentitySha256) : {
-		kind: "agent-eval/analyst-benchmark-run",
-		createdAt: (/* @__PURE__ */ new Date()).toISOString(),
-		identitySha256,
-		localIdentitySha256,
-		identity
+const ModelSeveritySchema = z.enum([
+	"critical",
+	"high",
+	"medium",
+	"low",
+	"info"
+]);
+const AgentRxPredictionSchema = z.object({
+	step: z.number().int().positive(),
+	severity: ModelSeveritySchema,
+	claim: z.string().min(1),
+	confidence: z.number().min(0).max(1),
+	rationale: z.string().min(1).optional(),
+	recommended_action: z.string().min(1).optional()
+}).strict();
+const CodeTraceBlockPredictionSchema = z.object({
+	first_step: z.number().int().positive(),
+	last_step: z.number().int().positive(),
+	consequence_step: z.number().int().positive(),
+	escape_status: z.enum(["escaped", "unescaped"]),
+	severity: ModelSeveritySchema,
+	claim: z.string().min(1),
+	confidence: z.number().min(0).max(1),
+	rationale: z.string().min(1).optional(),
+	recommended_action: z.string().min(1).optional()
+}).strict().superRefine((block, ctx) => {
+	if (block.last_step < block.first_step) {
+		ctx.addIssue({
+			code: z.ZodIssueCode.custom,
+			message: `failure block last_step ${block.last_step} precedes first_step ${block.first_step}`
+		});
+		return;
+	}
+	const length = block.last_step - block.first_step + 1;
+	if (length > 12) ctx.addIssue({
+		code: z.ZodIssueCode.custom,
+		message: `failure block spans ${length} steps; the maximum is 12`
+	});
+	if (block.consequence_step < block.first_step) ctx.addIssue({
+		code: z.ZodIssueCode.custom,
+		message: `failure block consequence_step ${block.consequence_step} precedes first_step ${block.first_step}`
+	});
+});
+const AgentRxCategorySchema = z.enum([
+	"instruction-plan-adherence-failure",
+	"invention-of-new-information",
+	"invalid-invocation",
+	"misinterpretation-of-tool-output-handoff-failure",
+	"intent-plan-misalignment",
+	"underspecified-user-intent",
+	"intent-not-supported",
+	"guardrails-triggered",
+	"system-failure",
+	"inconclusive"
+]);
+const CodeTraceModelResponseEnvelopeSchema = z.object({
+	report: z.string().min(1).max(4e3),
+	findings: z.array(z.unknown()).max(16)
+}).strict();
+const AgentRxModelResponseSchema = z.object({
+	report: z.string().min(1).max(4e3),
+	findings: z.array(AgentRxPredictionSchema.extend({ category: AgentRxCategorySchema }).strict()).max(1)
+}).strict();
+function parsePublicBenchmarkModelResponse(dataset, value) {
+	if (dataset === "agentrx") return {
+		...AgentRxModelResponseSchema.parse(value),
+		rejectedBlocks: []
 	};
-	const localReceipt = {
-		...localReceiptInput,
-		runIdentitySha256: identitySha256,
-		localIdentitySha256
+	const envelope = CodeTraceModelResponseEnvelopeSchema.parse(value);
+	const findings = [];
+	const rejectedBlocks = [];
+	for (const [index, block] of envelope.findings.entries()) {
+		const parsed = CodeTraceBlockPredictionSchema.safeParse(block);
+		if (parsed.success) {
+			findings.push(parsed.data);
+			continue;
+		}
+		rejectedBlocks.push(`block ${index}: ${parsed.error.issues.map((issue) => `${issue.path.join(".") || "<root>"} ${issue.message}`).join("; ")}`);
+	}
+	if (findings.length === 0 && envelope.findings.length > 0) throw new ValidationError(`every reported failure block was malformed: ${rejectedBlocks.join(" | ")}`);
+	return {
+		report: envelope.report,
+		findings,
+		rejectedBlocks
 	};
-	const manifestContent = `${JSON.stringify(manifest, null, 2)}\n`;
-	const localReceiptContent = `${JSON.stringify(localReceipt, null, 2)}\n`;
-	const initializationCompleteContent = renderInitializationComplete(manifest);
-	await assertAbsentOrExact(paths.observations, "", "benchmark observation log");
-	await assertAbsentOrExact(paths.localReceipt, localReceiptContent, "benchmark local run receipt");
-	await assertAbsentOrExact(paths.manifest, manifestContent, "benchmark run manifest");
-	for (const path of [
-		paths.costLedger,
-		paths.modelResponses,
-		paths.result,
-		paths.report
-	]) if (await regularFileExists(path)) throw new Error(`benchmark initialization marker is missing but later run artifact exists: ${path}`);
-	if (await regularFileExists(paths.initializationComplete)) throw new Error(`benchmark initialization marker already exists during partial initialization: ${paths.initializationComplete}`);
-	await writeExclusiveOrVerify(paths.observations, "");
-	await writeExclusiveOrVerify(paths.localReceipt, localReceiptContent);
-	await writeExclusiveOrVerify(paths.manifest, manifestContent);
-	await writeExclusiveOrVerify(paths.initializationComplete, initializationCompleteContent);
-	return manifest;
 }
-async function readAndValidateResumeFiles(paths, currentIdentity, currentIdentitySha256, currentLocalIdentitySha256, localReceiptInput) {
-	if (!await regularFileExists(paths.initializationComplete)) return initializeRunFiles(paths, currentIdentity, currentIdentitySha256, currentLocalIdentitySha256, localReceiptInput);
-	const manifest = await readAndValidateManifest(paths.manifest, currentIdentity, currentIdentitySha256, currentLocalIdentitySha256);
-	const localReceiptContent = await readRegularFile(paths.localReceipt, "benchmark local run receipt");
-	const value = parseJson(localReceiptContent, paths.localReceipt);
-	if (!isRecord$1(value)) throw new TypeError(`benchmark local run receipt must be an object: ${paths.localReceipt}`);
-	assertExactKeys(value, [
-		"kind",
-		"runIdentitySha256",
-		"localIdentitySha256",
-		"local",
-		"command",
-		"environment",
-		"files"
-	], "benchmark local run receipt");
-	if (value.kind !== "agent-eval/analyst-benchmark-local-run" || value.runIdentitySha256 !== currentIdentitySha256 || value.localIdentitySha256 !== currentLocalIdentitySha256 || !isRecord$1(value.local)) throw new Error("benchmark local run receipt does not match the requested resume");
-	const expectedLocalReceipt = {
-		...localReceiptInput,
-		runIdentitySha256: currentIdentitySha256,
-		localIdentitySha256: currentLocalIdentitySha256
-	};
-	if (digestCanonical(value.local) !== currentLocalIdentitySha256 || canonicalJson(value.local) !== canonicalJson(localReceiptInput.local)) throw new Error("benchmark local paths or endpoint do not match the requested resume");
-	if (localReceiptContent !== `${JSON.stringify(expectedLocalReceipt, null, 2)}\n`) throw new Error(`benchmark local run receipt does not exactly match: ${paths.localReceipt}`);
-	if (await readRegularFile(paths.initializationComplete, "benchmark initialization marker") !== renderInitializationComplete(manifest)) throw new Error(`benchmark initialization marker does not match the run manifest: ${paths.initializationComplete}`);
-	return manifest;
+async function publicBenchmarkPredictionsToFindings(options) {
+	if (options.predictions.length === 0 && options.dataset === "agentrx") return {
+		findings: [],
+		diagnostics: void 0
+	};
+	if (options.dataset === "agentrx") {
+		const prediction = options.predictions[0];
+		if (!("step" in prediction)) throw new Error("AgentRx model output must name a single root-cause step");
+		if (!prediction.category) throw new Error("AgentRx model output is missing its failure category");
+		const evidenceByStep = await resolveAssistantStepEvidence({
+			trajectoryId: options.trajectoryId,
+			steps: [prediction.step],
+			store: options.store,
+			...options.signal ? { signal: options.signal } : {}
+		});
+		const [finding] = agentRxPredictionsToFindings(options.trajectoryId, [{
+			failure_case: prediction.category,
+			step_number: prediction.step,
+			description: prediction.rationale ?? prediction.claim
+		}], {
+			analystId: options.analystId,
+			producedAt: options.producedAt,
+			confidence: prediction.confidence
+		});
+		if (!finding) throw new Error("AgentRx output adapter produced no root-cause finding");
+		return {
+			findings: [{
+				...finding,
+				evidence_refs: [evidenceByStep.get(prediction.step)],
+				metadata: {
+					...finding.metadata,
+					model: options.providerModel
+				}
+			}],
+			diagnostics: void 0
+		};
+	}
+	const blocks = options.predictions.map((prediction) => {
+		if (!("first_step" in prediction)) throw new Error("CodeTraceBench model output must report first_step/last_step failure blocks");
+		return {
+			firstStep: prediction.first_step,
+			lastStep: prediction.last_step,
+			consequenceStep: prediction.consequence_step,
+			escapeStatus: prediction.escape_status,
+			severity: prediction.severity,
+			claim: prediction.claim,
+			confidence: prediction.confidence,
+			...prediction.rationale === void 0 ? {} : { rationale: prediction.rationale },
+			...prediction.recommended_action === void 0 ? {} : { recommendedAction: prediction.recommended_action },
+			metadata: {
+				analysis_mode: "direct-baseline",
+				model: options.providerModel
+			}
+		};
+	});
+	return expandCodeTraceFailureBlocks({
+		trajectoryId: options.trajectoryId,
+		blocks,
+		store: options.store,
+		analystId: options.analystId,
+		producedAt: options.producedAt,
+		...options.signal ? { signal: options.signal } : {}
+	});
+}
+async function prepareSingleTraceContext(store, context) {
+	const storeContext = context.signal ? { signal: context.signal } : void 0;
+	const overview = await store.getOverview(void 0, storeContext);
+	if (overview.total_traces !== 1 || overview.sample_trace_ids.length !== 1) throw new Error(`public model benchmark requires exactly one trace, received ${overview.total_traces}`);
+	const traceId = overview.sample_trace_ids[0];
+	for (const perAttributeByteCap of TRACE_PROJECTION_ATTRIBUTE_BYTE_CAPS) {
+		const viewed = await store.viewTrace({
+			trace_id: traceId,
+			per_attribute_byte_cap: perAttributeByteCap
+		}, storeContext);
+		if (!viewed.spans) continue;
+		return JSON.stringify({
+			trace_id: traceId,
+			per_attribute_byte_cap: perAttributeByteCap,
+			spans: viewed.spans
+		});
+	}
 }
-async function readAndValidateManifest(path, currentIdentity, currentIdentitySha256, currentLocalIdentitySha256) {
-	const content = await readRegularFile(path, "benchmark run manifest");
-	const manifest = await readAndValidateManifestContent(path, content, currentIdentity, currentIdentitySha256, currentLocalIdentitySha256);
-	if (content !== `${JSON.stringify(manifest, null, 2)}\n`) throw new Error(`benchmark run manifest does not exactly match: ${path}`);
-	return manifest;
+function trajectoryIdFromCaseId$1(dataset, caseId) {
+	const prefix = dataset === "agentrx" ? "agentrx:" : "codetrace:";
+	if (!caseId.startsWith(prefix) || caseId.length === prefix.length) throw new Error(`unexpected ${dataset} benchmark case id '${caseId}'`);
+	return caseId.slice(prefix.length);
 }
-async function readAndValidateManifestContent(path, content, currentIdentity, currentIdentitySha256, currentLocalIdentitySha256) {
-	const value = parseJson(content, path);
-	if (!isRecord$1(value)) throw new TypeError(`benchmark run manifest must be an object: ${path}`);
-	assertExactKeys(value, [
-		"kind",
-		"createdAt",
-		"identitySha256",
-		"localIdentitySha256",
-		"identity"
-	], "benchmark run manifest");
-	if (value.kind !== "agent-eval/analyst-benchmark-run") throw new TypeError(`unsupported benchmark run manifest: ${path}`);
-	if (typeof value.createdAt !== "string" || !Number.isFinite(Date.parse(value.createdAt))) throw new TypeError(`benchmark run manifest has an invalid createdAt: ${path}`);
-	if (!isSha256(value.identitySha256) || !isSha256(value.localIdentitySha256) || !isRecord$1(value.identity)) throw new TypeError(`benchmark run manifest has an invalid identity: ${path}`);
-	if (digestCanonical(value.identity) !== value.identitySha256) throw new Error(`benchmark run manifest identity digest does not match its contents: ${path}`);
-	if (currentIdentitySha256 !== value.identitySha256 || currentLocalIdentitySha256 !== value.localIdentitySha256 || canonicalJson(currentIdentity) !== canonicalJson(value.identity)) throw new Error(`benchmark resume configuration or inputs do not match ${ANALYST_BENCHMARK_MANIFEST_FILE}`);
+//#endregion
+//#region src/analyst/benchmark-public-consensus.ts
+/**
+* Step-level majority vote across independent analyst samples.
+*
+* Each sample's accepted, evidence-resolved steps count as one vote per step.
+* Steps present in at least ceil(k/2) samples survive; surviving steps are
+* reassembled into contiguous consensus blocks. Each consensus block borrows
+* its metadata (consequence step, escape status, claim, severity, rationale)
+* from the contributing sample block with the largest step overlap — ties go
+* to the higher-confidence block, then to the earlier sample — while its
+* confidence is the mean across every contributor. The returned blocks still
+* pass through the shared expansion, so width, count, and evidence rules are
+* enforced there, never assumed here.
+*/
+function consensusCodeTraceBlocks(sampleAssignments) {
+	const samples = sampleAssignments.length;
+	if (samples < 2) throw new RangeError("step-level consensus requires at least two samples");
+	const threshold = Math.ceil(samples / 2);
+	const votesByStep = /* @__PURE__ */ new Map();
+	sampleAssignments.forEach((assignments, sample) => {
+		const seen = /* @__PURE__ */ new Set();
+		for (const { step } of assignments) {
+			if (!Number.isSafeInteger(step) || step < 0) throw new RangeError(`sample ${sample} assigned a non-step value: ${step}`);
+			if (seen.has(step)) throw new Error(`sample ${sample} assigned step ${step} to more than one block`);
+			seen.add(step);
+			votesByStep.set(step, (votesByStep.get(step) ?? 0) + 1);
+		}
+	});
+	const stepVotes = [...votesByStep].sort(([left], [right]) => left - right).map(([step, votes]) => ({
+		step,
+		votes,
+		kept: votes >= threshold
+	}));
+	const keptSteps = stepVotes.filter((entry) => entry.kept).map((entry) => entry.step);
+	const blocks = [];
+	const blockDecisions = [];
+	for (const segment of contiguousSegments(keptSteps)) {
+		const contributors = segmentContributors(sampleAssignments, segment);
+		const donor = contributors.reduce(betterDonor);
+		const confidence = contributors.reduce((sum, contributor) => sum + contributor.block.confidence, 0) / contributors.length;
+		blocks.push({
+			firstStep: segment.firstStep,
+			lastStep: segment.lastStep,
+			consequenceStep: donor.block.consequenceStep,
+			escapeStatus: donor.block.escapeStatus,
+			severity: donor.block.severity,
+			claim: donor.block.claim,
+			confidence,
+			...donor.block.rationale === void 0 ? {} : { rationale: donor.block.rationale },
+			...donor.block.recommendedAction === void 0 ? {} : { recommendedAction: donor.block.recommendedAction },
+			metadata: {
+				...donor.block.metadata,
+				consensus_samples: samples,
+				consensus_threshold: threshold,
+				consensus_contributors: contributors.length,
+				consensus_donor_sample: donor.sample
+			}
+		});
+		blockDecisions.push({
+			firstStep: segment.firstStep,
+			lastStep: segment.lastStep,
+			consequenceStep: donor.block.consequenceStep,
+			escapeStatus: donor.block.escapeStatus,
+			confidence,
+			donor: publicContributor(donor),
+			contributors: contributors.map(publicContributor)
+		});
+	}
 	return {
-		kind: "agent-eval/analyst-benchmark-run",
-		createdAt: value.createdAt,
-		identitySha256: currentIdentitySha256,
-		localIdentitySha256: currentLocalIdentitySha256,
-		identity: currentIdentity
+		blocks,
+		decision: {
+			samples,
+			threshold,
+			stepVotes,
+			blocks: blockDecisions
+		}
 	};
 }
-function renderInitializationComplete(manifest) {
-	return `${JSON.stringify({
-		kind: "agent-eval/analyst-benchmark-initialization-complete",
-		runIdentitySha256: manifest.identitySha256,
-		localIdentitySha256: manifest.localIdentitySha256,
-		createdAt: manifest.createdAt
-	}, null, 2)}\n`;
+/**
+* All (sample, block) pairs whose accepted steps intersect the segment,
+* ordered by sample then by first overlapping step — the deterministic
+* tie-break order for donor selection.
+*/
+function segmentContributors(sampleAssignments, segment) {
+	const contributors = [];
+	sampleAssignments.forEach((assignments, sample) => {
+		const overlapByBlock = /* @__PURE__ */ new Map();
+		for (const { step, block } of assignments) {
+			if (step < segment.firstStep || step > segment.lastStep) continue;
+			overlapByBlock.set(block, (overlapByBlock.get(block) ?? 0) + 1);
+		}
+		for (const [block, overlapSteps] of overlapByBlock) contributors.push({
+			sample,
+			block,
+			overlapSteps
+		});
+	});
+	return contributors;
 }
-async function assertAbsentOrExact(path, expected, label) {
-	const existing = await readOptionalRegularFile(path, label);
-	if (existing !== void 0 && existing !== expected) throw new Error(`${label} does not exactly match interrupted initialization: ${path}`);
+function betterDonor(left, right) {
+	if (right.overlapSteps !== left.overlapSteps) return right.overlapSteps > left.overlapSteps ? right : left;
+	if (right.block.confidence !== left.block.confidence) return right.block.confidence > left.block.confidence ? right : left;
+	return left;
 }
-async function readOptionalRegularFile(path, label) {
-	if (!await regularFileExists(path)) return void 0;
-	return readRegularFile(path, label);
+function publicContributor(contributor) {
+	return {
+		sample: contributor.sample,
+		firstStep: contributor.block.firstStep,
+		lastStep: contributor.block.lastStep,
+		consequenceStep: contributor.block.consequenceStep,
+		confidence: contributor.block.confidence,
+		overlapSteps: contributor.overlapSteps
+	};
 }
-function createObservationAppender(path, runIdentitySha256, progress) {
-	let writes = Promise.resolve();
-	const seen = new Set(progress.observations.map(observationKey));
-	return (observation) => {
-		const write = writes.then(async () => {
-			assertAnalystBenchmarkObservation(observation, "benchmark observation");
-			const key = observationKey(observation);
-			if (seen.has(key)) throw new Error(`refusing duplicate benchmark observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}'`);
-			const rowWithoutDigest = {
-				sequence: progress.nextSequence,
-				runIdentitySha256,
-				previousRowSha256: progress.previousRowSha256,
-				observation
-			};
-			const row = {
-				...rowWithoutDigest,
-				rowSha256: digestCanonical(rowWithoutDigest)
-			};
-			await appendDurable(path, `${JSON.stringify(row)}\n`);
-			progress.nextSequence += 1;
-			progress.previousRowSha256 = row.rowSha256;
-			progress.observations.push(observation);
-			seen.add(key);
+function contiguousSegments(sortedSteps) {
+	const segments = [];
+	for (const step of sortedSteps) {
+		const current = segments[segments.length - 1];
+		if (current && step === current.lastStep + 1) {
+			current.lastStep = step;
+			continue;
+		}
+		segments.push({
+			firstStep: step,
+			lastStep: step
 		});
-		writes = write;
-		return write;
+	}
+	return segments;
+}
+//#endregion
+//#region src/analyst/benchmark-public-rlm.ts
+/** Public benchmark candidate that runs the actual recursive trace analyst. */
+function createPublicBenchmarkRlmRunner(dataset, config) {
+	const costLedger = config.costLedger ?? new CostLedger();
+	const samples = config.dspyRlm?.samples ?? 1;
+	if (!Number.isSafeInteger(samples) || samples < 1) throw new RangeError("dspyRlm.samples must be a positive safe integer");
+	if (samples > 1 && dataset !== "codetracebench") throw new Error("dspyRlm.samples > 1 requires the codetracebench dataset; step-level consensus is defined on its block grammar");
+	const limits = {
+		maxIterations: config.dspyRlm?.maxIterations ?? 14,
+		maxLlmCalls: config.dspyRlm?.maxLlmCalls ?? 8,
+		maxToolCalls: config.dspyRlm?.maxToolCalls ?? 80,
+		maxOutputChars: config.dspyRlm?.maxOutputChars ?? 8e3
+	};
+	const pricing = config.pricing ?? pricingForModel(config.model);
+	const engine = createDspyRlmTraceEngine({
+		baseUrl: config.baseUrl,
+		apiKey: config.apiKey,
+		model: config.model,
+		maxOutputTokens: config.maxOutputTokens,
+		timeoutMs: config.timeoutMs,
+		maxCostUsd: config.maxCostUsdPerAnalysis ?? 1,
+		pricing,
+		...config.dspyRlm?.runner ? { runner: config.dspyRlm.runner } : {}
+	});
+	const instructions = config.instructionsOverride?.text ?? publicBenchmarkRlmInstructions(dataset);
+	const protocolSha256 = effectiveAnalystProtocolSha256(dataset, config.instructionsOverride);
+	const definition = publicBenchmarkDefinition(dataset, limits, instructions);
+	const { instructionsOverride: _rlmOnlyOverride, ...directConfig } = config;
+	const abstentionFallbackRunner = createPublicBenchmarkDirectRunner(dataset, {
+		...directConfig,
+		costLedger
+	});
+	return {
+		id: "dspy-rlm",
+		async analyze(input, context) {
+			const trajectoryId = trajectoryIdFromCaseId(dataset, context.caseId);
+			const tags = {
+				benchmarkCaseId: context.caseId,
+				benchmarkRepetition: String(context.repetition)
+			};
+			let usage;
+			let rawFindings = [];
+			try {
+				if (!input.traceStore) throw new Error(`${dataset} DSPy RLM runner requires a trace store`);
+				if (samples > 1) {
+					const store = input.traceStore;
+					const caseUsageFilter = {
+						channel: "analyst",
+						tags
+					};
+					const sampleRuns = [];
+					const sampleAssignments = [];
+					let totalModelCalls = 0;
+					let totalToolCalls = 0;
+					for (let sample = 0; sample < samples; sample += 1) {
+						let sampleUsage;
+						const completed = await runTraceAnalyst({
+							definition,
+							engine,
+							store,
+							context: {
+								runId: context.caseId,
+								correlationId: `${context.caseId}:${context.repetition}:sample-${sample}`,
+								costLedger,
+								costPhase: "analyst.public-benchmark.dspy-rlm",
+								tags,
+								recordUsage: (receipt) => {
+									sampleUsage = receipt;
+									usage = usageReceiptFromCostLedger(costLedger, caseUsageFilter);
+								},
+								signal: context.signal
+							}
+						});
+						const producedAt = (/* @__PURE__ */ new Date()).toISOString();
+						const sampleFindings = completed.findings.map((finding) => makeFinding({
+							analyst_id: "dspy-rlm",
+							area: "incorrect",
+							subject: finding.subject,
+							claim: finding.claim,
+							rationale: finding.rationale,
+							severity: finding.severity,
+							confidence: finding.confidence,
+							evidence_refs: evidenceRefsFromRawFinding(finding),
+							recommended_action: finding.recommended_action,
+							metadata: {
+								analysis_mode: "recursive",
+								engine: "dspy-rlm",
+								model: config.model,
+								sample,
+								...codeTraceBlockMetadataFromSubject(finding.subject) ?? {}
+							},
+							produced_at: producedAt
+						}));
+						rawFindings = [...rawFindings, ...sampleFindings];
+						const adapted = await adaptPublicBenchmarkFindings({
+							dataset,
+							trajectoryId,
+							findings: sampleFindings,
+							analystId: "dspy-rlm",
+							store,
+							...context.signal ? { signal: context.signal } : {}
+						});
+						const assignments = adapted.stepBlocks ?? [];
+						sampleAssignments.push(assignments);
+						totalModelCalls += completed.modelCalls;
+						totalToolCalls += completed.toolCalls;
+						sampleRuns.push({
+							sample,
+							answer: completed.answer,
+							trajectory: completed.trajectory,
+							modelCalls: completed.modelCalls,
+							toolCalls: completed.toolCalls,
+							runtime: completed.runtime,
+							blocks: sampleBlockRecords(assignments),
+							steps: assignments.map((assignment) => assignment.step),
+							...adapted.diagnostics ? { blockDiagnostics: adapted.diagnostics } : {},
+							...sampleUsage ? { usage: sampleUsage } : {}
+						});
+					}
+					const consensus = consensusCodeTraceBlocks(sampleAssignments);
+					const expanded = await expandCodeTraceFailureBlocks({
+						trajectoryId,
+						blocks: consensus.blocks,
+						store,
+						analystId: "dspy-rlm",
+						producedAt: (/* @__PURE__ */ new Date()).toISOString(),
+						...context.signal ? { signal: context.signal } : {}
+					});
+					let fallback;
+					if (consensus.blocks.length === 0) fallback = await abstentionFallbackRunner.analyze(input, context);
+					usage = usageReceiptFromCostLedger(costLedger, caseUsageFilter);
+					return {
+						findings: fallback && !fallback.error ? fallback.findings : expanded.findings,
+						usage,
+						metadata: {
+							analysisMode: "recursive",
+							engine: "dspy-rlm",
+							protocolSha256,
+							samples,
+							sampleRuns,
+							consensus: consensus.decision,
+							blockDiagnostics: expanded.diagnostics,
+							modelCalls: totalModelCalls,
+							toolCalls: totalToolCalls,
+							...fallback ? {
+								abstentionFallback: "direct",
+								...fallback.metadata ? { abstentionFallbackMetadata: fallback.metadata } : {},
+								...fallback.error ? { abstentionFallbackError: fallback.error } : {}
+							} : {}
+						}
+					};
+				}
+				const completed = await runTraceAnalyst({
+					definition,
+					engine,
+					store: input.traceStore,
+					context: {
+						runId: context.caseId,
+						correlationId: `${context.caseId}:${context.repetition}`,
+						costLedger,
+						costPhase: "analyst.public-benchmark.dspy-rlm",
+						tags,
+						recordUsage: (receipt) => {
+							usage = receipt;
+						},
+						signal: context.signal
+					}
+				});
+				const producedAt = (/* @__PURE__ */ new Date()).toISOString();
+				rawFindings = completed.findings.map((finding) => makeFinding({
+					analyst_id: "dspy-rlm",
+					area: dataset === "agentrx" ? "root-cause" : "incorrect",
+					subject: finding.subject,
+					claim: finding.claim,
+					rationale: finding.rationale,
+					severity: finding.severity,
+					confidence: finding.confidence,
+					evidence_refs: evidenceRefsFromRawFinding(finding),
+					recommended_action: finding.recommended_action,
+					metadata: {
+						analysis_mode: "recursive",
+						engine: "dspy-rlm",
+						model: config.model,
+						...dataset === "codetracebench" ? codeTraceBlockMetadataFromSubject(finding.subject) : {}
+					},
+					produced_at: producedAt
+				}));
+				const adapted = await adaptPublicBenchmarkFindings({
+					dataset,
+					trajectoryId,
+					findings: rawFindings,
+					analystId: "dspy-rlm",
+					store: input.traceStore,
+					...context.signal ? { signal: context.signal } : {}
+				});
+				let fallback;
+				if (completed.findings.length === 0) {
+					fallback = await abstentionFallbackRunner.analyze(input, context);
+					usage = usageReceiptFromCostLedger(costLedger, {
+						channel: "analyst",
+						tags: {
+							benchmarkCaseId: context.caseId,
+							benchmarkRepetition: String(context.repetition)
+						}
+					});
+				}
+				return {
+					findings: fallback && !fallback.error ? fallback.findings : adapted.findings,
+					usage,
+					metadata: {
+						analysisMode: "recursive",
+						engine: "dspy-rlm",
+						protocolSha256,
+						...adapted.diagnostics ? { blockDiagnostics: adapted.diagnostics } : {},
+						answer: completed.answer,
+						trajectory: completed.trajectory,
+						modelCalls: completed.modelCalls,
+						toolCalls: completed.toolCalls,
+						runtime: completed.runtime,
+						...fallback ? {
+							abstentionFallback: "direct",
+							...fallback.metadata ? { abstentionFallbackMetadata: fallback.metadata } : {},
+							...fallback.error ? { abstentionFallbackError: fallback.error } : {}
+						} : {}
+					}
+				};
+			} catch (error) {
+				if (context.signal?.aborted) throw error;
+				if (isPaidCallControlError(error)) throw error;
+				return {
+					findings: [],
+					usage,
+					error: publicBenchmarkError(error, [config.apiKey]),
+					metadata: {
+						analysisMode: "recursive",
+						engine: "dspy-rlm",
+						...samples > 1 ? { samples } : {},
+						rawFindings
+					}
+				};
+			}
+		}
 	};
 }
-async function readProgress(path, runIdentitySha256, caseIds, repetitions, analystRunnerId) {
-	const rawLines = (await readRegularFile(path, "benchmark observation log")).split("\n");
-	if (rawLines.at(-1) === "") rawLines.pop();
-	const observations = [];
-	const seen = /* @__PURE__ */ new Set();
-	const executionIndexes = /* @__PURE__ */ new Set();
-	let previousRowSha256 = null;
-	const allowedCases = new Set(caseIds);
-	const plannedObservationCount = caseIds.length * 2 * repetitions;
-	for (const [index, line] of rawLines.entries()) {
-		if (!line.trim()) throw new Error(`benchmark observation log contains an empty row at line ${index + 1}`);
-		const parsed = parseJson(line, `${path}:${index + 1}`);
-		if (!isRecord$1(parsed)) throw new TypeError(`benchmark observation row ${index + 1} must be an object`);
-		assertExactKeys(parsed, [
-			"sequence",
-			"runIdentitySha256",
-			"previousRowSha256",
-			"observation",
-			"rowSha256"
-		], `benchmark observation row ${index + 1}`);
-		assertAnalystBenchmarkObservation(parsed.observation, `benchmark observation row ${index + 1}.observation`);
-		const observation = parsed.observation;
-		const key = observationKey(observation);
-		if (seen.has(key)) throw new Error(`duplicate benchmark observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}' at line ${index + 1}`);
-		if (parsed.sequence !== index) throw new Error(`benchmark observation row ${index + 1} has sequence ${String(parsed.sequence)}; expected ${index}`);
-		if (parsed.runIdentitySha256 !== runIdentitySha256) throw new Error(`benchmark observation row ${index + 1} belongs to another run`);
-		if (parsed.previousRowSha256 !== previousRowSha256) throw new Error(`benchmark observation row ${index + 1} breaks the digest chain`);
-		if (!isSha256(parsed.rowSha256)) throw new TypeError(`benchmark observation row ${index + 1} has an invalid digest`);
-		if (digestCanonical({
-			sequence: parsed.sequence,
-			runIdentitySha256: parsed.runIdentitySha256,
-			previousRowSha256: parsed.previousRowSha256,
-			observation
-		}) !== parsed.rowSha256) throw new Error(`benchmark observation row ${index + 1} digest does not match its contents`);
-		if (!allowedCases.has(observation.caseId) || observation.runnerId !== "empty" && observation.runnerId !== analystRunnerId || observation.repetition >= repetitions || observation.executionIndex >= plannedObservationCount) throw new Error(`benchmark observation row ${index + 1} does not match a planned case, runner, and repetition`);
-		if (executionIndexes.has(observation.executionIndex)) throw new Error(`duplicate benchmark executionIndex ${observation.executionIndex} at line ${index + 1}`);
-		observations.push(observation);
-		seen.add(key);
-		executionIndexes.add(observation.executionIndex);
-		previousRowSha256 = parsed.rowSha256;
+/** Per-sample accepted blocks with the exact steps the expansion kept for each. */
+function sampleBlockRecords(assignments) {
+	const stepsByBlock = /* @__PURE__ */ new Map();
+	for (const { step, block } of assignments) {
+		const steps = stepsByBlock.get(block);
+		if (steps) steps.push(step);
+		else stepsByBlock.set(block, [step]);
 	}
+	return [...stepsByBlock].map(([block, acceptedSteps]) => ({
+		firstStep: block.firstStep,
+		lastStep: block.lastStep,
+		consequenceStep: block.consequenceStep,
+		escapeStatus: block.escapeStatus,
+		severity: block.severity,
+		confidence: block.confidence,
+		claim: block.claim,
+		acceptedSteps
+	}));
+}
+function publicBenchmarkDefinition(dataset, limits, instructions) {
 	return {
-		observations,
-		nextSequence: observations.length,
-		previousRowSha256
+		id: dataset === "agentrx" ? "agentrx-dspy-rlm" : "codetracebench-dspy-rlm",
+		description: dataset === "agentrx" ? "Localizes the first unrecoverable root-cause step." : "Localizes every incorrect state-changing assistant step.",
+		area: dataset === "agentrx" ? "root-cause" : "incorrect",
+		version: "1.0.0",
+		question: dataset === "agentrx" ? "What is the first unrecoverable root cause in this failed trajectory?" : "Which assistant steps are incorrect under the CodeTraceBench definition?",
+		instructions,
+		toolGroup: "singleTrace",
+		limits
 	};
 }
-async function writeExclusiveOrVerify(path, content) {
-	try {
-		await writeExclusive(path, content);
-	} catch (error) {
-		if (!isNodeError(error, "EEXIST")) throw error;
-		if (await readRegularFile(path, "existing benchmark artifact") !== content) throw new Error(`refusing to replace existing benchmark artifact: ${path}`);
-	}
+function pricingForModel(model) {
+	const pricing = resolveModelPricing(model);
+	if (!pricing) throw new Error(`no pricing is configured for '${model}'; provide PublicAnalystBenchmarkModelConfig.pricing`);
+	return {
+		inputUsdPerMillion: pricing.input * 1e3,
+		outputUsdPerMillion: pricing.output * 1e3
+	};
 }
-async function regularFileExists(path) {
-	try {
-		const fileStat = await lstat(path);
-		if (!fileStat.isFile() || fileStat.isSymbolicLink()) throw new Error(`benchmark artifact path must be a real file: ${path}`);
-		return true;
-	} catch (error) {
-		if (isNodeError(error, "ENOENT")) return false;
-		throw error;
-	}
+function trajectoryIdFromCaseId(dataset, caseId) {
+	const prefix = dataset === "agentrx" ? "agentrx:" : "codetrace:";
+	if (!caseId.startsWith(prefix) || caseId.length === prefix.length) throw new Error(`unexpected ${dataset} benchmark case id '${caseId}'`);
+	return caseId.slice(prefix.length);
 }
-async function appendDurable(path, content) {
-	const handle = await open(path, constants.O_APPEND | constants.O_WRONLY | constants.O_NOFOLLOW);
+function isPaidCallControlError(error) {
+	return error instanceof CostAccountingIncompleteError || error instanceof CostCallConflictError || error instanceof CostCeilingReachedError || error instanceof CostLedgerPersistenceError || error instanceof CostReceiptCaptureError || error instanceof CostReservationExceededError;
+}
+//#endregion
+//#region src/analyst/benchmark-public-data.ts
+const DEFAULT_MAX_PUBLIC_BENCHMARK_LABEL_BYTES = 256 * 1024 * 1024;
+const INPUT_OPEN_FLAGS = constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0);
+async function loadPublicBenchmarkRows(path) {
+	return parsePublicBenchmarkRows((await readImmutableInputSnapshot(resolve(path), DEFAULT_MAX_PUBLIC_BENCHMARK_LABEL_BYTES)).text, path);
+}
+function parsePublicBenchmarkRows(text, path) {
+	const trimmed = text.trim();
+	if (!trimmed) throw new Error(`public analyst benchmark dataset is empty: ${path}`);
+	let parsed;
 	try {
-		await handle.writeFile(content, "utf8");
-		await handle.sync();
-	} finally {
-		await handle.close();
+		parsed = JSON.parse(trimmed);
+	} catch {
+		return parseJsonl(trimmed, path);
 	}
+	if (Array.isArray(parsed)) return records(parsed, path);
+	if (isRecord(parsed) && Array.isArray(parsed.data)) return records(parsed.data, `${path}.data`);
+	if (isRecord(parsed) && Array.isArray(parsed.cases)) return records(parsed.cases, `${path}.cases`);
+	if (isRecord(parsed)) return [parsed];
+	throw new TypeError(`public analyst benchmark dataset must contain JSON objects: ${path}`);
 }
-async function writeExclusive(path, content) {
-	const temporary = `${path}.tmp-${process.pid}-${randomUUID()}`;
-	let handle;
-	try {
-		handle = await open(temporary, "wx");
-		await handle.writeFile(content, "utf8");
-		await handle.sync();
-		await handle.close();
-		handle = void 0;
-		await link(temporary, path);
-		await syncDirectory(dirname(path));
-	} finally {
-		await handle?.close().catch(() => void 0);
-		await unlink(temporary).catch(() => void 0);
+function selectPublicBenchmarkRows(dataset, rows, options) {
+	positiveSafeInteger(options.limit, "limit");
+	safeInteger(options.seed, "seed");
+	if (rows.length === 0) throw new Error("public analyst benchmark dataset has no rows");
+	const byId = /* @__PURE__ */ new Map();
+	for (const row of rows) {
+		const id = publicBenchmarkRowId(dataset, row);
+		if (byId.has(id)) throw new Error(`public analyst benchmark dataset repeats trajectory id '${id}'`);
+		byId.set(id, row);
 	}
+	return [...byId].sort(([left], [right]) => selectionKey(options.seed, left).localeCompare(selectionKey(options.seed, right)) || left.localeCompare(right)).slice(0, Math.min(options.limit, byId.size)).map(([, row]) => row);
 }
-async function readRegularFile(path, label) {
-	let fileStat;
-	try {
-		fileStat = await lstat(path);
-	} catch (error) {
-		if (isNodeError(error, "ENOENT")) throw new Error(`${label} is missing: ${path}`);
-		throw error;
+function publicBenchmarkDistributions(dataset, rows) {
+	const values = {
+		class: [],
+		agent: [],
+		model: [],
+		difficulty: [],
+		solved: []
+	};
+	for (const row of rows) {
+		const benchmarkCase = dataset === "agentrx" ? agentRxBenchmarkCase(row, void 0) : codeTraceBenchCase(row, void 0);
+		values.class.push(dataset === "codetracebench" ? benchmarkCase.expectedIssues.length > 0 ? "positive" : row.solved === true ? "trusted-negative" : row.solved === false ? "unlabeled-failure" : "unlabeled-unknown" : benchmarkCase.expectedIssues[0]?.areas?.[0]);
+		values.agent.push(scalarDistributionValue(row.agent) ?? (dataset === "agentrx" ? rootAgent(row) : void 0));
+		values.model.push(scalarDistributionValue(row.model));
+		values.difficulty.push(scalarDistributionValue(row.difficulty));
+		values.solved.push(scalarDistributionValue(row.solved));
 	}
-	if (!fileStat.isFile() || fileStat.isSymbolicLink()) throw new Error(`${label} must be a real file: ${path}`);
-	return readFile(path, "utf8");
+	return {
+		class: valueDistribution(values.class),
+		agent: valueDistribution(values.agent),
+		model: valueDistribution(values.model),
+		difficulty: valueDistribution(values.difficulty),
+		solved: valueDistribution(values.solved)
+	};
 }
-async function syncDirectory(path) {
-	const directory = await open(path, "r");
-	try {
-		await directory.sync();
-	} finally {
-		await directory.close();
+function publicBenchmarkSelectionReport(dataset, source, selected, seed) {
+	const census = source.length === selected.length;
+	return {
+		method: census ? "census" : "deterministic-hash",
+		seed,
+		sourceCount: source.length,
+		selectedCount: selected.length,
+		stratified: false,
+		representativeOfInput: census,
+		source: publicBenchmarkDistributions(dataset, source),
+		selected: publicBenchmarkDistributions(dataset, selected)
+	};
+}
+async function preparePublicAnalystBenchmark(options) {
+	const labelsPath = resolve(options.labelsPath);
+	const traceRoot = resolve(options.traceDir);
+	const artifactRoot = options.artifactDir ? resolve(options.artifactDir) : void 0;
+	const labelSnapshot = await readImmutableInputSnapshot(labelsPath, DEFAULT_MAX_PUBLIC_BENCHMARK_LABEL_BYTES);
+	const rows = parsePublicBenchmarkRows(labelSnapshot.text, labelsPath);
+	const selected = selectPublicBenchmarkRows(options.dataset, rows, {
+		limit: options.limit,
+		seed: options.seed
+	});
+	const stores = await indexSelectedSingleTraceFiles(traceRoot, new Set(selected.map((row) => publicBenchmarkRowId(options.dataset, row))));
+	const resolver = traceStoreEvidenceResolver((input) => {
+		if (!input.traceStore) throw new Error("prepared benchmark case has no trace store");
+		return input.traceStore;
+	});
+	const traceFiles = [];
+	const verificationArtifacts = [];
+	const cases = [];
+	for (const row of selected) {
+		const trajectoryId = publicBenchmarkRowId(options.dataset, row);
+		const indexed = stores.get(trajectoryId);
+		if (!indexed) throw new Error(`public analyst benchmark trace directory has no single-trace OTLP JSONL for '${trajectoryId}'`);
+		let modelVisibleOtlp = indexed.text;
+		let traceStore = indexed.store;
+		let artifactDir;
+		let verificationManifest;
+		if (options.dataset === "codetracebench") {
+			if (!options.artifactDir?.trim()) throw new Error("--artifact-dir is required for CodeTraceBench so final verification evidence is not omitted");
+			const artifacts = await loadCodeTraceVerificationArtifacts({
+				artifactDir: options.artifactDir,
+				row,
+				maxBytes: options.maxArtifactBytes ?? 8388608
+			});
+			for (const artifact of artifacts.files) assertNoBenchmarkLabelsInArtifact({
+				traceId: trajectoryId,
+				relativePath: artifact.relativePath,
+				content: artifact.content
+			});
+			verificationManifest = shareableVerificationManifest(artifacts.manifest, artifactRoot ?? resolve(options.artifactDir));
+			const collisions = await indexed.store.hasSpans({
+				trace_id: trajectoryId,
+				span_ids: [...artifacts.manifest.files.map((file) => file.spanId), artifacts.manifest.outcomeSpanId]
+			});
+			if (collisions.length > 0) throw new Error(`CodeTraceBench '${trajectoryId}' trace already contains benchmark verification span '${collisions[0]}'`);
+			modelVisibleOtlp = appendVerificationArtifactsToOtlp(indexed.text, trajectoryId, artifacts, indexed.latestTimestamp);
+			traceStore = otlpTextToTraceAnalysisStore(modelVisibleOtlp);
+			artifactDir = artifacts.manifest.status === "present" ? artifacts.manifest.caseDirectory : void 0;
+			verificationArtifacts.push(verificationManifest);
+		}
+		const labelLeakScan = assertNoBenchmarkLabelsInTrace({
+			traceId: trajectoryId,
+			otlpText: modelVisibleOtlp
+		});
+		const input = {
+			traceStore,
+			artifactDir
+		};
+		const benchmarkCase = options.dataset === "agentrx" ? agentRxBenchmarkCase(row, input, { stepCount: indexed.stepCount }) : codeTraceBenchCase(row, input);
+		for (const evidence of benchmarkCase.labeledEvidence ?? []) if (!await resolver({
+			caseId: benchmarkCase.id,
+			caseInput: input,
+			evidence: {
+				kind: evidence.kind ?? "span",
+				uri: evidence.uri
+			}
+		})) throw new Error(`${benchmarkCase.id}: missing labeled span ${spanIdFromEvidence(evidence.uri) ?? evidence.uri} in ${indexed.path}`);
+		cases.push({
+			...benchmarkCase,
+			metadata: {
+				...benchmarkCase.metadata,
+				traceFileRelativePath: slashRelative(traceRoot, indexed.path),
+				traceFileSha256: indexed.sha256,
+				labelLeakScan,
+				...verificationManifest ? { verificationArtifacts: verificationManifest } : {}
+			}
+		});
+		traceFiles.push({
+			traceId: trajectoryId,
+			relativePath: slashRelative(traceRoot, indexed.path),
+			sha256: indexed.sha256
+		});
 	}
-}
-function isNodeError(error, code) {
-	return error instanceof Error && "code" in error && error.code === code;
-}
-//#endregion
-//#region src/analyst/benchmark-comparison.ts
-function compareAnalystRunners(result, options) {
-	const confidence = options.confidence ?? .95;
-	const resamples = options.resamples ?? 2e3;
-	assertComparisonControls(confidence, resamples);
-	const runnerIds = new Set(result.summaries.map((summary) => summary.runnerId));
-	if (!runnerIds.has(options.baselineRunnerId)) throw new TypeError(`unknown baseline analyst runner '${options.baselineRunnerId}'`);
-	if (!runnerIds.has(options.candidateRunnerId)) throw new TypeError(`unknown candidate analyst runner '${options.candidateRunnerId}'`);
-	if (options.baselineRunnerId === options.candidateRunnerId) throw new TypeError("baseline and candidate analyst runners must be different");
-	const baseline = observationsByCase(result.observations, options.baselineRunnerId);
-	const candidate = observationsByCase(result.observations, options.candidateRunnerId);
-	const populationRepresentativenessProven = result.provenance.metadata?.populationRepresentativenessProven === true;
-	const metrics = METRICS.map((metric) => compareMetric({
-		metric,
-		baseline,
-		candidate,
-		confidence,
-		resamples,
-		seed: options.seed,
-		populationRepresentativenessProven
-	}));
 	return {
-		baselineRunnerId: options.baselineRunnerId,
-		candidateRunnerId: options.candidateRunnerId,
-		metrics
+		cases,
+		sourceRowCount: rows.length,
+		selectedCaseIds: cases.map((testCase) => testCase.id),
+		labelsSha256: labelSnapshot.sha256,
+		traceFiles,
+		verificationArtifacts,
+		selection: publicBenchmarkSelectionReport(options.dataset, rows, selected, options.seed)
 	};
 }
-function compareMetric(options) {
-	const pairedCases = [];
-	let eligibleObservations = 0;
-	let pairedObservations = 0;
-	let baselineMissingObservations = 0;
-	let candidateMissingObservations = 0;
-	let asymmetricMissingObservations = 0;
-	const caseIds = /* @__PURE__ */ new Set([...options.baseline.keys(), ...options.candidate.keys()]);
-	for (const caseId of caseIds) {
-		const baselineByRepetition = new Map((options.baseline.get(caseId) ?? []).map((observation) => [observation.repetition, observation]));
-		const candidateByRepetition = new Map((options.candidate.get(caseId) ?? []).map((observation) => [observation.repetition, observation]));
-		const caseBefore = [];
-		const caseAfter = [];
-		let clusterId;
-		const repetitions = /* @__PURE__ */ new Set([...baselineByRepetition.keys(), ...candidateByRepetition.keys()]);
-		for (const repetition of repetitions) {
-			const baselineObservation = baselineByRepetition.get(repetition);
-			const candidateObservation = candidateByRepetition.get(repetition);
-			const identity = baselineObservation ?? candidateObservation;
-			if (!identity || !metricApplies(identity, options.metric)) continue;
-			if (baselineObservation && candidateObservation) assertSameCaseIdentity(baselineObservation, candidateObservation);
-			eligibleObservations += 1;
-			clusterId = identity.clusterId;
-			const baselineValue = baselineObservation ? metricValue(baselineObservation, options.metric) : null;
-			const candidateValue = candidateObservation ? metricValue(candidateObservation, options.metric) : null;
-			const baselineMissing = baselineValue === null;
-			const candidateMissing = candidateValue === null;
-			if (baselineMissing) baselineMissingObservations += 1;
-			if (candidateMissing) candidateMissingObservations += 1;
-			if (baselineMissing !== candidateMissing) asymmetricMissingObservations += 1;
-			if (baselineMissing || candidateMissing) continue;
-			caseBefore.push(baselineValue);
-			caseAfter.push(candidateValue);
-			pairedObservations += 1;
-		}
-		if (caseBefore.length === 0 || !clusterId) continue;
-		pairedCases.push({
-			clusterId,
-			baseline: mean$1(caseBefore),
-			candidate: mean$1(caseAfter)
+async function indexSelectedSingleTraceFiles(traceDir, selectedTraceIds) {
+	if (selectedTraceIds.size === 0) throw new Error("public analyst benchmark selected no trace ids");
+	const files = (await readdir(traceDir, { withFileTypes: true })).filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")).map((entry) => resolve(traceDir, entry.name)).sort();
+	if (files.length === 0) throw new Error(`public analyst benchmark trace directory has no JSONL files: ${traceDir}`);
+	const indexed = /* @__PURE__ */ new Map();
+	for (const path of files) {
+		const snapshot = await readImmutableInputSnapshot(path, DEFAULT_MAX_TRACE_FILE_BYTES);
+		const store = createOtlpBufferTraceStore(snapshot.bytes);
+		const overview = await store.getOverview();
+		if (overview.total_traces !== 1 || overview.sample_trace_ids.length !== 1) throw new Error(`public analyst benchmark trace file must contain exactly one trace: ${path} contains ${overview.total_traces}`);
+		if (!overview.time_range) throw new Error(`public analyst benchmark trace file has no valid timestamps: ${path}`);
+		const traceId = overview.sample_trace_ids[0];
+		if (!selectedTraceIds.has(traceId)) continue;
+		if (indexed.has(traceId)) throw new Error(`public analyst benchmark trace id '${traceId}' appears in multiple files`);
+		indexed.set(traceId, {
+			path,
+			sha256: snapshot.sha256,
+			store,
+			latestTimestamp: overview.time_range.latest,
+			text: snapshot.text,
+			stepCount: traceStepCount(snapshot.text, path)
 		});
 	}
-	const byCluster = /* @__PURE__ */ new Map();
-	for (const pairedCase of pairedCases) {
-		const rows = byCluster.get(pairedCase.clusterId) ?? [];
-		rows.push(pairedCase);
-		byCluster.set(pairedCase.clusterId, rows);
-	}
-	const before = [...byCluster.values()].map((rows) => mean$1(rows.map((row) => row.baseline)));
-	const after = [...byCluster.values()].map((rows) => mean$1(rows.map((row) => row.candidate)));
-	const interval = before.length === 0 ? null : pairedBootstrap(before, after, {
-		confidence: options.confidence,
-		resamples: options.resamples,
-		statistic: "mean",
-		seed: options.seed
-	});
-	const survivorOnly = pairedObservations < eligibleObservations;
-	const limitations = [];
-	if (!interval?.gateEligible) limitations.push("fewer-than-20-independent-clusters");
-	if (!options.populationRepresentativenessProven) limitations.push("population-representativeness-not-proven");
-	if (survivorOnly) limitations.push("missing-observations");
-	const comparison = {
-		metric: options.metric,
-		direction: LOWER_IS_BETTER.has(options.metric) ? "lower" : "higher",
-		pairedCases: pairedCases.length,
-		pairedClusters: before.length,
-		eligibleObservations,
-		pairedObservations,
-		baselineMissingObservations,
-		candidateMissingObservations,
-		asymmetricMissingObservations,
-		survivorOnly,
-		baselineMean: before.length === 0 ? null : mean$1(before),
-		candidateMean: after.length === 0 ? null : mean$1(after),
-		meanDelta: interval?.mean ?? null,
-		intervalLow: interval?.low ?? null,
-		intervalHigh: interval?.high ?? null,
-		confidence: options.confidence,
-		resamples: options.resamples,
-		minimumSampleMet: interval?.gateEligible ?? false,
-		populationInferenceEligible: limitations.length === 0,
-		inferenceLimitations: limitations
-	};
-	assertValidComparison(comparison);
-	return comparison;
+	return indexed;
 }
-const METRICS = [
-	"completion",
-	"issueRecall",
-	"findingPrecision",
-	"f1",
-	"criticalStepAccuracy",
-	"citationCoverage",
-	"citationExcerptCoverage",
-	"citationLabelAgreement",
-	"citationResolution",
-	"trustedNegativeAccuracy",
-	"latencyMs",
-	"calls",
-	"inputTokens",
-	"outputTokens",
-	"reasoningTokens",
-	"cachedTokens",
-	"cacheWriteTokens",
-	"costUsd"
-];
-const LOWER_IS_BETTER = /* @__PURE__ */ new Set([
-	"latencyMs",
-	"calls",
-	"inputTokens",
-	"outputTokens",
-	"reasoningTokens",
-	"cachedTokens",
-	"cacheWriteTokens",
-	"costUsd"
-]);
-function observationsByCase(observations, runnerId) {
-	const byCase = /* @__PURE__ */ new Map();
-	for (const observation of observations) {
-		if (observation.runnerId !== runnerId) continue;
-		const rows = byCase.get(observation.caseId) ?? [];
-		rows.push(observation);
-		byCase.set(observation.caseId, rows);
+async function readImmutableInputSnapshot(path, maxBytes) {
+	if (!Number.isSafeInteger(maxBytes) || maxBytes < 1) throw new RangeError("benchmark input maxBytes must be a positive safe integer");
+	const handle = await open(path, INPUT_OPEN_FLAGS);
+	try {
+		return await readImmutableInputHandle(handle, path, maxBytes);
+	} finally {
+		await handle.close();
 	}
-	return byCase;
-}
-function assertSameCaseIdentity(baseline, candidate) {
-	if (baseline.clusterId !== candidate.clusterId || baseline.labelState !== candidate.labelState) throw new Error(`analyst comparison case identity differs for '${baseline.caseId}' repetition ${baseline.repetition}`);
-}
-function metricApplies(observation, metric) {
-	if (metric === "trustedNegativeAccuracy") return observation.labelState === "trusted-negative";
-	if (metric === "issueRecall" || metric === "findingPrecision" || metric === "f1") return observation.labelState === "positive";
-	if (metric === "criticalStepAccuracy") return observation.labelState === "positive" && observation.score.criticalStepAccuracy !== null;
-	return true;
 }
-function metricValue(observation, metric) {
-	if (metric === "completion") return observation.error ? 0 : 1;
-	if (metric === "latencyMs") return observation.latencyMs;
-	if (metric === "trustedNegativeAccuracy") {
-		if (observation.error) return 0;
-		return observation.score.predictionOnLabelEmptyCase ? 0 : 1;
+async function readImmutableInputHandle(handle, path, maxBytes) {
+	const before = await handle.stat({ bigint: true });
+	if (!before.isFile()) throw new TypeError(`public analyst benchmark input must be a regular file: ${path}`);
+	if (before.size > BigInt(maxBytes)) throw new RangeError(`public analyst benchmark input exceeds ${maxBytes} bytes: ${path} has ${before.size}`);
+	const size = Number(before.size);
+	const bytes = Buffer.allocUnsafe(size);
+	let offset = 0;
+	while (offset < size) {
+		const result = await handle.read(bytes, offset, size - offset, offset);
+		if (result.bytesRead === 0) throw new Error(`public analyst benchmark input changed while being read: ${path}`);
+		offset += result.bytesRead;
 	}
-	if (observation.error && (metric === "issueRecall" || metric === "findingPrecision" || metric === "f1" || metric === "criticalStepAccuracy")) return 0;
-	if (observation.error && (metric === "citationCoverage" || metric === "citationExcerptCoverage" || metric === "citationLabelAgreement" || metric === "citationResolution")) return null;
-	if (metric === "issueRecall") return observation.score.issueRecall;
-	if (metric === "findingPrecision") return observation.score.findingPrecision;
-	if (metric === "f1") return observation.score.f1;
-	if (metric === "criticalStepAccuracy") return observation.score.criticalStepAccuracy;
-	if (metric === "citationCoverage") return observation.score.citationCoverage;
-	if (metric === "citationExcerptCoverage") return observation.score.citationExcerptCoverage;
-	if (metric === "citationLabelAgreement") return observation.score.citationLabelAgreement;
-	if (metric === "citationResolution") return observation.evidenceResolution?.validity ?? null;
-	if (metric === "calls") return observation.usage?.calls ?? null;
-	if (metric === "inputTokens") return observation.usage?.tokens?.input ?? null;
-	if (metric === "outputTokens") return observation.usage?.tokens?.output ?? null;
-	if (metric === "reasoningTokens") return observation.usage?.tokens?.reasoning ?? null;
-	if (metric === "cachedTokens") return observation.usage?.tokens?.cached ?? null;
-	if (metric === "cacheWriteTokens") return observation.usage?.tokens?.cacheWrite ?? null;
-	if (observation.usage?.cost.kind === "uncaptured") return null;
-	return observation.usage?.cost.usd ?? null;
-}
-function mean$1(values) {
-	return values.reduce((sum, value) => sum + value, 0) / values.length;
-}
-function assertComparisonControls(confidence, resamples) {
-	if (!Number.isSafeInteger(resamples) || resamples <= 0 || resamples > 1e6) throw new Error(`compareAnalystRunners: resamples must be a positive safe integer no greater than 1000000, got ${String(resamples)}`);
-	if (!Number.isFinite(confidence) || confidence <= 0 || confidence >= 1) throw new Error(`compareAnalystRunners: confidence must be a finite number in (0,1), got ${String(confidence)}`);
+	const overflow = Buffer.allocUnsafe(1);
+	const extra = await handle.read(overflow, 0, 1, size);
+	const after = await handle.stat({ bigint: true });
+	if (extra.bytesRead !== 0 || before.dev !== after.dev || before.ino !== after.ino || before.size !== after.size || before.mtimeNs !== after.mtimeNs || before.ctimeNs !== after.ctimeNs) throw new Error(`public analyst benchmark input changed while being read: ${path}`);
+	let text;
+	try {
+		text = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
+	} catch (error) {
+		throw new TypeError(`public analyst benchmark input is not valid UTF-8: ${path}: ${error instanceof Error ? error.message : String(error)}`);
+	}
+	return Object.freeze({
+		bytes,
+		sha256: sha256Digest(bytes),
+		text
+	});
 }
-function assertValidComparison(comparison) {
-	if ([
-		"pairedCases",
-		"pairedClusters",
-		"eligibleObservations",
-		"pairedObservations",
-		"baselineMissingObservations",
-		"candidateMissingObservations",
-		"asymmetricMissingObservations",
-		"confidence",
-		"resamples"
-	].some((field) => !Number.isFinite(comparison[field])) || [
-		"baselineMean",
-		"candidateMean",
-		"meanDelta",
-		"intervalLow",
-		"intervalHigh"
-	].some((field) => comparison[field] !== null && !Number.isFinite(comparison[field]))) throw new Error(`compareAnalystRunners: ${comparison.metric} produced non-finite comparison output`);
-	if (comparison.intervalLow !== null && comparison.intervalHigh !== null && comparison.intervalLow > comparison.intervalHigh) throw new Error(`compareAnalystRunners: ${comparison.metric} produced an invalid confidence interval`);
+function traceStepCount(text, path) {
+	const steps = parseJsonl(text, path).map((row) => row.span_id).filter((spanId) => typeof spanId === "string").map((spanId) => /^step-(\d+)$/.exec(spanId)?.[1]).filter((step) => step !== void 0).map(Number).filter((step) => Number.isSafeInteger(step) && step > 0);
+	if (steps.length === 0) throw new Error(`public analyst benchmark trace has no step-<n> spans: ${path}`);
+	return Math.max(...steps);
 }
-//#endregion
-//#region src/analyst/benchmark-public-calibration.ts
-function summarizeCodeTraceCalibration(result) {
+function shareableVerificationManifest(manifest, artifactRoot) {
 	return {
-		protocol: "labeled-positive-and-solved-negative",
-		rationale: "Uses rows with incorrect-step labels as positives and solved label-empty rows as trusted negatives. Failed label-empty rows remain in the published result but are not treated as clean controls.",
-		runners: result.provenance.runnerIds.map((runnerId) => summarizeRunner(runnerId, result.observations.filter((observation) => observation.runnerId === runnerId)))
+		...manifest,
+		caseDirectory: slashRelative(artifactRoot, manifest.caseDirectory),
+		caseDirectoriesSearched: manifest.caseDirectoriesSearched.map((path) => slashRelative(artifactRoot, path)),
+		files: manifest.files.map((file) => ({
+			...file,
+			path: slashRelative(artifactRoot, file.path)
+		}))
 	};
 }
-function renderCodeTraceCalibrationMarkdown(summary) {
-	return [
-		"## CodeTraceBench Calibrated View",
-		"",
-		summary.rationale,
-		"",
-		"| Runner | Completed/selected | Failed | Positive runs | Trusted negative runs | Unlabeled runs | Failed label-empty | Unknown label-empty | Matched/expected steps | Predicted steps | Precision | Recall | F1 | Official all-row F1 | Official rows | Trusted-negative false positives | Trusted-negative failures | Unlabeled predictions | Unlabeled failures |",
-		"| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
-		...summary.runners.map((runner) => `| ${escapeCell$1(runner.runnerId)} | ${runner.completedRuns}/${runner.selectedRuns} | ${runner.failedRuns} | ${runner.positiveRuns} | ${runner.trustedNegativeRuns} | ${runner.unlabeledRuns} | ${runner.failedLabelEmptyRuns} | ${runner.unknownLabelEmptyRuns} | ${runner.matchedIncorrectSteps}/${runner.expectedIncorrectSteps} | ${runner.predictedIncorrectSteps} | ${rate$1(runner.precision)} | ${rate$1(runner.recall)} | ${rate$1(runner.f1)} | ${rate$1(runner.officialAllRowF1)} | ${runner.officialAllRowRuns} | ${rate$1(runner.trustedNegativeFalsePositiveRate)} | ${rate$1(runner.trustedNegativeFailureRate)} | ${rate$1(runner.unlabeledPredictionRate)} | ${rate$1(runner.unlabeledFailureRate)} |`)
-	].join("\n");
-}
-function summarizeRunner(runnerId, observations) {
-	const positive = observations.filter((observation) => observation.labelState === "positive");
-	const trustedNegative = observations.filter((observation) => observation.labelState === "trusted-negative");
-	const excluded = observations.filter((observation) => observation.labelState === "unlabeled");
-	const selected = [...positive, ...trustedNegative];
-	const expected = sum(positive.map((observation) => observation.score.expectedIssueCount));
-	const predicted = sum(selected.map((observation) => observation.error ? 0 : observation.findings.length));
-	const matched = sum(positive.map((observation) => observation.score.matchedIssueIds.length));
-	const precision = predicted === 0 ? expected > 0 ? 0 : null : matched / predicted;
-	const recall = ratio(matched, expected);
-	const completedTrustedNegative = trustedNegative.filter((observation) => !observation.error);
-	const completedExcluded = excluded.filter((observation) => !observation.error);
-	const officialRows = observations.map(officialCodeTraceF1);
-	return {
-		runnerId,
-		selectedRuns: selected.length,
-		positiveRuns: positive.length,
-		trustedNegativeRuns: trustedNegative.length,
-		unlabeledRuns: excluded.length,
-		failedLabelEmptyRuns: excluded.filter((observation) => observation.caseMetadata?.solved === false).length,
-		unknownLabelEmptyRuns: excluded.filter((observation) => observation.caseMetadata?.solved !== false).length,
-		completedRuns: selected.filter((observation) => !observation.error).length,
-		failedRuns: selected.filter((observation) => observation.error).length,
-		expectedIncorrectSteps: expected,
-		predictedIncorrectSteps: predicted,
-		matchedIncorrectSteps: matched,
-		officialAllRowF1: mean(officialRows),
-		officialAllRowRuns: officialRows.length,
-		precision,
-		recall,
-		f1: harmonicMean(precision, recall),
-		trustedNegativeFalsePositiveRate: ratio(completedTrustedNegative.filter((observation) => observation.score.predictionOnLabelEmptyCase).length, completedTrustedNegative.length),
-		trustedNegativeFailureRate: ratio(trustedNegative.filter((observation) => observation.error).length, trustedNegative.length),
-		unlabeledPredictionRate: ratio(completedExcluded.filter((observation) => observation.findings.length > 0).length, completedExcluded.length),
-		unlabeledFailureRate: ratio(excluded.filter((observation) => Boolean(observation.error)).length, excluded.length)
-	};
+function slashRelative(root, path) {
+	const value = relative(root, path);
+	if (!value || value === ".." || value.startsWith(`..${sep}`)) {
+		if (!value) return ".";
+		throw new Error(`benchmark artifact path escapes its declared root: ${path}`);
+	}
+	return value.replaceAll("\\", "/");
 }
-function officialCodeTraceF1(observation) {
-	const trajectoryId = observation.caseMetadata?.trajectoryId;
-	if (typeof trajectoryId !== "string" || !trajectoryId.trim()) throw new TypeError(`${observation.caseId}: CodeTraceBench trajectoryId metadata is missing`);
-	const expected = new Set([...observation.score.matchedIssueIds, ...observation.score.missedIssueIds].map((issueId) => {
-		const match = /^incorrect:(\d+)$/.exec(issueId);
-		if (!match) throw new TypeError(`${observation.caseId}: invalid incorrect-step label '${issueId}'`);
-		return Number(match[1]);
-	}));
-	const predicted = /* @__PURE__ */ new Set();
-	if (!observation.error) for (const finding of observation.findings) {
-		if (finding.area !== "incorrect") continue;
-		for (const evidence of finding.evidence_refs) {
-			const location = codeTraceStepFromEvidence(evidence.uri);
-			if (!location || location.traceId !== trajectoryId) throw new TypeError(`${observation.caseId}: invalid CodeTraceBench prediction evidence '${evidence.uri}'`);
-			predicted.add(location.step);
+function parseJsonl(text, path) {
+	const rows = [];
+	for (const [index, line] of text.split(/\r?\n/).entries()) {
+		const trimmed = line.trim();
+		if (!trimmed) continue;
+		let parsed;
+		try {
+			parsed = JSON.parse(trimmed);
+		} catch (error) {
+			throw new Error(`${path}:${index + 1}: invalid JSON: ${error instanceof Error ? error.message : String(error)}`);
 		}
+		if (!isRecord(parsed)) throw new TypeError(`${path}:${index + 1}: dataset row must be a JSON object`);
+		rows.push(parsed);
 	}
-	let matched = 0;
-	for (const step of predicted) if (expected.has(step)) matched += 1;
-	return harmonicMean(predicted.size === 0 ? 0 : matched / predicted.size, expected.size === 0 ? 0 : matched / expected.size) ?? 0;
-}
-function sum(values) {
-	return values.reduce((total, value) => total + value, 0);
-}
-function ratio(numerator, denominator) {
-	return denominator === 0 ? null : numerator / denominator;
-}
-function mean(values) {
-	return values.length === 0 ? null : sum(values) / values.length;
+	if (rows.length === 0) throw new Error(`public analyst benchmark dataset is empty: ${path}`);
+	return rows;
 }
-function harmonicMean(left, right) {
-	if (left === null || right === null) return null;
-	return left + right === 0 ? 0 : 2 * left * right / (left + right);
+function records(values, path) {
+	return values.map((value, index) => {
+		if (!isRecord(value)) throw new TypeError(`${path}[${index}] must be a JSON object`);
+		return value;
+	});
 }
-function rate$1(value) {
-	return value === null ? "n/a" : value.toFixed(3);
+function publicBenchmarkRowId(dataset, row) {
+	const value = dataset === "agentrx" ? row.trajectory_id : row.traj_id;
+	if (typeof value !== "string" && typeof value !== "number" || !String(value).trim()) throw new TypeError(`${dataset} dataset row requires a non-empty ${dataset === "agentrx" ? "trajectory_id" : "traj_id"}`);
+	return String(value);
 }
-function escapeCell$1(value) {
-	return value.replaceAll("|", "\\|").replaceAll("\n", " ");
+function spanIdFromEvidence(uri) {
+	const match = /\/span\/([^/]+)$/.exec(uri);
+	return match?.[1] ? decodeURIComponent(match[1]) : null;
 }
-//#endregion
-//#region src/analyst/benchmark-command-result.ts
-async function readAnalystBenchmarkArtifact(path) {
-	const value = parseJson(await readRegularFile(path, "analyst benchmark result"), path);
-	assertAnalystBenchmarkArtifact(value, "analyst benchmark result");
-	return value;
+function selectionKey(seed, id) {
+	return sha256Digest(`${seed}\u0000${id}`);
 }
-function assertCompletedArtifactMatchesRun(artifact, manifest, observations, prepared) {
-	if (artifact.runIdentitySha256 !== manifest.identitySha256) throw new Error("completed benchmark result belongs to another run");
-	assertSameObservations(artifact.result.observations, observations);
-	const expectedCount = manifest.identity.inputs.selectedCaseIds.length * manifest.identity.config.runnerIds.length * manifest.identity.config.repetitions;
-	if (observations.length !== expectedCount) throw new Error(`completed benchmark result has ${observations.length} observations; expected ${expectedCount}`);
-	const { config, inputs } = manifest.identity;
-	const verificationAvailability = {
-		cases: prepared.verificationArtifacts.length,
-		resultFilesPresent: prepared.verificationArtifacts.filter((artifact) => artifact.status === "present").length,
-		resultFilesMissing: prepared.verificationArtifacts.filter((artifact) => artifact.status === "missing").length,
-		outcomes: {
-			passed: prepared.verificationArtifacts.filter((artifact) => artifact.outcome.status === "passed").length,
-			failed: prepared.verificationArtifacts.filter((artifact) => artifact.outcome.status === "failed").length,
-			unavailable: prepared.verificationArtifacts.filter((artifact) => artifact.outcome.status === "unavailable").length
-		}
-	};
-	const expectedInputs = {
-		dataset: config.dataset,
-		datasetRevision: config.datasetRevision,
-		datasetSplit: config.datasetSplit,
-		labelsSha256: inputs.labelsSha256,
-		sourceRowCount: inputs.sourceRowCount,
-		traceFiles: inputs.traceFiles.map((traceFile) => ({ ...traceFile })),
-		verificationArtifacts: prepared.verificationArtifacts,
-		verificationAvailability,
-		selection: {
-			limit: config.limit,
-			seed: config.seed,
-			selectedCaseIds: [...inputs.selectedCaseIds],
-			report: prepared.selection
-		},
-		execution: {
-			repetitions: config.repetitions,
-			concurrency: config.concurrency,
-			model: config.model.id,
-			maxOutputTokens: config.model.maxOutputTokens,
-			timeoutMs: config.model.timeoutMs,
-			maxCostUsd: config.maxCostUsd,
-			maxArtifactBytes: config.maxArtifactBytes,
-			analystProtocolSha256: config.analystProtocolSha256,
-			implementationSha256: config.implementationSha256,
-			dependencyLockSha256: config.dependencyLockSha256
+function valueDistribution(values) {
+	const counts = /* @__PURE__ */ new Map();
+	let missing = 0;
+	for (const value of values) {
+		if (value === void 0) {
+			missing += 1;
+			continue;
 		}
+		counts.set(value, (counts.get(value) ?? 0) + 1);
+	}
+	return {
+		total: values.length,
+		missing,
+		counts: Object.fromEntries([...counts].sort(([left], [right]) => left.localeCompare(right)))
 	};
-	if (canonicalJson(artifact.inputs) !== canonicalJson(expectedInputs)) throw new Error("completed benchmark result inputs do not match the run manifest");
-	const provenance = artifact.result.provenance;
-	const expectedDatasetId = config.dataset === "agentrx" ? "microsoft/AgentRx" : "NJU-LINK/CodeTraceBench";
-	const expectedOutputAdapter = config.dataset === "agentrx" ? "agentrx-taxonomy-and-root-step" : "codetracebench-incorrect-block";
-	if (provenance.id !== `${config.dataset}-real-model-analyst` || provenance.startedAt !== manifest.createdAt || !Number.isFinite(Date.parse(provenance.endedAt)) || Date.parse(provenance.endedAt) < Date.parse(provenance.startedAt) || canonicalJson(provenance.dataset) !== canonicalJson({
-		id: expectedDatasetId,
-		revision: config.datasetRevision,
-		split: config.datasetSplit
-	}) || provenance.caseCount !== inputs.selectedCaseIds.length || canonicalJson(provenance.runnerIds) !== canonicalJson(config.runnerIds) || provenance.repetitions !== config.repetitions || provenance.maxConcurrency !== Math.min(config.concurrency, expectedCount) || provenance.runnerOrderSeed !== config.seed || provenance.metadata?.model !== config.model.id || provenance.metadata?.outputAdapter !== expectedOutputAdapter || provenance.metadata?.caseSelection !== prepared.selection.method || provenance.metadata?.caseSelectionSeed !== config.seed || provenance.metadata?.selectionStratified !== prepared.selection.stratified || provenance.metadata?.protocolSha256 !== config.analystProtocolSha256 || provenance.metadata?.implementationSha256 !== config.implementationSha256 || provenance.metadata?.dependencyLockSha256 !== config.dependencyLockSha256 || provenance.metadata?.populationRepresentativenessProven !== false) throw new Error("completed benchmark result provenance does not match the run manifest");
-	const expectedSummaries = config.runnerIds.map((runnerId) => summarizeAnalystBenchmarkRunner(runnerId, observations.filter((observation) => observation.runnerId === runnerId)));
-	if (canonicalJson(artifact.result.summaries) !== canonicalJson(expectedSummaries)) throw new Error("completed benchmark summaries do not match durable observations");
-	const expectedComparisons = [compareAnalystRunners(artifact.result, {
-		baselineRunnerId: "empty",
-		candidateRunnerId: config.runnerIds[1],
-		seed: config.seed
-	})];
-	if (canonicalJson(artifact.comparisons) !== canonicalJson(expectedComparisons)) throw new Error("completed benchmark comparisons do not match durable observations");
-	if (config.dataset === "codetracebench") {
-		if (canonicalJson(artifact.codeTraceCalibration) !== canonicalJson(summarizeCodeTraceCalibration(artifact.result)) || artifact.agentRxCalibration !== void 0) throw new Error("completed CodeTraceBench calibration does not match durable observations");
-	} else if (canonicalJson(artifact.agentRxCalibration) !== canonicalJson(summarizeAgentRxCalibration(artifact.result, "f228165bfec60a801fd5fedd9d8ffe0f9de0c69d")) || artifact.codeTraceCalibration !== void 0) throw new Error("completed AgentRx calibration does not match durable observations");
 }
-function assertSameObservations(expected, actual) {
-	if (expected.length !== actual.length) throw new Error(`benchmark result has ${expected.length} observations but the durable log has ${actual.length}`);
-	const expectedByKey = new Map(expected.map((observation) => [observationKey(observation), canonicalJson(observation)]));
-	for (const observation of actual) {
-		const key = observationKey(observation);
-		if (expectedByKey.get(key) !== canonicalJson(observation)) throw new Error(`benchmark result does not match durable observation '${observation.runnerId}/${observation.caseId}/${observation.repetition}'`);
-		expectedByKey.delete(key);
-	}
-	if (expectedByKey.size > 0) throw new Error("benchmark result is missing durable observations");
+function scalarDistributionValue(value) {
+	if (typeof value === "string") return value.trim() || void 0;
+	if (typeof value === "number" || typeof value === "boolean") return String(value);
+}
+function rootAgent(row) {
+	const rootCauseId = row.root_cause_failure_id ?? row.root_cause?.failure_id;
+	return scalarDistributionValue(row.failures.find((failure) => String(failure.failure_id) === String(rootCauseId))?.failed_agent);
 }
 //#endregion
 //#region src/analyst/benchmark-report.ts
@@ -4474,11 +4869,12 @@ async function executeAnalystBenchmarkCommand(config, dependencies) {
 				},
 				metadata: {
 					model: config.model.model,
+					rlmSamples: config.rlmSamples,
 					outputAdapter: config.dataset === "agentrx" ? "agentrx-taxonomy-and-root-step" : "codetracebench-incorrect-block",
 					caseSelection: prepared.selection.method,
 					caseSelectionSeed: config.seed,
 					selectionStratified: prepared.selection.stratified,
-					protocolSha256: publicBenchmarkProtocolSha256(config.dataset),
+					protocolSha256: effectiveAnalystProtocolSha256(config.dataset, config.model.instructionsOverride),
 					implementationSha256: ANALYST_BENCHMARK_IMPLEMENTATION_SHA256,
 					dependencyLockSha256: ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256,
 					populationRepresentativenessProven: false
@@ -4522,12 +4918,14 @@ async function executeAnalystBenchmarkCommand(config, dependencies) {
 			execution: {
 				repetitions: config.repetitions,
 				concurrency: config.concurrency,
+				rlmSamples: config.rlmSamples,
 				model: config.model.model,
 				maxOutputTokens: config.model.maxOutputTokens,
 				timeoutMs: config.model.timeoutMs,
 				maxCostUsd: config.maxCostUsd,
 				maxArtifactBytes: config.maxArtifactBytes,
-				analystProtocolSha256: publicBenchmarkProtocolSha256(config.dataset),
+				analystProtocolSha256: effectiveAnalystProtocolSha256(config.dataset, config.model.instructionsOverride),
+				...config.model.instructionsOverride ? { instructionsOverrideSha256: config.model.instructionsOverride.sha256 } : {},
 				implementationSha256: ANALYST_BENCHMARK_IMPLEMENTATION_SHA256,
 				dependencyLockSha256: ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256
 			}
@@ -4608,6 +5006,14 @@ Controls:
   --seed <integer>                 Case-selection and comparison seed. Default: 0
   --concurrency <positive integer> Parallel benchmark jobs. Default: 1
   --repetitions <positive integer> Runs per case and runner. Default: 1
+  --rlm-samples <positive integer> Recursive-engine runs per case; above 1 the
+                                   step-level majority consensus is scored
+                                   (CodeTraceBench + dspy-rlm only). Default: 1
+  --instructions-file <path>       Replace the recursive analyst instructions
+                                   with this file's text (dspy-rlm only). The
+                                   recorded protocol digest binds the stock
+                                   protocol to the override text, and
+                                   result.json records instructionsOverrideSha256.
   --max-output-tokens <positive>   Model output limit per call. Default: 16384
   --python <executable>             Python with agent-eval-rpc[dspy]. Default: python
   --timeout-ms <positive>          Model analyst deadline per case. Default: 300000
@@ -4636,6 +5042,12 @@ function parseCommandConfig(argv, env) {
 	const python = flags.get("python")?.trim();
 	const analyst = flags.get("analyst")?.trim() ?? "dspy-rlm";
 	if (analyst !== "dspy-rlm" && analyst !== "direct") throw new Error("--analyst must be 'dspy-rlm' or 'direct'");
+	const rlmSamples = positiveFlag(flags, "rlm-samples", 1);
+	if (rlmSamples > 1 && analyst !== "dspy-rlm") throw new Error("--rlm-samples above 1 requires --analyst dspy-rlm");
+	const instructionsFile = flags.get("instructions-file")?.trim();
+	if (instructionsFile && analyst !== "dspy-rlm") throw new Error("--instructions-file requires --analyst dspy-rlm");
+	const instructionsOverride = instructionsFile ? readAnalystInstructionsOverride(instructionsFile) : void 0;
+	if (rlmSamples > 1 && dataset !== "codetracebench") throw new Error("--rlm-samples above 1 requires --dataset codetracebench; step-level consensus is defined on its block grammar");
 	return {
 		dataset,
 		analyst,
@@ -4652,12 +5064,17 @@ function parseCommandConfig(argv, env) {
 			maxOutputTokens: positiveFlag(flags, "max-output-tokens", 16384),
 			timeoutMs: positiveFlag(flags, "timeout-ms", 3e5),
 			maxCostUsdPerAnalysis: maxCostUsd,
-			...python ? { dspyRlm: { runner: { command: python } } } : {}
+			...instructionsOverride ? { instructionsOverride } : {},
+			dspyRlm: {
+				...python ? { runner: { command: python } } : {},
+				samples: rlmSamples
+			}
 		},
 		limit: positiveFlag(flags, "limit"),
 		seed: integerFlag(flags, "seed", 0),
 		concurrency: positiveFlag(flags, "concurrency", 1),
 		repetitions: positiveFlag(flags, "repetitions", 1),
+		rlmSamples,
 		maxCostUsd,
 		maxArtifactBytes: positiveFlag(flags, "max-artifact-bytes", DEFAULT_MAX_VERIFICATION_ARTIFACT_BYTES),
 		apiKeyEnv,
@@ -4703,6 +5120,8 @@ const KNOWN_FLAGS = /* @__PURE__ */ new Set([
 	"seed",
 	"concurrency",
 	"repetitions",
+	"rlm-samples",
+	"instructions-file",
 	"max-output-tokens",
 	"python",
 	"timeout-ms",
@@ -4826,6 +5245,6 @@ function shellQuote(value) {
 	return /^[a-zA-Z0-9_./:@%+=,-]+$/.test(value) ? value : `'${value.replaceAll("'", "'\\''")}'`;
 }
 //#endregion
-export { ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 as A, ANALYST_BENCHMARK_LOCAL_RECEIPT_FILE as B, publicBenchmarkSystemPrompt as C, parseVerificationOutcome as D, loadCodeTraceVerificationArtifacts as E, ANALYST_BENCHMARK_IMPLEMENTATION_FILES as F, summarizeAgentRxCalibration as G, ANALYST_BENCHMARK_OBSERVATIONS_FILE as H, ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 as I, agentRxBenchmarkCase as J, codeTraceBenchCase as K, analystBenchmarkDependencyLockDigest as L, ANALYST_BENCHMARK_EVIDENCE_IMPLEMENTATION_SHA256 as M, ANALYST_BENCHMARK_EVIDENCE_PACKAGE_VERSION as N, ANALYST_BENCHMARK_DEPENDENCY_LOCK_DIGEST_ALGORITHM as O, ANALYST_BENCHMARK_IMPLEMENTATION_DIGEST_ALGORITHM as P, normalizeBenchmarkLabel as Q, analystBenchmarkImplementationDigest as R, publicBenchmarkRlmInstructions as S, appendVerificationArtifactsToOtlp as T, AGENT_RX_UPSTREAM_REVISION as U, ANALYST_BENCHMARK_MANIFEST_FILE as V, renderAgentRxCalibrationMarkdown as W, normalizeAgentRxCategory as X, agentRxPredictionsToFindings as Y, roundAgentRxStep as Z, expandCodeTraceFailureBlocks as _, renderCodeTraceCalibrationMarkdown as a, MAX_INCORRECT_BLOCK_STEPS as b, createPublicBenchmarkRlmRunner as c, preparePublicAnalystBenchmark as d, publicBenchmarkDistributions as f, emptyPublicBenchmarkRunner as g, adaptPublicBenchmarkFindings as h, readAnalystBenchmarkArtifact as i, ANALYST_BENCHMARK_EVIDENCE_DEPENDENCY_LOCK_SHA256 as j, ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES as k, createPublicBenchmarkDirectRunner as l, selectPublicBenchmarkRows as m, runAnalystBenchmarkCommand as n, summarizeCodeTraceCalibration as o, publicBenchmarkSelectionReport as p, codeTracerPredictionsToFindings as q, renderAnalystBenchmarkMarkdown as r, compareAnalystRunners as s, ANALYST_BENCHMARK_HELP as t, loadPublicBenchmarkRows as u, CODE_TRACE_BENCH_ANALYST_PROMPT as v, DEFAULT_MAX_VERIFICATION_ARTIFACT_BYTES as w, publicBenchmarkProtocolSha256 as x, MAX_INCORRECT_BLOCKS as y, ANALYST_BENCHMARK_COST_LEDGER_FILE as z };
+export { normalizeAgentRxCategory as $, parseVerificationOutcome as A, analystBenchmarkDependencyLockDigest as B, MAX_INCORRECT_BLOCK_STEPS as C, DEFAULT_MAX_VERIFICATION_ARTIFACT_BYTES as D, publicBenchmarkSystemPrompt as E, ANALYST_BENCHMARK_EVIDENCE_IMPLEMENTATION_SHA256 as F, ANALYST_BENCHMARK_OBSERVATIONS_FILE as G, ANALYST_BENCHMARK_COST_LEDGER_FILE as H, ANALYST_BENCHMARK_EVIDENCE_PACKAGE_VERSION as I, summarizeAgentRxCalibration as J, AGENT_RX_UPSTREAM_REVISION as K, ANALYST_BENCHMARK_IMPLEMENTATION_DIGEST_ALGORITHM as L, ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES as M, ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 as N, appendVerificationArtifactsToOtlp as O, ANALYST_BENCHMARK_EVIDENCE_DEPENDENCY_LOCK_SHA256 as P, agentRxPredictionsToFindings as Q, ANALYST_BENCHMARK_IMPLEMENTATION_FILES as R, MAX_INCORRECT_BLOCKS as S, publicBenchmarkRlmInstructions as T, ANALYST_BENCHMARK_LOCAL_RECEIPT_FILE as U, analystBenchmarkImplementationDigest as V, ANALYST_BENCHMARK_MANIFEST_FILE as W, codeTracerPredictionsToFindings as X, codeTraceBenchCase as Y, agentRxBenchmarkCase as Z, compareAnalystRunners as _, preparePublicAnalystBenchmark as a, readAnalystInstructionsOverride as b, selectPublicBenchmarkRows as c, adaptPublicBenchmarkFindings as d, roundAgentRxStep as et, emptyPublicBenchmarkRunner as f, summarizeCodeTraceCalibration as g, renderCodeTraceCalibrationMarkdown as h, loadPublicBenchmarkRows as i, ANALYST_BENCHMARK_DEPENDENCY_LOCK_DIGEST_ALGORITHM as j, loadCodeTraceVerificationArtifacts as k, createPublicBenchmarkRlmRunner as l, readAnalystBenchmarkArtifact as m, runAnalystBenchmarkCommand as n, publicBenchmarkDistributions as o, expandCodeTraceFailureBlocks as p, renderAgentRxCalibrationMarkdown as q, renderAnalystBenchmarkMarkdown as r, publicBenchmarkSelectionReport as s, ANALYST_BENCHMARK_HELP as t, normalizeBenchmarkLabel as tt, createPublicBenchmarkDirectRunner as u, analystInstructionsOverrideFromText as v, publicBenchmarkProtocolSha256 as w, CODE_TRACE_BENCH_ANALYST_PROMPT as x, effectiveAnalystProtocolSha256 as y, ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 as z };
 
-//# sourceMappingURL=benchmark-command-Bo_hkhQf.js.map
+//# sourceMappingURL=benchmark-command-D8xXNVVh.js.map