@tangle-network/agent-eval 0.139.2 → 0.139.3

package/dist/{benchmark-command-DzUZJl8M.js → benchmark-command-xvi2liH7.js}

@@ -1172,7 +1172,7 @@ const ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES = Object.freeze([
 	"package.json",
 	"pnpm-lock.yaml"
 ]);
-const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = "1ca280998a9f3a416b40cadb9c14789ada490a727a4ec78491c92bb2022e6c95";
+const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = "11ea62baf2914dba71dd62e45ba4bb8e856b2cefdd5b1b73dc2f14b4b6e6d115";
 /** The published benchmark evidence was produced at this package version, by
 * the retired one-shot direct runner, before trace analysts moved to the
 * recursive DSPy RLM engine. Both evidence digests below are historical facts
@@ -1203,6 +1203,7 @@ const ANALYST_BENCHMARK_IMPLEMENTATION_FILES = Object.freeze([
 	"src/analyst/benchmark-public-data.ts",
 	"src/analyst/benchmark-public-errors.ts",
 	"src/analyst/benchmark-public-model.ts",
+	"src/analyst/benchmark-public-prompt.ts",
 	"src/analyst/benchmark-public-rlm.ts",
 	"src/analyst/benchmark-public-types.ts",
 	"src/analyst/benchmark-real-model.ts",
@@ -1265,7 +1266,7 @@ const ANALYST_BENCHMARK_IMPLEMENTATION_FILES = Object.freeze([
 	"src/trace/otlp-attributes.ts",
 	"src/trace/raw-provider-sink.ts"
 ]);
-const ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 = "5e47fc9d9f49c468d3ef06c5d552925e6a026f9e22a75056a9c8c5a2879744c0";
+const ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 = "26820de559ca069145eb9649283d2efba51a0a7b8d3662d4c378b9ecebfd4777";
 function analystBenchmarkImplementationDigest() {
 	return ANALYST_BENCHMARK_IMPLEMENTATION_SHA256;
 }
@@ -1332,19 +1333,12 @@ async function validateCodeTraceFindingEvidence(options) {
 	if (citations.length === 0) return;
 	for (const citation of citations) if (!citation.location || citation.location.traceId !== options.trajectoryId) throw new Error(`model finding '${citation.findingId}' cites non-case evidence '${citation.evidence.uri}'`);
 	const spanIds = [...new Set(citations.map((citation) => `step-${citation.location.step}`))];
-	const spans = /* @__PURE__ */ new Map();
-	for (let offset = 0; offset < spanIds.length; offset += TRACE_ANALYSIS_LIMITS.viewSpans) {
-		const requested = spanIds.slice(offset, offset + TRACE_ANALYSIS_LIMITS.viewSpans);
-		const result = await options.store.viewSpans({
-			trace_id: options.trajectoryId,
-			span_ids: requested
-		}, options.signal ? { signal: options.signal } : void 0);
-		if (result.missing_span_ids.length > 0 || result.omitted_span_ids.length > 0) {
-			const unavailable = [...result.missing_span_ids, ...result.omitted_span_ids];
-			throw new Error(`model finding evidence is unavailable in the case trace: ${unavailable.join(", ")}`);
-		}
-		for (const span of result.spans) spans.set(span.span_id, span);
-	}
+	const { spans, missing } = await fetchTraceSpans(options.store, {
+		trajectoryId: options.trajectoryId,
+		spanIds,
+		...options.signal ? { signal: options.signal } : {}
+	});
+	if (missing.length > 0) throw new Error(`model finding evidence is unavailable in the case trace: ${missing.join(", ")}`);
 	for (const citation of citations) {
 		const spanId = `step-${citation.location.step}`;
 		const span = spans.get(spanId);
@@ -1353,31 +1347,45 @@ async function validateCodeTraceFindingEvidence(options) {
 		assertExactActionExcerpt(citation.findingId, citation.evidence, spanId, span.attributes.content);
 	}
 }
+/**
+* Resolve assistant-step evidence for a trajectory.
+*
+* `steps` are claims the model made explicitly: an unresolvable one is a model
+* error and throws. `optionalSteps` are derived by the runner (a block's
+* interior, a block's consequence step), so an unresolvable one is simply
+* absent from the returned map and the caller decides what that means.
+*/
 async function resolveAssistantStepEvidence(options) {
-	const steps = [...new Set(options.steps)];
-	for (const step of steps) if (!Number.isSafeInteger(step) || step < 1) throw new TypeError(`assistant evidence step must be a positive safe integer: ${step}`);
-	const spanIds = steps.map((step) => `step-${step}`);
-	const spans = /* @__PURE__ */ new Map();
-	for (let offset = 0; offset < spanIds.length; offset += TRACE_ANALYSIS_LIMITS.viewSpans) {
-		const requested = spanIds.slice(offset, offset + TRACE_ANALYSIS_LIMITS.viewSpans);
-		const result = await options.store.viewSpans({
-			trace_id: options.trajectoryId,
-			span_ids: requested
-		}, options.signal ? { signal: options.signal } : void 0);
-		if (result.missing_span_ids.length > 0 || result.omitted_span_ids.length > 0) {
-			const unavailable = [...result.missing_span_ids, ...result.omitted_span_ids];
-			throw new Error(`model selected unavailable assistant steps: ${unavailable.join(", ")}`);
-		}
-		for (const span of result.spans) spans.set(span.span_id, span);
-	}
+	const required = [...new Set(options.steps)];
+	const optional = [...new Set(options.optionalSteps ?? [])].filter((step) => !required.includes(step));
+	for (const step of [...required, ...optional]) if (!Number.isSafeInteger(step) || step < 1) throw new TypeError(`assistant evidence step must be a positive safe integer: ${step}`);
+	const steps = [...required, ...optional];
+	if (steps.length === 0) return /* @__PURE__ */ new Map();
+	const { spans, missing } = await fetchTraceSpans(options.store, {
+		trajectoryId: options.trajectoryId,
+		spanIds: steps.map((step) => `step-${step}`),
+		...options.signal ? { signal: options.signal } : {}
+	});
+	const missingRequired = missing.filter((spanId) => required.some((step) => `step-${step}` === spanId));
+	if (missingRequired.length > 0) throw new Error(`model selected unavailable assistant steps: ${missingRequired.join(", ")}`);
 	const evidence = /* @__PURE__ */ new Map();
 	for (const step of steps) {
 		const spanId = `step-${step}`;
+		const optionalStep = optional.includes(step);
 		const span = spans.get(spanId);
-		if (!span) throw new Error(`model selected missing assistant step '${spanId}'`);
-		if (span.kind !== "LLM") throw new Error(`model selected '${spanId}', which is ${span.kind}, not an assistant LLM span`);
+		if (!span) {
+			if (optionalStep) continue;
+			throw new Error(`model selected missing assistant step '${spanId}'`);
+		}
+		if (span.kind !== "LLM") {
+			if (optionalStep) continue;
+			throw new Error(`model selected '${spanId}', which is ${span.kind}, not an assistant LLM span`);
+		}
 		const content = span.attributes.content;
-		if (typeof content !== "string" || content.trim().length === 0) throw new Error(`model selected '${spanId}' without action content`);
+		if (typeof content !== "string" || content.trim().length === 0) {
+			if (optionalStep) continue;
+			throw new Error(`model selected '${spanId}' without action content`);
+		}
 		evidence.set(step, {
 			kind: "span",
 			uri: codeTraceStepEvidenceUri(options.trajectoryId, step),
@@ -1386,6 +1394,38 @@ async function resolveAssistantStepEvidence(options) {
 	}
 	return evidence;
 }
+/**
+* Read spans by id, paging over the store's byte-budget omissions.
+*
+* `omitted_span_ids` names spans that exist but did not fit the response
+* ceiling; the store guarantees at least one span lands per call, so
+* re-requesting exactly the omitted ids terminates. Only `missing_span_ids`
+* describes a span the trace does not contain.
+*/
+async function fetchTraceSpans(store, options) {
+	const spans = /* @__PURE__ */ new Map();
+	const missing = [];
+	const unique = [...new Set(options.spanIds)];
+	const context = options.signal ? { signal: options.signal } : void 0;
+	for (let offset = 0; offset < unique.length; offset += TRACE_ANALYSIS_LIMITS.viewSpans) {
+		let pending = unique.slice(offset, offset + TRACE_ANALYSIS_LIMITS.viewSpans);
+		while (pending.length > 0) {
+			const result = await store.viewSpans({
+				trace_id: options.trajectoryId,
+				span_ids: pending
+			}, context);
+			for (const span of result.spans) spans.set(span.span_id, span);
+			missing.push(...result.missing_span_ids);
+			const omitted = result.omitted_span_ids.filter((spanId) => !spans.has(spanId));
+			if (omitted.length >= pending.length) throw new Error(`trace '${options.trajectoryId}' cannot project spans within the store response budget: ${omitted.join(", ")}`);
+			pending = omitted;
+		}
+	}
+	return {
+		spans,
+		missing
+	};
+}
 function scanValue(value, traceId, path, depth, serializedDepth) {
 	if (depth > MAX_LABEL_SCAN_DEPTH) throw new Error(`trace '${traceId}' exceeds benchmark label scan depth at ${path}`);
 	if (Array.isArray(value)) return 1 + value.reduce((count, entry, index) => count + scanValue(entry, traceId, `${path}[${index}]`, depth + 1, serializedDepth), 0);
@@ -1445,119 +1485,6 @@ function assertExactActionExcerpt(findingId, evidence, spanId, content) {
 	if (!content.includes(excerpt)) throw new Error(`model finding '${findingId}' excerpt is not present in '${spanId}' action content`);
 }
 //#endregion
-//#region src/analyst/benchmark-public-adapters.ts
-function emptyPublicBenchmarkRunner() {
-	return {
-		id: "empty",
-		analyze() {
-			return {
-				findings: [],
-				usage: {
-					calls: 0,
-					tokens: {
-						input: 0,
-						output: 0
-					},
-					cost: {
-						kind: "observed",
-						usd: 0
-					}
-				},
-				metadata: { baseline: "emit-no-findings" }
-			};
-		}
-	};
-}
-function adaptPublicBenchmarkFindings(dataset, trajectoryId, findings, analystId) {
-	return dataset === "agentrx" ? adaptAgentRxFindings(trajectoryId, findings, analystId) : adaptCodeTraceFindings(trajectoryId, findings, analystId);
-}
-function adaptAgentRxFindings(trajectoryId, findings, analystId) {
-	if (findings.length === 0) return [];
-	if (findings.length !== 1) throw new Error(`AgentRx model analyst must emit zero or one root cause, received ${findings.length}`);
-	const source = findings[0];
-	if (!source.subject) throw new Error("AgentRx model analyst finding is missing its failure-category subject");
-	const steps = exactFindingSteps(trajectoryId, source);
-	if (steps.length !== 1) throw new Error(`AgentRx model analyst must cite exactly one root-cause step, received ${steps.length}`);
-	const [adapted] = agentRxPredictionsToFindings(trajectoryId, [{
-		failure_case: source.subject,
-		step_number: steps[0],
-		description: source.rationale ?? source.claim
-	}], {
-		analystId,
-		producedAt: source.produced_at,
-		confidence: source.confidence
-	});
-	if (!adapted) throw new Error("AgentRx output adapter produced no root-cause finding");
-	return [{
-		...adapted,
-		metadata: {
-			...adapted.metadata,
-			sourceFindingId: source.finding_id
-		}
-	}];
-}
-function adaptCodeTraceFindings(trajectoryId, findings, analystId) {
-	const clean = findings.filter((finding) => finding.subject === "clean");
-	if (clean.length > 0) {
-		if (findings.length !== 1) throw new Error("CodeTraceBench model analyst mixed a clean verdict with incorrect steps");
-		exactFindingSteps(trajectoryId, clean[0]);
-		return [];
-	}
-	const byStep = /* @__PURE__ */ new Map();
-	for (const source of findings) {
-		const steps = exactFindingSteps(trajectoryId, source);
-		for (const step of steps) {
-			if (byStep.has(step)) continue;
-			byStep.set(step, makeFinding({
-				analyst_id: analystId,
-				area: "incorrect",
-				subject: `incorrect-step-${step}`,
-				claim: `Step ${step} is incorrect. ${source.claim}`,
-				rationale: source.rationale,
-				severity: source.severity,
-				confidence: source.confidence,
-				evidence_refs: [{
-					kind: "span",
-					uri: codeTraceStepEvidenceUri(trajectoryId, step),
-					excerpt: source.evidence_refs.find((evidence) => evidence.uri === codeTraceStepEvidenceUri(trajectoryId, step))?.excerpt
-				}],
-				recommended_action: source.recommended_action,
-				metadata: { sourceFindingId: source.finding_id },
-				produced_at: source.produced_at,
-				id_basis: `incorrect-step-${step}`
-			}));
-		}
-	}
-	return [...byStep].sort(([left], [right]) => left - right).map(([, finding]) => finding);
-}
-function exactFindingSteps(trajectoryId, finding) {
-	if (finding.evidence_refs.length === 0) throw new Error(`model finding '${finding.finding_id}' has no step evidence`);
-	const steps = finding.evidence_refs.map((evidence) => {
-		const parsed = codeTraceStepFromEvidence(evidence.uri);
-		if (!parsed || parsed.traceId !== trajectoryId) throw new Error(`model finding '${finding.finding_id}' cites non-case evidence '${evidence.uri}'`);
-		return parsed.step;
-	});
-	return [...new Set(steps)];
-}
-//#endregion
-//#region src/analyst/benchmark-public-types.ts
-function requiredString(value, field) {
-	const trimmed = value.trim();
-	if (!trimmed) throw new TypeError(`${field} must be a non-empty string`);
-	return trimmed;
-}
-function positiveSafeInteger(value, field) {
-	if (!Number.isSafeInteger(value) || value < 1) throw new RangeError(`${field} must be a positive safe integer`);
-	return value;
-}
-function safeInteger(value, field) {
-	if (!Number.isSafeInteger(value)) throw new RangeError(`${field} must be a safe integer`);
-	return value;
-}
-function isRecord(value) {
-	return typeof value === "object" && value !== null && !Array.isArray(value);
-}
-//#endregion
 //#region src/analyst/benchmark-verification-outcome.ts
 const MAX_REPORTED_CHECKS = 20;
 const SWE_MULTI_NO_TEST_RESULTS = "After applying the fix patch, no test results were captured when executing the test command.";
@@ -2157,6 +2084,376 @@ function isNodeError$1(error, code) {
 	return error instanceof Error && "code" in error && error.code === code;
 }
 //#endregion
+//#region src/analyst/benchmark-public-prompt.ts
+/** Widest contiguous failure block a model may report. The published corpus's
+* widest labeled block is 8 steps and its widest stage span is 9, so this bound
+* never binds honest enumeration; it caps how far one over-wide block can push
+* unlabeled steps into the precision denominator. */
+const MAX_INCORRECT_BLOCK_STEPS = 12;
+/** Most blocks a model may report for one trajectory. The published corpus's
+* densest case carries 4 disjoint labeled blocks. Together with the per-block
+* cap this bounds one case at 192 predicted steps without a second ceiling. */
+const MAX_INCORRECT_BLOCKS = 16;
+const TRACE_PROJECTION_ATTRIBUTE_BYTE_CAPS = [
+	4096,
+	2048,
+	1024,
+	512,
+	256,
+	128,
+	64
+];
+const CODE_TRACE_BENCH_ANALYST_PROMPT = `Analyze exactly one coding-agent trajectory and its attached final verification.
+Your task is the CodeTraceBench incorrect-step task: identify every wrong state-changing action, bad hypothesis that drives an action, and regression.
+An incorrect step remains incorrect when the agent later recovers or the final verification passes; a trajectory that ends in success still contains every mistake the agent made along the way.
+Incorrect steps occur in contiguous failure blocks: one mistake plus every consecutive following step that commits to, compounds, or acts on it.
+Report each failure block as exactly one finding whose first_step is the block's first incorrect step and whose last_step is its last, covering every consecutive step between them.
+Set first_step to the first step that commits the mistake, not the step that planned it and not a later step that repeats it.
+Extend last_step one step at a time, and only while the next step independently satisfies the incorrect-step definition on its own action and its own following observation.
+Stop at the first step where the agent detects the problem, inspects it, or begins repairing it: a diagnostic probe, a test run that exposes the defect, or a repair action ends the block and is never inside it.
+A one-step block is a complete and correct answer.
+Every step inside a block is scored on its own: naming a correct step costs exactly as much as missing an incorrect one, and naming only the first step of a longer block forfeits every unnamed step.
+Report blocks separated by at least one correct step as separate findings, and never let two blocks overlap.
+Inspect the complete supplied trace data.
+Use the final-verification outcome as evidence about the final state, not as a rule for whether earlier steps were incorrect.
+For each candidate block, inspect every assistant action in it and its following observation.
+Admit a block only when you can point at the specific evidence it produced: name as consequence_step the step number whose action or observation shows the damage — a failing command, a wrong file state, a repeated failure, or rework the agent had to do because of this block. That step is the block's own last step when its observation already shows the damage, and a later step otherwise.
+When you cannot name that later step number from the trace you were given, drop the block; a plausible story about why a step looks wrong is not evidence that it was.
+Judge that consequence from the trajectory itself: a passing final verification is not evidence that a block caused nothing, and a failing final verification is not evidence that any particular block caused it.
+For every block, decide whether the agent escaped the failure.
+Mark escape_status "escaped" only when you can name the single later step that fully reversed the block, the agent needed no other step to recover, and nothing after that step revisits the same file, command, or hypothesis; write that step number in the rationale.
+Mark escape_status "unescaped" in every other case, including whenever you are unsure.
+A passing final verification never makes a block escaped: the agent may have made the mistake and repaired it over several steps, and those steps are still incorrect.
+Label a failed command when the assistant caused it through a wrong action or unsupported hypothesis.
+Label the later corrective action only when that action is itself wrong.
+Do not label a diagnostic probe merely because it exposes an earlier defect.
+Do not label a redundant but correct read or search; CodeTraceBench scores unuseful steps separately, and this run scores incorrect steps only.
+Do not label a step solely because final verification failed.
+When final verification is unavailable, use only directly observed trajectory evidence.
+Every step in a reported block MUST be the positive integer n from an existing assistant LLM span named step-<n>.
+Never select an EVALUATOR, TOOL, CHAIN, final-verification, benchmark-verification, or message-<n> span.
+Before emitting a finding, inspect every covered span's attributes.content and describe only the actions shown there.
+Report at most 16 blocks and at most 12 steps in one block; when more candidates than that exist, report the ones you can support with the clearest downstream evidence.
+When the trajectory has no incorrect steps, return an empty findings array.`;
+const AGENT_RX_PROMPT = `Analyze exactly one failed agent trajectory.
+Find the first unrecoverable critical failure, not every later symptom.
+Inspect the complete supplied trace data.
+Emit zero findings only when the trace does not contain enough evidence.
+Otherwise emit exactly one finding.
+Its category MUST be exactly one of:
+instruction-plan-adherence-failure
+invention-of-new-information
+invalid-invocation
+misinterpretation-of-tool-output-handoff-failure
+intent-plan-misalignment
+underspecified-user-intent
+intent-not-supported
+guardrails-triggered
+system-failure
+inconclusive
+Its step is the positive integer n from the first unrecoverable assistant span named step-<n>.`;
+const AGENT_RX_JSON_CONTRACT = `Each finding must contain only:
+- "step": a positive integer matching an existing assistant LLM span named step-<n>
+- "severity": "critical", "high", "medium", "low", or "info"
+- "claim": one sentence
+- "confidence": a number from 0 through 1
+- optional "rationale" and "recommended_action" strings
+- "category": one allowed failure category listed above`;
+const CODE_TRACE_JSON_CONTRACT = `Each finding is one contiguous failure block and must contain only:
+- "first_step": a positive integer, the block's first incorrect step, matching an existing assistant LLM span named step-<n>
+- "last_step": a positive integer >= first_step, the block's last incorrect step; every step from first_step through last_step must be an existing assistant LLM span, and a block spans at most 12 steps
+- "consequence_step": a positive integer >= first_step, the step whose action or following observation shows the damage this block caused; it may sit inside the block when the damage is already visible there
+- "escape_status": "escaped" only when one single later step fully reversed the block and nothing afterwards revisits it, "unescaped" otherwise and whenever you are unsure
+- "severity": "critical", "high", "medium", "low", or "info"
+- "claim": one sentence describing the block's failure
+- "confidence": a number from 0 through 1
+- optional "rationale" and "recommended_action" strings`;
+const AGENT_RX_RLM_CONTRACT = `Use the trace tools to inspect the action and its following observation.
+Emit exactly one finding whose subject is exactly one of the allowed failure categories.
+Cite exactly one assistant span named step-<n> as trace://<URL-encoded-trace-id>/span/step-<n>.
+The excerpt must quote the assistant action exactly.`;
+const CODE_TRACE_RLM_CONTRACT = `Use the trace tools rather than asking for the whole trajectory in the prompt.
+Keep retrieved trace objects in Python variables.
+Never print an entire trace, full source file, or more than 12000 characters in one iteration.
+Build a compact table of assistant step ids, actions, following observations, and final verification.
+Inspect suspicious steps with viewSpans or searchSpan instead of repeatedly printing the table.
+This runner emits no JSON fields, so the block is encoded in the finding's subject.
+Emit exactly one finding per contiguous failure block.
+Set the finding's subject to incorrect-steps-<first_step>-<last_step>-<escape_status>-consequence-<consequence_step>, using the same four values the task defines; for a block covering only step 7 that the agent never escaped and whose damage shows at step 9, the subject is incorrect-steps-7-7-unescaped-consequence-9.
+The runner expands the block to one scored step per member and builds every scored citation itself.
+Cite the block's first step and its last step as trace://<URL-encoded-trace-id>/span/step-<n>, each excerpt an exact quote from that step's own action content.
+Give the rationale as the concrete downstream evidence visible at the consequence step.
+Submit as soon as every candidate failure block has a supported verdict.
+Return no finding for a clean trajectory.`;
+/** One-shot JSON transport prompt for the direct runner. */
+function publicBenchmarkSystemPrompt(dataset) {
+	const fieldContract = dataset === "agentrx" ? AGENT_RX_JSON_CONTRACT : CODE_TRACE_JSON_CONTRACT;
+	return `${publicBenchmarkTaskPrompt(dataset)}
+
+${fieldContract}
+
+Return exactly one JSON object with:
+- "report": a concise evidence-based explanation, at most 4000 characters
+- "findings": the strict finding array
+Use an empty findings array when the trace does not support a finding.
+Do not return a bare array, markdown, trace URIs, copied excerpts, or fields not listed above.
+The runner constructs exact trace URIs and action previews from each selected step.`;
+}
+/** Tool-loop prompt for the recursive runner. Same task, subject-encoded block. */
+function publicBenchmarkRlmInstructions(dataset) {
+	const outputContract = dataset === "agentrx" ? AGENT_RX_RLM_CONTRACT : CODE_TRACE_RLM_CONTRACT;
+	return `${publicBenchmarkTaskPrompt(dataset)}
+${outputContract}`;
+}
+function publicBenchmarkTaskPrompt(dataset) {
+	return dataset === "agentrx" ? AGENT_RX_PROMPT : CODE_TRACE_BENCH_ANALYST_PROMPT;
+}
+/** Digest of every prompt a runner can send plus the shared transport limits.
+* Both runner contracts are hashed so an edit to either one changes the digest
+* a run records, whichever runner executed. */
+function publicBenchmarkProtocolSha256(dataset) {
+	return sha256Digest(JSON.stringify({
+		dataset,
+		systemPrompt: publicBenchmarkSystemPrompt(dataset),
+		rlmInstructions: publicBenchmarkRlmInstructions(dataset),
+		transport: {
+			attempts: 1,
+			jsonMode: true,
+			thinking: "disabled"
+		},
+		blockLimits: dataset === "agentrx" ? null : {
+			maxBlocks: 16,
+			maxBlockSteps: 12
+		},
+		traceProjectionAttributeByteCaps: TRACE_PROJECTION_ATTRIBUTE_BYTE_CAPS,
+		evidence: {
+			location: "model-selected-positive-integer-assistant-step",
+			uri: "deterministic-trace-uri",
+			excerpt: `exact-action-prefix-512`
+		}
+	}));
+}
+//#endregion
+//#region src/analyst/benchmark-public-adapters.ts
+function emptyPublicBenchmarkRunner() {
+	return {
+		id: "empty",
+		analyze() {
+			return {
+				findings: [],
+				usage: {
+					calls: 0,
+					tokens: {
+						input: 0,
+						output: 0
+					},
+					cost: {
+						kind: "observed",
+						usd: 0
+					}
+				},
+				metadata: { baseline: "emit-no-findings" }
+			};
+		}
+	};
+}
+async function adaptPublicBenchmarkFindings(options) {
+	if (options.dataset === "agentrx") return {
+		findings: adaptAgentRxFindings(options.trajectoryId, options.findings, options.analystId),
+		diagnostics: void 0
+	};
+	return adaptCodeTraceFindings(options.trajectoryId, options.findings, options.analystId, options.store, options.signal);
+}
+function adaptAgentRxFindings(trajectoryId, findings, analystId) {
+	if (findings.length === 0) return [];
+	if (findings.length !== 1) throw new Error(`AgentRx model analyst must emit zero or one root cause, received ${findings.length}`);
+	const source = findings[0];
+	if (!source.subject) throw new Error("AgentRx model analyst finding is missing its failure-category subject");
+	const steps = exactFindingSteps(trajectoryId, source);
+	if (steps.length !== 1) throw new Error(`AgentRx model analyst must cite exactly one root-cause step, received ${steps.length}`);
+	const [adapted] = agentRxPredictionsToFindings(trajectoryId, [{
+		failure_case: source.subject,
+		step_number: steps[0],
+		description: source.rationale ?? source.claim
+	}], {
+		analystId,
+		producedAt: source.produced_at,
+		confidence: source.confidence
+	});
+	if (!adapted) throw new Error("AgentRx output adapter produced no root-cause finding");
+	return [{
+		...adapted,
+		metadata: {
+			...adapted.metadata,
+			sourceFindingId: source.finding_id
+		}
+	}];
+}
+const CODE_TRACE_BLOCK_SUBJECT = /^incorrect-steps-(\d+)-(\d+)-(escaped|unescaped)-consequence-(\d+)$/;
+async function adaptCodeTraceFindings(trajectoryId, findings, analystId, store, signal) {
+	const clean = findings.filter((finding) => finding.subject === "clean");
+	if (clean.length > 0) {
+		if (findings.length !== 1) throw new Error("CodeTraceBench model analyst mixed a clean verdict with incorrect steps");
+		exactFindingSteps(trajectoryId, clean[0]);
+		return {
+			findings: [],
+			diagnostics: emptyCodeTraceBlockDiagnostics()
+		};
+	}
+	await validateCodeTraceFindingEvidence({
+		trajectoryId,
+		findings,
+		store,
+		...signal ? { signal } : {}
+	});
+	return expandCodeTraceFailureBlocks({
+		trajectoryId,
+		blocks: findings.map((source) => codeTraceBlockFromFinding(trajectoryId, source)),
+		store,
+		analystId,
+		...findings[0] ? { producedAt: findings[0].produced_at } : {},
+		...signal ? { signal } : {}
+	});
+}
+function codeTraceBlockFromFinding(trajectoryId, source) {
+	const parsed = CODE_TRACE_BLOCK_SUBJECT.exec(source.subject ?? "");
+	if (!parsed) throw new Error(`CodeTraceBench model finding '${source.finding_id}' must set subject to incorrect-steps-<first>-<last>-<escaped|unescaped>-consequence-<step>, received '${source.subject ?? ""}'`);
+	const firstStep = Number(parsed[1]);
+	const lastStep = Number(parsed[2]);
+	const consequenceStep = Number(parsed[4]);
+	const cited = exactFindingSteps(trajectoryId, source);
+	for (const step of cited) if (step < firstStep || step > lastStep) throw new Error(`CodeTraceBench model finding '${source.finding_id}' cites step ${step} outside its block ${firstStep}-${lastStep}`);
+	return {
+		firstStep,
+		lastStep,
+		consequenceStep,
+		escapeStatus: parsed[3],
+		severity: source.severity,
+		claim: source.claim,
+		confidence: source.confidence,
+		...source.rationale === void 0 ? {} : { rationale: source.rationale },
+		...source.recommended_action === void 0 ? {} : { recommendedAction: source.recommended_action },
+		metadata: { sourceFindingId: source.finding_id }
+	};
+}
+/**
+* Expand contiguous failure blocks into one scored finding per member step.
+*
+* The official scorer matches on area plus the exact step evidence URI, so
+* blocks never reach it: every runner reports blocks, and this function turns
+* them into the per-step findings the benchmark defines.
+*/
+async function expandCodeTraceFailureBlocks(options) {
+	const diagnostics = emptyCodeTraceBlockDiagnostics();
+	diagnostics.reportedBlocks = options.blocks.length;
+	if (options.blocks.length === 0) return {
+		findings: [],
+		diagnostics
+	};
+	assertCodeTraceBlockShape(options.blocks);
+	const boundarySteps = options.blocks.flatMap((block) => [block.firstStep, block.lastStep]);
+	const derivedSteps = options.blocks.flatMap((block) => [block.consequenceStep, ...interiorSteps(block)]);
+	const evidenceByStep = await resolveAssistantStepEvidence({
+		trajectoryId: options.trajectoryId,
+		steps: boundarySteps,
+		optionalSteps: derivedSteps,
+		store: options.store,
+		...options.signal ? { signal: options.signal } : {}
+	});
+	const byStep = /* @__PURE__ */ new Map();
+	for (const block of options.blocks) {
+		if (!evidenceByStep.has(block.consequenceStep)) {
+			diagnostics.blocksWithoutConsequenceEvidence.push(block);
+			continue;
+		}
+		if (block.escapeStatus === "escaped") diagnostics.escapedBlocks += 1;
+		for (let step = block.firstStep; step <= block.lastStep; step += 1) {
+			if (!evidenceByStep.has(step)) {
+				diagnostics.unresolvedBlockInteriorSteps.push(step);
+				continue;
+			}
+			if (byStep.has(step)) {
+				diagnostics.overlappingBlockSteps.push(step);
+				continue;
+			}
+			byStep.set(step, block);
+		}
+	}
+	return {
+		findings: [...byStep].sort(([left], [right]) => left - right).map(([step, block]) => makeFinding({
+			analyst_id: options.analystId,
+			area: "incorrect",
+			subject: `incorrect-step-${step}`,
+			claim: `Step ${step} is incorrect. ${block.claim}`,
+			rationale: block.rationale,
+			severity: block.severity,
+			confidence: block.confidence,
+			evidence_refs: [evidenceByStep.get(step)],
+			recommended_action: block.recommendedAction,
+			metadata: {
+				...block.metadata,
+				block_first_step: block.firstStep,
+				block_last_step: block.lastStep,
+				block_consequence_step: block.consequenceStep,
+				escape_status: block.escapeStatus
+			},
+			...options.producedAt === void 0 ? {} : { produced_at: options.producedAt },
+			id_basis: `incorrect-step-${step}`
+		})),
+		diagnostics
+	};
+}
+function assertCodeTraceBlockShape(blocks) {
+	if (blocks.length > 16) throw new Error(`model reported ${blocks.length} failure blocks; the maximum is 16`);
+	for (const block of blocks) {
+		if (block.lastStep < block.firstStep) throw new Error(`failure block last_step ${block.lastStep} precedes first_step ${block.firstStep}`);
+		const length = block.lastStep - block.firstStep + 1;
+		if (length > 12) throw new Error(`failure block spans ${length} steps; the maximum is 12`);
+		if (block.consequenceStep < block.firstStep) throw new Error(`failure block consequence_step ${block.consequenceStep} precedes first_step ${block.firstStep}`);
+	}
+}
+function interiorSteps(block) {
+	const steps = [];
+	for (let step = block.firstStep + 1; step < block.lastStep; step += 1) steps.push(step);
+	return steps;
+}
+function emptyCodeTraceBlockDiagnostics() {
+	return {
+		reportedBlocks: 0,
+		escapedBlocks: 0,
+		blocksWithoutConsequenceEvidence: [],
+		unresolvedBlockInteriorSteps: [],
+		overlappingBlockSteps: []
+	};
+}
+function exactFindingSteps(trajectoryId, finding) {
+	if (finding.evidence_refs.length === 0) throw new Error(`model finding '${finding.finding_id}' has no step evidence`);
+	const steps = finding.evidence_refs.map((evidence) => {
+		const parsed = codeTraceStepFromEvidence(evidence.uri);
+		if (!parsed || parsed.traceId !== trajectoryId) throw new Error(`model finding '${finding.finding_id}' cites non-case evidence '${evidence.uri}'`);
+		return parsed.step;
+	});
+	return [...new Set(steps)];
+}
+//#endregion
+//#region src/analyst/benchmark-public-types.ts
+function requiredString(value, field) {
+	const trimmed = value.trim();
+	if (!trimmed) throw new TypeError(`${field} must be a non-empty string`);
+	return trimmed;
+}
+function positiveSafeInteger(value, field) {
+	if (!Number.isSafeInteger(value) || value < 1) throw new RangeError(`${field} must be a positive safe integer`);
+	return value;
+}
+function safeInteger(value, field) {
+	if (!Number.isSafeInteger(value)) throw new RangeError(`${field} must be a safe integer`);
+	return value;
+}
+function isRecord(value) {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+//#endregion
 //#region src/analyst/benchmark-public-data.ts
 const DEFAULT_MAX_PUBLIC_BENCHMARK_LABEL_BYTES = 256 * 1024 * 1024;
 const INPUT_OPEN_FLAGS = constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0);
@@ -2655,15 +2952,6 @@ function fileContext() {
 }
 //#endregion
 //#region src/analyst/benchmark-public-model.ts
-const TRACE_PROJECTION_ATTRIBUTE_BYTE_CAPS = [
-	4096,
-	2048,
-	1024,
-	512,
-	256,
-	128,
-	64
-];
 /** One-shot JSON baseline. This is not a recursive trace analyst. */
 function createPublicBenchmarkDirectRunner(dataset, config) {
 	const model = requiredString(config.model, "model");
@@ -2677,7 +2965,7 @@ function createPublicBenchmarkDirectRunner(dataset, config) {
 		responseCacheDir: requiredString(config.durability.responseCacheDir, "durability.responseCacheDir")
 	} : void 0;
 	const actor = dataset === "agentrx" ? "agentrx-root-cause-localizer" : "codetracebench-step-localizer";
-	const outputAdapter = dataset === "agentrx" ? "agentrx-taxonomy-and-root-step" : "codetracebench-incorrect-step";
+	const outputAdapter = dataset === "agentrx" ? "agentrx-taxonomy-and-root-step" : "codetracebench-incorrect-block";
 	const llmOptions = {
 		baseUrl,
 		apiKey,
@@ -2697,6 +2985,7 @@ function createPublicBenchmarkDirectRunner(dataset, config) {
 				benchmarkRepetition: String(context.repetition)
 			};
 			let rawPredictions = [];
+			let rejectedBlocks = [];
 			let modelFindings = [];
 			let providerModel = model;
 			let producedAt;
@@ -2748,6 +3037,7 @@ function createPublicBenchmarkDirectRunner(dataset, config) {
 					};
 					const response = parsePublicBenchmarkModelResponse(dataset, cached.response);
 					rawPredictions = response.findings;
+					rejectedBlocks = response.rejectedBlocks;
 					providerModel = cached.metadata.providerModel;
 					producedAt = cached.metadata.producedAt;
 					modelMetadata = {
@@ -2784,7 +3074,7 @@ function createPublicBenchmarkDirectRunner(dataset, config) {
 									...cacheIdentity,
 									callId: providerCallId,
 									status: "succeeded",
-									response,
+									response: completed.value,
 									metadata: {
 										providerModel: completed.result.model,
 										providerDurationMs: completed.result.durationMs,
@@ -2816,6 +3106,7 @@ function createPublicBenchmarkDirectRunner(dataset, config) {
 					if (!paid.succeeded) throw paid.error;
 					const response = paid.value.response;
 					rawPredictions = response.findings;
+					rejectedBlocks = response.rejectedBlocks;
 					providerModel = paid.value.result.model;
 					producedAt = paid.value.producedAt;
 					modelMetadata = {
@@ -2828,7 +3119,7 @@ function createPublicBenchmarkDirectRunner(dataset, config) {
 						cost: costReceiptMetadata(paid.receipt)
 					};
 				}
-				modelFindings = await publicBenchmarkPredictionsToFindings({
+				const converted = await publicBenchmarkPredictionsToFindings({
 					dataset,
 					trajectoryId,
 					predictions: rawPredictions,
@@ -2838,6 +3129,14 @@ function createPublicBenchmarkDirectRunner(dataset, config) {
 					producedAt: requiredString(producedAt ?? "", "finding producedAt"),
 					...context.signal ? { signal: context.signal } : {}
 				});
+				modelFindings = converted.findings;
+				if (converted.diagnostics) modelMetadata = {
+					...modelMetadata,
+					blockDiagnostics: {
+						...converted.diagnostics,
+						rejectedBlocks
+					}
+				};
 				if (dataset === "codetracebench") await validateCodeTraceFindingEvidence({
 					trajectoryId,
 					findings: modelFindings,
@@ -2935,7 +3234,7 @@ const ModelSeveritySchema = z.enum([
 	"low",
 	"info"
 ]);
-const PublicBenchmarkPredictionSchema = z.object({
+const AgentRxPredictionSchema = z.object({
 	step: z.number().int().positive(),
 	severity: ModelSeveritySchema,
 	claim: z.string().min(1),
@@ -2943,6 +3242,34 @@ const PublicBenchmarkPredictionSchema = z.object({
 	rationale: z.string().min(1).optional(),
 	recommended_action: z.string().min(1).optional()
 }).strict();
+const CodeTraceBlockPredictionSchema = z.object({
+	first_step: z.number().int().positive(),
+	last_step: z.number().int().positive(),
+	consequence_step: z.number().int().positive(),
+	escape_status: z.enum(["escaped", "unescaped"]),
+	severity: ModelSeveritySchema,
+	claim: z.string().min(1),
+	confidence: z.number().min(0).max(1),
+	rationale: z.string().min(1).optional(),
+	recommended_action: z.string().min(1).optional()
+}).strict().superRefine((block, ctx) => {
+	if (block.last_step < block.first_step) {
+		ctx.addIssue({
+			code: z.ZodIssueCode.custom,
+			message: `failure block last_step ${block.last_step} precedes first_step ${block.first_step}`
+		});
+		return;
+	}
+	const length = block.last_step - block.first_step + 1;
+	if (length > 12) ctx.addIssue({
+		code: z.ZodIssueCode.custom,
+		message: `failure block spans ${length} steps; the maximum is 12`
+	});
+	if (block.consequence_step < block.first_step) ctx.addIssue({
+		code: z.ZodIssueCode.custom,
+		message: `failure block consequence_step ${block.consequence_step} precedes first_step ${block.first_step}`
+	});
+});
 const AgentRxCategorySchema = z.enum([
 	"instruction-plan-adherence-failure",
 	"invention-of-new-information",
@@ -2955,28 +3282,52 @@ const AgentRxCategorySchema = z.enum([
 	"system-failure",
 	"inconclusive"
 ]);
-const CodeTraceModelResponseSchema = z.object({
+const CodeTraceModelResponseEnvelopeSchema = z.object({
 	report: z.string().min(1).max(4e3),
-	findings: z.array(PublicBenchmarkPredictionSchema).max(200)
+	findings: z.array(z.unknown()).max(16)
 }).strict();
 const AgentRxModelResponseSchema = z.object({
 	report: z.string().min(1).max(4e3),
-	findings: z.array(PublicBenchmarkPredictionSchema.extend({ category: AgentRxCategorySchema }).strict()).max(1)
+	findings: z.array(AgentRxPredictionSchema.extend({ category: AgentRxCategorySchema }).strict()).max(1)
 }).strict();
 function parsePublicBenchmarkModelResponse(dataset, value) {
-	return dataset === "agentrx" ? AgentRxModelResponseSchema.parse(value) : CodeTraceModelResponseSchema.parse(value);
+	if (dataset === "agentrx") return {
+		...AgentRxModelResponseSchema.parse(value),
+		rejectedBlocks: []
+	};
+	const envelope = CodeTraceModelResponseEnvelopeSchema.parse(value);
+	const findings = [];
+	const rejectedBlocks = [];
+	for (const [index, block] of envelope.findings.entries()) {
+		const parsed = CodeTraceBlockPredictionSchema.safeParse(block);
+		if (parsed.success) {
+			findings.push(parsed.data);
+			continue;
+		}
+		rejectedBlocks.push(`block ${index}: ${parsed.error.issues.map((issue) => `${issue.path.join(".") || "<root>"} ${issue.message}`).join("; ")}`);
+	}
+	if (findings.length === 0 && envelope.findings.length > 0) throw new ValidationError(`every reported failure block was malformed: ${rejectedBlocks.join(" | ")}`);
+	return {
+		report: envelope.report,
+		findings,
+		rejectedBlocks
+	};
 }
 async function publicBenchmarkPredictionsToFindings(options) {
-	if (options.predictions.length === 0) return [];
-	const evidenceByStep = await resolveAssistantStepEvidence({
-		trajectoryId: options.trajectoryId,
-		steps: options.predictions.map((prediction) => prediction.step),
-		store: options.store,
-		...options.signal ? { signal: options.signal } : {}
-	});
+	if (options.predictions.length === 0 && options.dataset === "agentrx") return {
+		findings: [],
+		diagnostics: void 0
+	};
 	if (options.dataset === "agentrx") {
 		const prediction = options.predictions[0];
+		if (!("step" in prediction)) throw new Error("AgentRx model output must name a single root-cause step");
 		if (!prediction.category) throw new Error("AgentRx model output is missing its failure category");
+		const evidenceByStep = await resolveAssistantStepEvidence({
+			trajectoryId: options.trajectoryId,
+			steps: [prediction.step],
+			store: options.store,
+			...options.signal ? { signal: options.signal } : {}
+		});
 		const [finding] = agentRxPredictionsToFindings(options.trajectoryId, [{
 			failure_case: prediction.category,
 			step_number: prediction.step,
@@ -2987,69 +3338,44 @@ async function publicBenchmarkPredictionsToFindings(options) {
 			confidence: prediction.confidence
 		});
 		if (!finding) throw new Error("AgentRx output adapter produced no root-cause finding");
-		return [{
-			...finding,
-			evidence_refs: [evidenceByStep.get(prediction.step)],
+		return {
+			findings: [{
+				...finding,
+				evidence_refs: [evidenceByStep.get(prediction.step)],
+				metadata: {
+					...finding.metadata,
+					model: options.providerModel
+				}
+			}],
+			diagnostics: void 0
+		};
+	}
+	const blocks = options.predictions.map((prediction) => {
+		if (!("first_step" in prediction)) throw new Error("CodeTraceBench model output must report first_step/last_step failure blocks");
+		return {
+			firstStep: prediction.first_step,
+			lastStep: prediction.last_step,
+			consequenceStep: prediction.consequence_step,
+			escapeStatus: prediction.escape_status,
+			severity: prediction.severity,
+			claim: prediction.claim,
+			confidence: prediction.confidence,
+			...prediction.rationale === void 0 ? {} : { rationale: prediction.rationale },
+			...prediction.recommended_action === void 0 ? {} : { recommendedAction: prediction.recommended_action },
 			metadata: {
-				...finding.metadata,
+				analysis_mode: "direct-baseline",
 				model: options.providerModel
 			}
-		}];
-	}
-	const byStep = /* @__PURE__ */ new Map();
-	for (const prediction of options.predictions) if (!byStep.has(prediction.step)) byStep.set(prediction.step, prediction);
-	return [...byStep].sort(([left], [right]) => left - right).map(([step, prediction]) => makeFinding({
-		analyst_id: options.analystId,
-		area: "incorrect",
-		subject: `incorrect-step-${step}`,
-		claim: `Step ${step} is incorrect. ${prediction.claim}`,
-		rationale: prediction.rationale,
-		severity: prediction.severity,
-		confidence: prediction.confidence,
-		evidence_refs: [evidenceByStep.get(step)],
-		recommended_action: prediction.recommended_action,
-		metadata: {
-			analysis_mode: "direct-baseline",
-			model: options.providerModel
-		},
-		produced_at: options.producedAt,
-		id_basis: `incorrect-step-${step}`
-	}));
-}
-function publicBenchmarkSystemPrompt(dataset) {
-	return `${dataset === "agentrx" ? AGENT_RX_PROMPT : CODE_TRACE_BENCH_ANALYST_PROMPT}
-
-Each finding must contain only:
-- "step": a positive integer matching an existing assistant LLM span named step-<n>
-- "severity": "critical", "high", "medium", "low", or "info"
-- "claim": one sentence
-- "confidence": a number from 0 through 1
-- optional "rationale" and "recommended_action" strings
-${dataset === "agentrx" ? `- "category": one allowed failure category listed above` : ""}
-
-Return exactly one JSON object with:
-- "report": a concise evidence-based explanation, at most 4000 characters
-- "findings": the strict finding array
-Use an empty findings array when the trace does not support a finding.
-Do not return a bare array, markdown, trace URIs, copied excerpts, or fields not listed above.
-The runner constructs exact trace URIs and action previews from each selected step.`;
-}
-function publicBenchmarkProtocolSha256(dataset) {
-	return sha256Digest(JSON.stringify({
-		dataset,
-		systemPrompt: publicBenchmarkSystemPrompt(dataset),
-		transport: {
-			attempts: 1,
-			jsonMode: true,
-			thinking: "disabled"
-		},
-		traceProjectionAttributeByteCaps: TRACE_PROJECTION_ATTRIBUTE_BYTE_CAPS,
-		evidence: {
-			location: "model-selected-positive-integer-assistant-step",
-			uri: "deterministic-trace-uri",
-			excerpt: `exact-action-prefix-512`
-		}
-	}));
+		};
+	});
+	return expandCodeTraceFailureBlocks({
+		trajectoryId: options.trajectoryId,
+		blocks,
+		store: options.store,
+		analystId: options.analystId,
+		producedAt: options.producedAt,
+		...options.signal ? { signal: options.signal } : {}
+	});
 }
 async function prepareSingleTraceContext(store, context) {
 	const storeContext = context.signal ? { signal: context.signal } : void 0;
@@ -3069,40 +3395,6 @@ async function prepareSingleTraceContext(store, context) {
 		});
 	}
 }
-const AGENT_RX_PROMPT = `Analyze exactly one failed agent trajectory.
-Find the first unrecoverable critical failure, not every later symptom.
-Inspect the complete supplied trace data.
-Emit zero findings only when the trace does not contain enough evidence.
-Otherwise emit exactly one finding.
-Its category MUST be exactly one of:
-instruction-plan-adherence-failure
-invention-of-new-information
-invalid-invocation
-misinterpretation-of-tool-output-handoff-failure
-intent-plan-misalignment
-underspecified-user-intent
-intent-not-supported
-guardrails-triggered
-system-failure
-inconclusive
-Its step is the positive integer n from the first unrecoverable assistant span named step-<n>.`;
-const CODE_TRACE_BENCH_ANALYST_PROMPT = `Analyze exactly one coding-agent trajectory and its attached final verification.
-Your task is the CodeTraceBench incorrect-step task: identify every wrong state-changing action, bad hypothesis that drives an action, and regression.
-An incorrect step remains incorrect when the agent later recovers or the final verification passes.
-Inspect the complete supplied trace data.
-Use the final-verification outcome as evidence about the final state, not as a rule for whether earlier steps were incorrect.
-For each candidate, inspect the assistant action and its following observation.
-Label a failed command when the assistant caused it through a wrong action or unsupported hypothesis.
-Label the later corrective action only when that action is itself wrong.
-Do not label a diagnostic probe merely because it exposes an earlier defect.
-Do not label a redundant but correct read or search; CodeTraceBench scores unuseful steps separately, and this run scores incorrect steps only.
-Do not label a step solely because final verification failed.
-When final verification is unavailable, use only directly observed trajectory evidence.
-Emit one finding per incorrect assistant step.
-Each finding's step MUST be the positive integer n from an existing assistant LLM span named step-<n>.
-Never select an EVALUATOR, TOOL, CHAIN, final-verification, benchmark-verification, or message-<n> span.
-Before emitting a finding, inspect its candidate span's attributes.content and describe only the action shown there.
-When the trajectory has no incorrect steps, return an empty findings array.`;
 function trajectoryIdFromCaseId$1(dataset, caseId) {
 	const prefix = dataset === "agentrx" ? "agentrx:" : "codetrace:";
 	if (!caseId.startsWith(prefix) || caseId.length === prefix.length) throw new Error(`unexpected ${dataset} benchmark case id '${caseId}'`);
@@ -3110,34 +3402,6 @@ function trajectoryIdFromCaseId$1(dataset, caseId) {
 }
 //#endregion
 //#region src/analyst/benchmark-public-rlm.ts
-const AGENT_RX_RLM_INSTRUCTIONS = `Analyze exactly one failed agent trajectory.
-Find the first unrecoverable critical failure, not every later symptom.
-Use the trace tools to inspect the action and its following observation.
-Emit zero findings only when evidence does not support a root cause.
-Otherwise emit exactly one finding whose subject is exactly one of:
-instruction-plan-adherence-failure
-invention-of-new-information
-invalid-invocation
-misinterpretation-of-tool-output-handoff-failure
-intent-plan-misalignment
-underspecified-user-intent
-intent-not-supported
-guardrails-triggered
-system-failure
-inconclusive
-Cite exactly one assistant span named step-<n> as trace://<trace-id>/span/step-<n>.
-The excerpt must quote the assistant action exactly.`;
-const CODE_TRACE_RLM_INSTRUCTIONS = `${CODE_TRACE_BENCH_ANALYST_PROMPT}
-Use the trace tools rather than asking for the whole trajectory in the prompt.
-Keep retrieved trace objects in Python variables.
-Never print an entire trace, full source file, or more than 12000 characters in one iteration.
-Build a compact table of assistant step ids, actions, following observations, and final verification.
-Inspect suspicious steps with viewSpans or searchSpan instead of repeatedly printing the table.
-Submit as soon as every state-changing assistant step has a supported verdict.
-For each incorrect step, set subject to incorrect-step-<n>.
-Cite the assistant span as trace://<URL-encoded-trace-id>/span/step-<n>.
-The evidence excerpt must be an exact quote from that span's action content.
-Return no finding for a clean trajectory.`;
 /** Public benchmark candidate that runs the actual recursive trace analyst. */
 function createPublicBenchmarkRlmRunner(dataset, config) {
 	const costLedger = config.costLedger ?? new CostLedger();
@@ -3205,20 +3469,22 @@ function createPublicBenchmarkRlmRunner(dataset, config) {
 					},
 					produced_at: producedAt
 				}));
-				const findings = adaptPublicBenchmarkFindings(dataset, trajectoryId, rawFindings, "dspy-rlm");
-				if (dataset === "codetracebench") await validateCodeTraceFindingEvidence({
+				const adapted = await adaptPublicBenchmarkFindings({
+					dataset,
 					trajectoryId,
-					findings,
+					findings: rawFindings,
+					analystId: "dspy-rlm",
 					store: input.traceStore,
 					...context.signal ? { signal: context.signal } : {}
 				});
 				return {
-					findings,
+					findings: adapted.findings,
 					usage,
 					metadata: {
 						analysisMode: "recursive",
 						engine: "dspy-rlm",
 						protocolSha256: publicBenchmarkProtocolSha256(dataset),
+						...adapted.diagnostics ? { blockDiagnostics: adapted.diagnostics } : {},
 						answer: completed.answer,
 						trajectory: completed.trajectory,
 						modelCalls: completed.modelCalls,
@@ -3250,7 +3516,7 @@ function publicBenchmarkDefinition(dataset, limits) {
 		area: dataset === "agentrx" ? "root-cause" : "incorrect",
 		version: "1.0.0",
 		question: dataset === "agentrx" ? "What is the first unrecoverable root cause in this failed trajectory?" : "Which assistant steps are incorrect under the CodeTraceBench definition?",
-		instructions: dataset === "agentrx" ? AGENT_RX_RLM_INSTRUCTIONS : CODE_TRACE_RLM_INSTRUCTIONS,
+		instructions: publicBenchmarkRlmInstructions(dataset),
 		toolGroup: "singleTrace",
 		limits
 	};
@@ -3341,7 +3607,7 @@ function createRunIdentity(config, prepared) {
 			analystProtocolSha256: publicBenchmarkProtocolSha256(config.dataset),
 			implementationSha256: ANALYST_BENCHMARK_IMPLEMENTATION_SHA256,
 			dependencyLockSha256: ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256,
-			runnerIds: ["empty", "dspy-rlm"]
+			runnerIds: ["empty", config.analyst]
 		},
 		inputs: {
 			labelsSha256: prepared.labelsSha256,
@@ -3512,7 +3778,7 @@ function createObservationAppender(path, runIdentitySha256, progress) {
 		return write;
 	};
 }
-async function readProgress(path, runIdentitySha256, caseIds, repetitions) {
+async function readProgress(path, runIdentitySha256, caseIds, repetitions, analystRunnerId) {
 	const rawLines = (await readRegularFile(path, "benchmark observation log")).split("\n");
 	if (rawLines.at(-1) === "") rawLines.pop();
 	const observations = [];
@@ -3546,7 +3812,7 @@ async function readProgress(path, runIdentitySha256, caseIds, repetitions) {
 			previousRowSha256: parsed.previousRowSha256,
 			observation
 		}) !== parsed.rowSha256) throw new Error(`benchmark observation row ${index + 1} digest does not match its contents`);
-		if (!allowedCases.has(observation.caseId) || observation.runnerId !== "empty" && observation.runnerId !== "dspy-rlm" || observation.repetition >= repetitions || observation.executionIndex >= plannedObservationCount) throw new Error(`benchmark observation row ${index + 1} does not match a planned case, runner, and repetition`);
+		if (!allowedCases.has(observation.caseId) || observation.runnerId !== "empty" && observation.runnerId !== analystRunnerId || observation.repetition >= repetitions || observation.executionIndex >= plannedObservationCount) throw new Error(`benchmark observation row ${index + 1} does not match a planned case, runner, and repetition`);
 		if (executionIndexes.has(observation.executionIndex)) throw new Error(`duplicate benchmark executionIndex ${observation.executionIndex} at line ${index + 1}`);
 		observations.push(observation);
 		seen.add(key);
@@ -3991,7 +4257,7 @@ function assertCompletedArtifactMatchesRun(artifact, manifest, observations, pre
 	if (canonicalJson(artifact.inputs) !== canonicalJson(expectedInputs)) throw new Error("completed benchmark result inputs do not match the run manifest");
 	const provenance = artifact.result.provenance;
 	const expectedDatasetId = config.dataset === "agentrx" ? "microsoft/AgentRx" : "NJU-LINK/CodeTraceBench";
-	const expectedOutputAdapter = config.dataset === "agentrx" ? "agentrx-taxonomy-and-root-step" : "codetracebench-incorrect-step";
+	const expectedOutputAdapter = config.dataset === "agentrx" ? "agentrx-taxonomy-and-root-step" : "codetracebench-incorrect-block";
 	if (provenance.id !== `${config.dataset}-real-model-analyst` || provenance.startedAt !== manifest.createdAt || !Number.isFinite(Date.parse(provenance.endedAt)) || Date.parse(provenance.endedAt) < Date.parse(provenance.startedAt) || canonicalJson(provenance.dataset) !== canonicalJson({
 		id: expectedDatasetId,
 		revision: config.datasetRevision,
@@ -4001,7 +4267,7 @@ function assertCompletedArtifactMatchesRun(artifact, manifest, observations, pre
 	if (canonicalJson(artifact.result.summaries) !== canonicalJson(expectedSummaries)) throw new Error("completed benchmark summaries do not match durable observations");
 	const expectedComparisons = [compareAnalystRunners(artifact.result, {
 		baselineRunnerId: "empty",
-		candidateRunnerId: "dspy-rlm",
+		candidateRunnerId: config.runnerIds[1],
 		seed: config.seed
 	})];
 	if (canonicalJson(artifact.comparisons) !== canonicalJson(expectedComparisons)) throw new Error("completed benchmark comparisons do not match durable observations");
@@ -4134,7 +4400,7 @@ async function executeAnalystBenchmarkCommand(config, dependencies) {
 	const localIdentitySha256 = digestCanonical(localReceipt.local);
 	const identitySha256 = digestCanonical(identity);
 	const manifest = config.resume ? await readAndValidateResumeFiles(paths, identity, identitySha256, localIdentitySha256, localReceipt) : await initializeRunFiles(paths, identity, identitySha256, localIdentitySha256, localReceipt);
-	const progress = await readProgress(paths.observations, manifest.identitySha256, prepared.selectedCaseIds, config.repetitions);
+	const progress = await readProgress(paths.observations, manifest.identitySha256, prepared.selectedCaseIds, config.repetitions, config.analyst);
 	const costLedger = createRunCostLedger({
 		storage: fsCampaignStorage(),
 		runDir: paths.directory,
@@ -4147,10 +4413,10 @@ async function executeAnalystBenchmarkCommand(config, dependencies) {
 		const markdown = renderArtifactMarkdown(artifact);
 		await writeExclusiveOrVerify(paths.report, markdown);
 		printSuccessSummary(artifact, paths);
-		return benchmarkExitCode(artifact.result);
+		return benchmarkExitCode(artifact.result, config.analyst);
 	}
 	if (await regularFileExists(paths.report)) throw new Error(`benchmark report exists without a completed result; refusing ambiguous resume: ${paths.report}`);
-	const createAnalystRunner = dependencies.createAnalystRunner ?? ((dataset, model) => createPublicBenchmarkRlmRunner(dataset, model));
+	const createAnalystRunner = dependencies.createAnalystRunner ?? ((dataset, model) => config.analyst === "direct" ? createPublicBenchmarkDirectRunner(dataset, model) : createPublicBenchmarkRlmRunner(dataset, model));
 	const runners = [emptyPublicBenchmarkRunner(), createAnalystRunner(config.dataset, {
 		...config.model,
 		costLedger,
@@ -4172,7 +4438,7 @@ async function executeAnalystBenchmarkCommand(config, dependencies) {
 			initialObservations: progress.observations,
 			signal: runAbort.signal,
 			onObservation: async (observation) => {
-				assertObservationAccountingComplete(observation, costLedger);
+				assertObservationAccountingComplete(observation, costLedger, config.analyst);
 				await appendObservation(observation);
 			},
 			resolveEvidence: traceStoreEvidenceResolver((input) => {
@@ -4193,7 +4459,7 @@ async function executeAnalystBenchmarkCommand(config, dependencies) {
 				},
 				metadata: {
 					model: config.model.model,
-					outputAdapter: config.dataset === "agentrx" ? "agentrx-taxonomy-and-root-step" : "codetracebench-incorrect-step",
+					outputAdapter: config.dataset === "agentrx" ? "agentrx-taxonomy-and-root-step" : "codetracebench-incorrect-block",
 					caseSelection: prepared.selection.method,
 					caseSelectionSeed: config.seed,
 					selectionStratified: prepared.selection.stratified,
@@ -4211,11 +4477,11 @@ async function executeAnalystBenchmarkCommand(config, dependencies) {
 	}
 	assertCostLedgerFinalizable(costLedger);
 	result.provenance.startedAt = manifest.createdAt;
-	const persisted = await readProgress(paths.observations, manifest.identitySha256, prepared.selectedCaseIds, config.repetitions);
+	const persisted = await readProgress(paths.observations, manifest.identitySha256, prepared.selectedCaseIds, config.repetitions, config.analyst);
 	assertSameObservations(result.observations, persisted.observations);
 	const comparisons = [compareAnalystRunners(result, {
 		baselineRunnerId: "empty",
-		candidateRunnerId: "dspy-rlm",
+		candidateRunnerId: config.analyst,
 		seed: config.seed
 	})];
 	const codeTraceCalibration = config.dataset === "codetracebench" ? summarizeCodeTraceCalibration(result) : void 0;
@@ -4260,7 +4526,7 @@ async function executeAnalystBenchmarkCommand(config, dependencies) {
 	await writeExclusiveOrVerify(paths.result, `${JSON.stringify(artifact, null, 2)}\n`);
 	await writeExclusiveOrVerify(paths.report, markdown);
 	printSuccessSummary(artifact, paths);
-	return benchmarkExitCode(result);
+	return benchmarkExitCode(result, config.analyst);
 }
 const NON_SCORABLE_COST_ERRORS = /* @__PURE__ */ new Set([
 	"CostAccountingIncompleteError",
@@ -4270,9 +4536,9 @@ const NON_SCORABLE_COST_ERRORS = /* @__PURE__ */ new Set([
 	"CostReceiptCaptureError",
 	"CostReservationExceededError"
 ]);
-function assertObservationAccountingComplete(observation, costLedger) {
+function assertObservationAccountingComplete(observation, costLedger, analystRunnerId) {
 	if (observation.error && NON_SCORABLE_COST_ERRORS.has(observation.error.class)) throw new CostAccountingIncompleteError(`Analyst benchmark stopped before scoring: ${observation.error.message}`);
-	if (observation.runnerId !== "dspy-rlm") return;
+	if (observation.runnerId !== analystRunnerId) return;
 	const summary = costLedger.summary({
 		channel: "analyst",
 		tags: {
@@ -4302,6 +4568,9 @@ Run the recursive DSPy trace analyst against public AgentRx or CodeTraceBench la
 
 Required:
   --dataset agentrx|codetracebench
+  --analyst dspy-rlm|direct        Scored analyst. Default: dspy-rlm.
+                                   'direct' is the retired one-shot runner that
+                                   produced the published evidence.
   --labels <dataset.json|dataset.jsonl>
   --trace-dir <one-trace-per-file OTLP JSONL directory>
   --artifact-dir <extracted artifact root>  Required for CodeTraceBench
@@ -4344,8 +4613,11 @@ function parseCommandConfig(argv, env) {
 	if (!apiKey) throw new Error(`--api-key-env points to an empty or missing variable: ${apiKeyEnv}`);
 	const maxCostUsd = positiveFiniteFlag(flags, "max-cost-usd", 5);
 	const python = flags.get("python")?.trim();
+	const analyst = flags.get("analyst")?.trim() ?? "dspy-rlm";
+	if (analyst !== "dspy-rlm" && analyst !== "direct") throw new Error("--analyst must be 'dspy-rlm' or 'direct'");
 	return {
 		dataset,
+		analyst,
 		labelsPath: requiredFlag(flags, "labels"),
 		traceDir: requiredFlag(flags, "trace-dir"),
 		...artifactDir ? { artifactDir } : {},
@@ -4396,6 +4668,7 @@ function parseFlags(argv) {
 const KNOWN_FLAGS = /* @__PURE__ */ new Set([
 	"resume",
 	"dataset",
+	"analyst",
 	"labels",
 	"trace-dir",
 	"artifact-dir",
@@ -4519,8 +4792,8 @@ function renderArtifactMarkdown(artifact) {
 	const verificationMarkdown = artifact.inputs.dataset === "codetracebench" ? `\n\n${renderVerificationAvailability(artifact.inputs.verificationAvailability)}` : "";
 	return `${renderAnalystBenchmarkMarkdown(artifact.result, artifact.comparisons).trimEnd()}${calibrationMarkdown}${verificationMarkdown}\n\n${renderSelectionMarkdown(artifact.inputs.selection.report)}\n`;
 }
-function benchmarkExitCode(result) {
-	return result.summaries.find((summary) => summary.runnerId === "dspy-rlm")?.failedRuns ? 2 : 0;
+function benchmarkExitCode(result, analystRunnerId) {
+	return result.summaries.find((summary) => summary.runnerId === analystRunnerId)?.failedRuns ? 2 : 0;
 }
 function printSuccessSummary(artifact, paths) {
 	const failures = artifact.result.summaries.reduce((total, summary) => total + summary.failedRuns, 0);
@@ -4532,6 +4805,6 @@ function shellQuote(value) {
 	return /^[a-zA-Z0-9_./:@%+=,-]+$/.test(value) ? value : `'${value.replaceAll("'", "'\\''")}'`;
 }
 //#endregion
-export { ANALYST_BENCHMARK_IMPLEMENTATION_FILES as A, summarizeAgentRxCalibration as B, ANALYST_BENCHMARK_DEPENDENCY_LOCK_DIGEST_ALGORITHM as C, ANALYST_BENCHMARK_EVIDENCE_IMPLEMENTATION_SHA256 as D, ANALYST_BENCHMARK_EVIDENCE_DEPENDENCY_LOCK_SHA256 as E, ANALYST_BENCHMARK_LOCAL_RECEIPT_FILE as F, normalizeAgentRxCategory as G, codeTracerPredictionsToFindings as H, ANALYST_BENCHMARK_MANIFEST_FILE as I, roundAgentRxStep as K, ANALYST_BENCHMARK_OBSERVATIONS_FILE as L, analystBenchmarkDependencyLockDigest as M, analystBenchmarkImplementationDigest as N, ANALYST_BENCHMARK_EVIDENCE_PACKAGE_VERSION as O, ANALYST_BENCHMARK_COST_LEDGER_FILE as P, AGENT_RX_UPSTREAM_REVISION as R, emptyPublicBenchmarkRunner as S, ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 as T, agentRxBenchmarkCase as U, codeTraceBenchCase as V, agentRxPredictionsToFindings as W, DEFAULT_MAX_VERIFICATION_ARTIFACT_BYTES as _, renderCodeTraceCalibrationMarkdown as a, parseVerificationOutcome as b, createPublicBenchmarkRlmRunner as c, publicBenchmarkProtocolSha256 as d, loadPublicBenchmarkRows as f, selectPublicBenchmarkRows as g, publicBenchmarkSelectionReport as h, readAnalystBenchmarkArtifact as i, ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 as j, ANALYST_BENCHMARK_IMPLEMENTATION_DIGEST_ALGORITHM as k, CODE_TRACE_BENCH_ANALYST_PROMPT as l, publicBenchmarkDistributions as m, runAnalystBenchmarkCommand as n, summarizeCodeTraceCalibration as o, preparePublicAnalystBenchmark as p, normalizeBenchmarkLabel as q, renderAnalystBenchmarkMarkdown as r, compareAnalystRunners as s, ANALYST_BENCHMARK_HELP as t, createPublicBenchmarkDirectRunner as u, appendVerificationArtifactsToOtlp as v, ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES as w, adaptPublicBenchmarkFindings as x, loadCodeTraceVerificationArtifacts as y, renderAgentRxCalibrationMarkdown as z };
+export { ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 as A, ANALYST_BENCHMARK_LOCAL_RECEIPT_FILE as B, publicBenchmarkSystemPrompt as C, parseVerificationOutcome as D, loadCodeTraceVerificationArtifacts as E, ANALYST_BENCHMARK_IMPLEMENTATION_FILES as F, summarizeAgentRxCalibration as G, ANALYST_BENCHMARK_OBSERVATIONS_FILE as H, ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 as I, agentRxBenchmarkCase as J, codeTraceBenchCase as K, analystBenchmarkDependencyLockDigest as L, ANALYST_BENCHMARK_EVIDENCE_IMPLEMENTATION_SHA256 as M, ANALYST_BENCHMARK_EVIDENCE_PACKAGE_VERSION as N, ANALYST_BENCHMARK_DEPENDENCY_LOCK_DIGEST_ALGORITHM as O, ANALYST_BENCHMARK_IMPLEMENTATION_DIGEST_ALGORITHM as P, normalizeBenchmarkLabel as Q, analystBenchmarkImplementationDigest as R, publicBenchmarkRlmInstructions as S, appendVerificationArtifactsToOtlp as T, AGENT_RX_UPSTREAM_REVISION as U, ANALYST_BENCHMARK_MANIFEST_FILE as V, renderAgentRxCalibrationMarkdown as W, normalizeAgentRxCategory as X, agentRxPredictionsToFindings as Y, roundAgentRxStep as Z, expandCodeTraceFailureBlocks as _, renderCodeTraceCalibrationMarkdown as a, MAX_INCORRECT_BLOCK_STEPS as b, createPublicBenchmarkRlmRunner as c, preparePublicAnalystBenchmark as d, publicBenchmarkDistributions as f, emptyPublicBenchmarkRunner as g, adaptPublicBenchmarkFindings as h, readAnalystBenchmarkArtifact as i, ANALYST_BENCHMARK_EVIDENCE_DEPENDENCY_LOCK_SHA256 as j, ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES as k, createPublicBenchmarkDirectRunner as l, selectPublicBenchmarkRows as m, runAnalystBenchmarkCommand as n, summarizeCodeTraceCalibration as o, publicBenchmarkSelectionReport as p, codeTracerPredictionsToFindings as q, renderAnalystBenchmarkMarkdown as r, compareAnalystRunners as s, ANALYST_BENCHMARK_HELP as t, loadPublicBenchmarkRows as u, CODE_TRACE_BENCH_ANALYST_PROMPT as v, DEFAULT_MAX_VERIFICATION_ARTIFACT_BYTES as w, publicBenchmarkProtocolSha256 as x, MAX_INCORRECT_BLOCKS as y, ANALYST_BENCHMARK_COST_LEDGER_FILE as z };
 
-//# sourceMappingURL=benchmark-command-DzUZJl8M.js.map
+//# sourceMappingURL=benchmark-command-xvi2liH7.js.map