@tangle-network/agent-eval 0.138.0 → 0.139.0

package/dist/{benchmark-command-CMqVqReF.js → benchmark-command-BKfjOBJ5.js}

@@ -1,11 +1,16 @@
 import { c as ValidationError, t as AgentEvalError } from "./errors-D-LKuDhb.js";
-import { a as CostLedgerPersistenceError, i as CostLedger, n as CostCallConflictError, o as CostReceiptCaptureError, r as CostCeilingReachedError, s as CostReservationExceededError, t as CostAccountingIncompleteError } from "./cost-ledger-CHDLA0Ss.js";
-import { c as callLlmJson, f as maximumChargeForLlmRequest, l as costReceiptFromLlm, r as LlmResponseError, t as LlmCallError, u as costReceiptFromLlmError } from "./llm-client-Cj3c7PEm.js";
-import { d as TRACE_ANALYSIS_LIMITS, i as otlpTextToTraceAnalysisStore, r as createOtlpBufferTraceStore, t as DEFAULT_MAX_TRACE_FILE_BYTES } from "./store-otlp-BenKynPE.js";
-import { d as usageReceiptFromCostLedger, m as makeFinding, n as createRunCostLedger, r as fsCampaignStorage, t as acquireSingleRunLock } from "./single-run-lock-D_bS5xhj.js";
-import { _ as canonicalString, c as writeLedgerFileAtomically, s as withLedgerFileLock, v as hashCanonical } from "./ledger-core-C0Yx1I14.js";
+import { s as resolveModelPricing } from "./metrics-C9YY1OcL.js";
+import { a as CostLedgerPersistenceError, i as CostLedger, n as CostCallConflictError, o as CostReceiptCaptureError, r as CostCeilingReachedError, s as CostReservationExceededError, t as CostAccountingIncompleteError } from "./cost-ledger-CZ9diLxY.js";
+import { c as callLlmJson, f as maximumChargeForLlmRequest, l as costReceiptFromLlm, r as LlmResponseError, t as LlmCallError, u as costReceiptFromLlmError } from "./llm-client-bkztEfIx.js";
+import { O as evidenceRefsFromRawFinding, h as TRACE_ANALYSIS_LIMITS, i as runTraceAnalyst } from "./kind-factory-CFxA0JQX.js";
+import { o as makeFinding, r as usageReceiptFromCostLedger } from "./usage-receipt-CgxMEBZq.js";
+import { i as hashCanonical, r as canonicalString } from "./canonical-D011XM8r.js";
+import { n as createRunCostLedger, r as fsCampaignStorage, t as acquireSingleRunLock } from "./single-run-lock-BTTtPZ9N.js";
+import { t as createDspyRlmTraceEngine } from "./dspy-rlm-engine-DTkVyDX-.js";
+import { c as writeLedgerFileAtomically, s as withLedgerFileLock } from "./ledger-core-Dxz0Rkwa.js";
 import { E as pairedBootstrap } from "./statistics-ByxzSiOM.js";
-import { i as summarizeAnalystBenchmarkRunner, n as runAnalystBenchmark, r as traceStoreEvidenceResolver } from "./benchmark-D8dkki-J.js";
+import { i as otlpTextToTraceAnalysisStore, r as createOtlpBufferTraceStore, t as DEFAULT_MAX_TRACE_FILE_BYTES } from "./store-otlp-DX4fGIcf.js";
+import { i as summarizeAnalystBenchmarkRunner, n as runAnalystBenchmark, r as traceStoreEvidenceResolver } from "./benchmark-CYtcIF2V.js";
 import { z } from "zod";
 import { constants, existsSync, lstatSync, readFileSync } from "node:fs";
 import * as nodePath from "node:path";
@@ -1161,17 +1166,26 @@ function isSha256(value) {
 //#region src/analyst/benchmark-implementation.ts
 const ANALYST_BENCHMARK_IMPLEMENTATION_DIGEST_ALGORITHM = "sha256-canonical-source-manifest";
 const ANALYST_BENCHMARK_DEPENDENCY_LOCK_DIGEST_ALGORITHM = "sha256-canonical-file-manifest";
-const ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES = Object.freeze(["package.json", "pnpm-lock.yaml"]);
-const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = "d06e3e3f171cc6bd5cf36b14325f507f34b1c6a9d5129ce8a85bf3c1681f8223";
-/** The published benchmark evidence was produced at this package version.
-* A release changes package.json's version field, which is part of the lock
-* manifest but cannot change benchmark behavior, so the evidence stays bound
-* to the digest at its creation. A test proves the current lock differs from
-* the evidence lock by the version stamp alone; any real dependency change
-* still forces a new benchmark run or explicit retirement of the evidence. */
+const ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES = Object.freeze([
+	"clients/python/pyproject.toml",
+	"clients/python/uv.lock",
+	"package.json",
+	"pnpm-lock.yaml"
+]);
+const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = "6f3dc59755ff2a26fba4c2c1ad670f436cdd183faf252a5b9c2d50e6b190b460";
+/** The published benchmark evidence was produced at this package version, by
+* the retired one-shot direct runner, before trace analysts moved to the
+* recursive DSPy RLM engine. Both evidence digests below are historical facts
+* about that artifact: the current implementation and dependency manifest have
+* since changed, so they cannot describe the current engine. A fresh certified
+* run must replace the published evidence before any accuracy number is
+* attributed to the engine that ships today. */
 const ANALYST_BENCHMARK_EVIDENCE_PACKAGE_VERSION = "0.137.0";
 const ANALYST_BENCHMARK_EVIDENCE_DEPENDENCY_LOCK_SHA256 = "1e03f2daed356d60316aabefb407ec1e437ac94d408d61eea4ae096e9c6fbb5b";
+const ANALYST_BENCHMARK_EVIDENCE_IMPLEMENTATION_SHA256 = "4dba263b6256a30d56c7fdb2d992d3a953c0035d731f359b704db806f68f75ac";
 const ANALYST_BENCHMARK_IMPLEMENTATION_FILES = Object.freeze([
+	"clients/python/src/agent_eval_rpc/dspy_rlm_bridge.py",
+	"clients/python/src/agent_eval_rpc/optimizer_bridge_common.py",
 	"src/analyst/benchmark-agentrx-calibration.ts",
 	"src/analyst/benchmark-command-artifact.ts",
 	"src/analyst/benchmark-command-persistence.ts",
@@ -1189,6 +1203,7 @@ const ANALYST_BENCHMARK_IMPLEMENTATION_FILES = Object.freeze([
 	"src/analyst/benchmark-public-data.ts",
 	"src/analyst/benchmark-public-errors.ts",
 	"src/analyst/benchmark-public-model.ts",
+	"src/analyst/benchmark-public-rlm.ts",
 	"src/analyst/benchmark-public-types.ts",
 	"src/analyst/benchmark-real-model.ts",
 	"src/analyst/benchmark-report.ts",
@@ -1198,8 +1213,22 @@ const ANALYST_BENCHMARK_IMPLEMENTATION_FILES = Object.freeze([
 	"src/analyst/benchmark-verification-artifacts.ts",
 	"src/analyst/benchmark-verification-outcome.ts",
 	"src/analyst/benchmark.ts",
+	"src/analyst/dspy-rlm-engine.ts",
+	"src/analyst/engine.ts",
+	"src/analyst/exact-types.ts",
+	"src/analyst/finding-signature.ts",
+	"src/analyst/finding-subject.ts",
+	"src/analyst/kind-factory.ts",
+	"src/analyst/parse-tolerant.ts",
+	"src/analyst/tool-groups.ts",
+	"src/analyst/trace-tool-callback.ts",
 	"src/analyst/types.ts",
 	"src/analyst/usage-receipt.ts",
+	"src/campaign/external-optimizer-contracts.ts",
+	"src/campaign/external-optimizer-http.ts",
+	"src/campaign/external-optimizer-model-proxy.ts",
+	"src/campaign/external-optimizer-resources.ts",
+	"src/campaign/external-optimizer-subprocess.ts",
 	"src/campaign/search-ledger-errors.ts",
 	"src/campaign/search-ledger-file.ts",
 	"src/campaign/single-run-lock.ts",
@@ -1210,6 +1239,7 @@ const ANALYST_BENCHMARK_IMPLEMENTATION_FILES = Object.freeze([
 	"src/judge-calibration.ts",
 	"src/ledger-core/atomic-file-lock.ts",
 	"src/ledger-core/canonical.ts",
+	"src/ledger-core/deep-freeze.ts",
 	"src/ledger-core/index.ts",
 	"src/ledger-core/journal-file.ts",
 	"src/ledger-core/journal.ts",
@@ -1229,12 +1259,13 @@ const ANALYST_BENCHMARK_IMPLEMENTATION_FILES = Object.freeze([
 	"src/trace-analyst/store-otlp.ts",
 	"src/trace-analyst/store-schemas.ts",
 	"src/trace-analyst/store.ts",
+	"src/trace-analyst/tools.ts",
 	"src/trace-analyst/types.ts",
 	"src/trace/attribute-vocabulary.ts",
 	"src/trace/otlp-attributes.ts",
 	"src/trace/raw-provider-sink.ts"
 ]);
-const ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 = "4dba263b6256a30d56c7fdb2d992d3a953c0035d731f359b704db806f68f75ac";
+const ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 = "5e47fc9d9f49c468d3ef06c5d552925e6a026f9e22a75056a9c8c5a2879744c0";
 function analystBenchmarkImplementationDigest() {
 	return ANALYST_BENCHMARK_IMPLEMENTATION_SHA256;
 }
@@ -2485,7 +2516,11 @@ function publicBenchmarkError(error, secrets = []) {
 function redactSensitiveText(value, secrets) {
 	let redacted = value;
 	for (const secret of secrets) if (secret) redacted = redacted.replaceAll(secret, "[REDACTED]");
-	return redacted.replace(/\bBearer\s+[^\s"',;]+/gi, "Bearer [REDACTED]").replace(/\b(api[_-]?key|access[_-]?token|refresh[_-]?token|password|secret)\b\s*[:=]\s*[^\s"',;]+/gi, "$1=[REDACTED]").slice(0, 500);
+	redacted = redacted.replace(/\bBearer\s+[^\s"',;]+/gi, "Bearer [REDACTED]").replace(/\b(api[_-]?key|access[_-]?token|refresh[_-]?token|password|secret)\b\s*[:=]\s*[^\s"',;]+/gi, "$1=[REDACTED]");
+	if (redacted.length <= 500) return redacted;
+	const head = redacted.slice(0, 180);
+	const marker = `...[${redacted.length - 460} chars omitted]...`;
+	return `${head}${marker}${redacted.slice(-(500 - head.length - marker.length))}`;
 }
 //#endregion
 //#region src/analyst/benchmark-response-cache.ts
@@ -2629,7 +2664,8 @@ const TRACE_PROJECTION_ATTRIBUTE_BYTE_CAPS = [
 	128,
 	64
 ];
-function createPublicBenchmarkModelRunner(dataset, config) {
+/** One-shot JSON baseline. This is not a recursive trace analyst. */
+function createPublicBenchmarkDirectRunner(dataset, config) {
 	const model = requiredString(config.model, "model");
 	const baseUrl = requiredString(config.baseUrl, "baseUrl");
 	const apiKey = requiredString(config.apiKey, "apiKey");
@@ -2652,9 +2688,9 @@ function createPublicBenchmarkModelRunner(dataset, config) {
 		...config.fetchImpl ? { fetch: config.fetchImpl } : {}
 	};
 	return {
-		id: "model",
+		id: "direct",
 		async analyze(input, context) {
-			const trajectoryId = trajectoryIdFromCaseId(dataset, context.caseId);
+			const trajectoryId = trajectoryIdFromCaseId$1(dataset, context.caseId);
 			const costTags = {
 				analystId: actor,
 				benchmarkCaseId: context.caseId,
@@ -2665,7 +2701,7 @@ function createPublicBenchmarkModelRunner(dataset, config) {
 			let providerModel = model;
 			let producedAt;
 			let modelMetadata = {
-				analysisMode: "single-pass",
+				analysisMode: "direct-baseline",
 				outputAdapter,
 				protocolSha256: publicBenchmarkProtocolSha256(dataset)
 			};
@@ -2797,7 +2833,7 @@ function createPublicBenchmarkModelRunner(dataset, config) {
 					trajectoryId,
 					predictions: rawPredictions,
 					store: input.traceStore,
-					analystId: "model",
+					analystId: "direct",
 					providerModel,
 					producedAt: requiredString(producedAt ?? "", "finding producedAt"),
 					...context.signal ? { signal: context.signal } : {}
@@ -2818,7 +2854,7 @@ function createPublicBenchmarkModelRunner(dataset, config) {
 				};
 			} catch (error) {
 				if (context.signal?.aborted) throw error;
-				if (isPaidCallControlError(error)) throw error;
+				if (isPaidCallControlError$1(error)) throw error;
 				return {
 					findings: [],
 					usage: usageReceiptFromCostLedger(costLedger, {
@@ -2856,7 +2892,7 @@ function assertCacheReceiptMatches(cached, receipt) {
 		receipt
 	});
 }
-function isPaidCallControlError(error) {
+function isPaidCallControlError$1(error) {
 	return error instanceof CostAccountingIncompleteError || error instanceof CostCallConflictError || error instanceof CostCeilingReachedError || error instanceof CostLedgerPersistenceError || error instanceof CostReceiptCaptureError || error instanceof CostReservationExceededError;
 }
 function receiptForProviderFailure(error, completedResult, model) {
@@ -2973,7 +3009,7 @@ async function publicBenchmarkPredictionsToFindings(options) {
 		evidence_refs: [evidenceByStep.get(step)],
 		recommended_action: prediction.recommended_action,
 		metadata: {
-			analysis_mode: "single-pass",
+			analysis_mode: "direct-baseline",
 			model: options.providerModel
 		},
 		produced_at: options.producedAt,
@@ -3067,11 +3103,174 @@ Each finding's step MUST be the positive integer n from an existing assistant LL
 Never select an EVALUATOR, TOOL, CHAIN, final-verification, benchmark-verification, or message-<n> span.
 Before emitting a finding, inspect its candidate span's attributes.content and describe only the action shown there.
 When the trajectory has no incorrect steps, return an empty findings array.`;
+function trajectoryIdFromCaseId$1(dataset, caseId) {
+	const prefix = dataset === "agentrx" ? "agentrx:" : "codetrace:";
+	if (!caseId.startsWith(prefix) || caseId.length === prefix.length) throw new Error(`unexpected ${dataset} benchmark case id '${caseId}'`);
+	return caseId.slice(prefix.length);
+}
+//#endregion
+//#region src/analyst/benchmark-public-rlm.ts
+const AGENT_RX_RLM_INSTRUCTIONS = `Analyze exactly one failed agent trajectory.
+Find the first unrecoverable critical failure, not every later symptom.
+Use the trace tools to inspect the action and its following observation.
+Emit zero findings only when evidence does not support a root cause.
+Otherwise emit exactly one finding whose subject is exactly one of:
+instruction-plan-adherence-failure
+invention-of-new-information
+invalid-invocation
+misinterpretation-of-tool-output-handoff-failure
+intent-plan-misalignment
+underspecified-user-intent
+intent-not-supported
+guardrails-triggered
+system-failure
+inconclusive
+Cite exactly one assistant span named step-<n> as trace://<trace-id>/span/step-<n>.
+The excerpt must quote the assistant action exactly.`;
+const CODE_TRACE_RLM_INSTRUCTIONS = `${CODE_TRACE_BENCH_ANALYST_PROMPT}
+Use the trace tools rather than asking for the whole trajectory in the prompt.
+Keep retrieved trace objects in Python variables.
+Never print an entire trace, full source file, or more than 12000 characters in one iteration.
+Build a compact table of assistant step ids, actions, following observations, and final verification.
+Inspect suspicious steps with viewSpans or searchSpan instead of repeatedly printing the table.
+Submit as soon as every state-changing assistant step has a supported verdict.
+For each incorrect step, set subject to incorrect-step-<n>.
+Cite the assistant span as trace://<URL-encoded-trace-id>/span/step-<n>.
+The evidence excerpt must be an exact quote from that span's action content.
+Return no finding for a clean trajectory.`;
+/** Public benchmark candidate that runs the actual recursive trace analyst. */
+function createPublicBenchmarkRlmRunner(dataset, config) {
+	const costLedger = config.costLedger ?? new CostLedger();
+	const limits = {
+		maxIterations: config.dspyRlm?.maxIterations ?? 8,
+		maxLlmCalls: config.dspyRlm?.maxLlmCalls ?? 4,
+		maxToolCalls: config.dspyRlm?.maxToolCalls ?? 32,
+		maxOutputChars: config.dspyRlm?.maxOutputChars ?? 8e3
+	};
+	const pricing = config.pricing ?? pricingForModel(config.model);
+	const engine = createDspyRlmTraceEngine({
+		baseUrl: config.baseUrl,
+		apiKey: config.apiKey,
+		model: config.model,
+		maxOutputTokens: config.maxOutputTokens,
+		timeoutMs: config.timeoutMs,
+		maxCostUsd: config.maxCostUsdPerAnalysis ?? 1,
+		pricing,
+		...config.dspyRlm?.runner ? { runner: config.dspyRlm.runner } : {}
+	});
+	const definition = publicBenchmarkDefinition(dataset, limits);
+	return {
+		id: "dspy-rlm",
+		async analyze(input, context) {
+			const trajectoryId = trajectoryIdFromCaseId(dataset, context.caseId);
+			const tags = {
+				benchmarkCaseId: context.caseId,
+				benchmarkRepetition: String(context.repetition)
+			};
+			let usage;
+			let rawFindings = [];
+			try {
+				if (!input.traceStore) throw new Error(`${dataset} DSPy RLM runner requires a trace store`);
+				const completed = await runTraceAnalyst({
+					definition,
+					engine,
+					store: input.traceStore,
+					context: {
+						runId: context.caseId,
+						correlationId: `${context.caseId}:${context.repetition}`,
+						costLedger,
+						costPhase: "analyst.public-benchmark.dspy-rlm",
+						tags,
+						recordUsage: (receipt) => {
+							usage = receipt;
+						},
+						signal: context.signal
+					}
+				});
+				const producedAt = (/* @__PURE__ */ new Date()).toISOString();
+				rawFindings = completed.findings.map((finding) => makeFinding({
+					analyst_id: "dspy-rlm",
+					area: dataset === "agentrx" ? "root-cause" : "incorrect",
+					subject: finding.subject,
+					claim: finding.claim,
+					rationale: finding.rationale,
+					severity: finding.severity,
+					confidence: finding.confidence,
+					evidence_refs: evidenceRefsFromRawFinding(finding),
+					recommended_action: finding.recommended_action,
+					metadata: {
+						analysis_mode: "recursive",
+						engine: "dspy-rlm",
+						model: config.model
+					},
+					produced_at: producedAt
+				}));
+				const findings = adaptPublicBenchmarkFindings(dataset, trajectoryId, rawFindings, "dspy-rlm");
+				if (dataset === "codetracebench") await validateCodeTraceFindingEvidence({
+					trajectoryId,
+					findings,
+					store: input.traceStore,
+					...context.signal ? { signal: context.signal } : {}
+				});
+				return {
+					findings,
+					usage,
+					metadata: {
+						analysisMode: "recursive",
+						engine: "dspy-rlm",
+						protocolSha256: publicBenchmarkProtocolSha256(dataset),
+						answer: completed.answer,
+						trajectory: completed.trajectory,
+						modelCalls: completed.modelCalls,
+						toolCalls: completed.toolCalls,
+						runtime: completed.runtime
+					}
+				};
+			} catch (error) {
+				if (context.signal?.aborted) throw error;
+				if (isPaidCallControlError(error)) throw error;
+				return {
+					findings: [],
+					usage,
+					error: publicBenchmarkError(error, [config.apiKey]),
+					metadata: {
+						analysisMode: "recursive",
+						engine: "dspy-rlm",
+						rawFindings
+					}
+				};
+			}
+		}
+	};
+}
+function publicBenchmarkDefinition(dataset, limits) {
+	return {
+		id: dataset === "agentrx" ? "agentrx-dspy-rlm" : "codetracebench-dspy-rlm",
+		description: dataset === "agentrx" ? "Localizes the first unrecoverable root-cause step." : "Localizes every incorrect state-changing assistant step.",
+		area: dataset === "agentrx" ? "root-cause" : "incorrect",
+		version: "1.0.0",
+		question: dataset === "agentrx" ? "What is the first unrecoverable root cause in this failed trajectory?" : "Which assistant steps are incorrect under the CodeTraceBench definition?",
+		instructions: dataset === "agentrx" ? AGENT_RX_RLM_INSTRUCTIONS : CODE_TRACE_RLM_INSTRUCTIONS,
+		toolGroup: "singleTrace",
+		limits
+	};
+}
+function pricingForModel(model) {
+	const pricing = resolveModelPricing(model);
+	if (!pricing) throw new Error(`no pricing is configured for '${model}'; provide PublicAnalystBenchmarkModelConfig.pricing`);
+	return {
+		inputUsdPerMillion: pricing.input * 1e3,
+		outputUsdPerMillion: pricing.output * 1e3
+	};
+}
 function trajectoryIdFromCaseId(dataset, caseId) {
 	const prefix = dataset === "agentrx" ? "agentrx:" : "codetrace:";
 	if (!caseId.startsWith(prefix) || caseId.length === prefix.length) throw new Error(`unexpected ${dataset} benchmark case id '${caseId}'`);
 	return caseId.slice(prefix.length);
 }
+function isPaidCallControlError(error) {
+	return error instanceof CostAccountingIncompleteError || error instanceof CostCallConflictError || error instanceof CostCeilingReachedError || error instanceof CostLedgerPersistenceError || error instanceof CostReceiptCaptureError || error instanceof CostReservationExceededError;
+}
 //#endregion
 //#region src/analyst/benchmark-command-persistence.ts
 const ANALYST_BENCHMARK_INITIALIZATION_COMPLETE_FILE = "initialization-complete.json";
@@ -3142,7 +3341,7 @@ function createRunIdentity(config, prepared) {
 			analystProtocolSha256: publicBenchmarkProtocolSha256(config.dataset),
 			implementationSha256: ANALYST_BENCHMARK_IMPLEMENTATION_SHA256,
 			dependencyLockSha256: ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256,
-			runnerIds: ["empty", "model"]
+			runnerIds: ["empty", "dspy-rlm"]
 		},
 		inputs: {
 			labelsSha256: prepared.labelsSha256,
@@ -3347,7 +3546,7 @@ async function readProgress(path, runIdentitySha256, caseIds, repetitions) {
 			previousRowSha256: parsed.previousRowSha256,
 			observation
 		}) !== parsed.rowSha256) throw new Error(`benchmark observation row ${index + 1} digest does not match its contents`);
-		if (!allowedCases.has(observation.caseId) || observation.runnerId !== "empty" && observation.runnerId !== "model" || observation.repetition >= repetitions || observation.executionIndex >= plannedObservationCount) throw new Error(`benchmark observation row ${index + 1} does not match a planned case, runner, and repetition`);
+		if (!allowedCases.has(observation.caseId) || observation.runnerId !== "empty" && observation.runnerId !== "dspy-rlm" || observation.repetition >= repetitions || observation.executionIndex >= plannedObservationCount) throw new Error(`benchmark observation row ${index + 1} does not match a planned case, runner, and repetition`);
 		if (executionIndexes.has(observation.executionIndex)) throw new Error(`duplicate benchmark executionIndex ${observation.executionIndex} at line ${index + 1}`);
 		observations.push(observation);
 		seen.add(key);
@@ -3802,7 +4001,7 @@ function assertCompletedArtifactMatchesRun(artifact, manifest, observations, pre
 	if (canonicalJson(artifact.result.summaries) !== canonicalJson(expectedSummaries)) throw new Error("completed benchmark summaries do not match durable observations");
 	const expectedComparisons = [compareAnalystRunners(artifact.result, {
 		baselineRunnerId: "empty",
-		candidateRunnerId: "model",
+		candidateRunnerId: "dspy-rlm",
 		seed: config.seed
 	})];
 	if (canonicalJson(artifact.comparisons) !== canonicalJson(expectedComparisons)) throw new Error("completed benchmark comparisons do not match durable observations");
@@ -3951,8 +4150,8 @@ async function executeAnalystBenchmarkCommand(config, dependencies) {
 		return benchmarkExitCode(artifact.result);
 	}
 	if (await regularFileExists(paths.report)) throw new Error(`benchmark report exists without a completed result; refusing ambiguous resume: ${paths.report}`);
-	const createModelRunner = dependencies.createModelRunner ?? ((dataset, model) => createPublicBenchmarkModelRunner(dataset, model));
-	const runners = [emptyPublicBenchmarkRunner(), createModelRunner(config.dataset, {
+	const createAnalystRunner = dependencies.createAnalystRunner ?? ((dataset, model) => createPublicBenchmarkRlmRunner(dataset, model));
+	const runners = [emptyPublicBenchmarkRunner(), createAnalystRunner(config.dataset, {
 		...config.model,
 		costLedger,
 		durability: {
@@ -4016,7 +4215,7 @@ async function executeAnalystBenchmarkCommand(config, dependencies) {
 	assertSameObservations(result.observations, persisted.observations);
 	const comparisons = [compareAnalystRunners(result, {
 		baselineRunnerId: "empty",
-		candidateRunnerId: "model",
+		candidateRunnerId: "dspy-rlm",
 		seed: config.seed
 	})];
 	const codeTraceCalibration = config.dataset === "codetracebench" ? summarizeCodeTraceCalibration(result) : void 0;
@@ -4073,7 +4272,7 @@ const NON_SCORABLE_COST_ERRORS = /* @__PURE__ */ new Set([
 ]);
 function assertObservationAccountingComplete(observation, costLedger) {
 	if (observation.error && NON_SCORABLE_COST_ERRORS.has(observation.error.class)) throw new CostAccountingIncompleteError(`Analyst benchmark stopped before scoring: ${observation.error.message}`);
-	if (observation.runnerId !== "model") return;
+	if (observation.runnerId !== "dspy-rlm") return;
 	const summary = costLedger.summary({
 		channel: "analyst",
 		tags: {
@@ -4081,7 +4280,7 @@ function assertObservationAccountingComplete(observation, costLedger) {
 			benchmarkRepetition: String(observation.repetition)
 		}
 	});
-	if (!summary.accountingComplete || summary.pendingCalls > 0 || summary.unresolvedCalls > 0) throw accountingError(costLedger, "the model observation has incomplete cost accounting", {
+	if (!summary.accountingComplete || summary.pendingCalls > 0 || summary.unresolvedCalls > 0) throw accountingError(costLedger, "the recursive analyst has incomplete cost accounting", {
 		channel: "analyst",
 		tags: {
 			benchmarkCaseId: observation.caseId,
@@ -4099,7 +4298,7 @@ function accountingError(costLedger, reason, filter) {
 }
 const ANALYST_BENCHMARK_HELP = `agent-eval analyst-benchmark
 
-Run a real-model trace analyst against public AgentRx or CodeTraceBench labels.
+Run the recursive DSPy trace analyst against public AgentRx or CodeTraceBench labels.
 
 Required:
   --dataset agentrx|codetracebench
@@ -4120,6 +4319,7 @@ Controls:
   --concurrency <positive integer> Parallel benchmark jobs. Default: 1
   --repetitions <positive integer> Runs per case and runner. Default: 1
   --max-output-tokens <positive>   Model output limit per call. Default: 4096
+  --python <executable>             Python with agent-eval-rpc[dspy]. Default: python
   --timeout-ms <positive>          Model analyst deadline per case. Default: 300000
   --max-cost-usd <positive>        Run-wide spend limit. Default: 5
   --max-artifact-bytes <positive>  Final evidence bytes per case. Default: ${DEFAULT_MAX_VERIFICATION_ARTIFACT_BYTES}
@@ -4142,6 +4342,8 @@ function parseCommandConfig(argv, env) {
 	if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(apiKeyEnv)) throw new Error("--api-key-env must be a valid environment variable name");
 	const apiKey = env[apiKeyEnv]?.trim();
 	if (!apiKey) throw new Error(`--api-key-env points to an empty or missing variable: ${apiKeyEnv}`);
+	const maxCostUsd = positiveFiniteFlag(flags, "max-cost-usd", 5);
+	const python = flags.get("python")?.trim();
 	return {
 		dataset,
 		labelsPath: requiredFlag(flags, "labels"),
@@ -4155,13 +4357,15 @@ function parseCommandConfig(argv, env) {
 			apiKey,
 			model: requiredFlag(flags, "model"),
 			maxOutputTokens: positiveFlag(flags, "max-output-tokens", 4096),
-			timeoutMs: positiveFlag(flags, "timeout-ms", 3e5)
+			timeoutMs: positiveFlag(flags, "timeout-ms", 3e5),
+			maxCostUsdPerAnalysis: maxCostUsd,
+			...python ? { dspyRlm: { runner: { command: python } } } : {}
 		},
 		limit: positiveFlag(flags, "limit"),
 		seed: integerFlag(flags, "seed", 0),
 		concurrency: positiveFlag(flags, "concurrency", 1),
 		repetitions: positiveFlag(flags, "repetitions", 1),
-		maxCostUsd: positiveFiniteFlag(flags, "max-cost-usd", 5),
+		maxCostUsd,
 		maxArtifactBytes: positiveFlag(flags, "max-artifact-bytes", DEFAULT_MAX_VERIFICATION_ARTIFACT_BYTES),
 		apiKeyEnv,
 		command: `agent-eval analyst-benchmark ${argv.filter((argument) => argument !== "--resume").map(shellQuote).join(" ")}`,
@@ -4206,6 +4410,7 @@ const KNOWN_FLAGS = /* @__PURE__ */ new Set([
 	"concurrency",
 	"repetitions",
 	"max-output-tokens",
+	"python",
 	"timeout-ms",
 	"max-cost-usd",
 	"max-artifact-bytes"
@@ -4315,7 +4520,7 @@ function renderArtifactMarkdown(artifact) {
 	return `${renderAnalystBenchmarkMarkdown(artifact.result, artifact.comparisons).trimEnd()}${calibrationMarkdown}${verificationMarkdown}\n\n${renderSelectionMarkdown(artifact.inputs.selection.report)}\n`;
 }
 function benchmarkExitCode(result) {
-	return result.summaries.find((summary) => summary.runnerId === "model")?.failedRuns ? 2 : 0;
+	return result.summaries.find((summary) => summary.runnerId === "dspy-rlm")?.failedRuns ? 2 : 0;
 }
 function printSuccessSummary(artifact, paths) {
 	const failures = artifact.result.summaries.reduce((total, summary) => total + summary.failedRuns, 0);
@@ -4327,6 +4532,6 @@ function shellQuote(value) {
 	return /^[a-zA-Z0-9_./:@%+=,-]+$/.test(value) ? value : `'${value.replaceAll("'", "'\\''")}'`;
 }
 //#endregion
-export { analystBenchmarkDependencyLockDigest as A, codeTracerPredictionsToFindings as B, ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES as C, ANALYST_BENCHMARK_IMPLEMENTATION_DIGEST_ALGORITHM as D, ANALYST_BENCHMARK_EVIDENCE_PACKAGE_VERSION as E, ANALYST_BENCHMARK_OBSERVATIONS_FILE as F, normalizeBenchmarkLabel as G, agentRxPredictionsToFindings as H, AGENT_RX_UPSTREAM_REVISION as I, renderAgentRxCalibrationMarkdown as L, ANALYST_BENCHMARK_COST_LEDGER_FILE as M, ANALYST_BENCHMARK_LOCAL_RECEIPT_FILE as N, ANALYST_BENCHMARK_IMPLEMENTATION_FILES as O, ANALYST_BENCHMARK_MANIFEST_FILE as P, summarizeAgentRxCalibration as R, ANALYST_BENCHMARK_DEPENDENCY_LOCK_DIGEST_ALGORITHM as S, ANALYST_BENCHMARK_EVIDENCE_DEPENDENCY_LOCK_SHA256 as T, normalizeAgentRxCategory as U, agentRxBenchmarkCase as V, roundAgentRxStep as W, appendVerificationArtifactsToOtlp as _, renderCodeTraceCalibrationMarkdown as a, adaptPublicBenchmarkFindings as b, CODE_TRACE_BENCH_ANALYST_PROMPT as c, loadPublicBenchmarkRows as d, preparePublicAnalystBenchmark as f, DEFAULT_MAX_VERIFICATION_ARTIFACT_BYTES as g, selectPublicBenchmarkRows as h, readAnalystBenchmarkArtifact as i, analystBenchmarkImplementationDigest as j, ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 as k, createPublicBenchmarkModelRunner as l, publicBenchmarkSelectionReport as m, runAnalystBenchmarkCommand as n, summarizeCodeTraceCalibration as o, publicBenchmarkDistributions as p, renderAnalystBenchmarkMarkdown as r, compareAnalystRunners as s, ANALYST_BENCHMARK_HELP as t, publicBenchmarkProtocolSha256 as u, loadCodeTraceVerificationArtifacts as v, ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 as w, emptyPublicBenchmarkRunner as x, parseVerificationOutcome as y, codeTraceBenchCase as z };
+export { ANALYST_BENCHMARK_IMPLEMENTATION_FILES as A, summarizeAgentRxCalibration as B, ANALYST_BENCHMARK_DEPENDENCY_LOCK_DIGEST_ALGORITHM as C, ANALYST_BENCHMARK_EVIDENCE_IMPLEMENTATION_SHA256 as D, ANALYST_BENCHMARK_EVIDENCE_DEPENDENCY_LOCK_SHA256 as E, ANALYST_BENCHMARK_LOCAL_RECEIPT_FILE as F, normalizeAgentRxCategory as G, codeTracerPredictionsToFindings as H, ANALYST_BENCHMARK_MANIFEST_FILE as I, roundAgentRxStep as K, ANALYST_BENCHMARK_OBSERVATIONS_FILE as L, analystBenchmarkDependencyLockDigest as M, analystBenchmarkImplementationDigest as N, ANALYST_BENCHMARK_EVIDENCE_PACKAGE_VERSION as O, ANALYST_BENCHMARK_COST_LEDGER_FILE as P, AGENT_RX_UPSTREAM_REVISION as R, emptyPublicBenchmarkRunner as S, ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 as T, agentRxBenchmarkCase as U, codeTraceBenchCase as V, agentRxPredictionsToFindings as W, DEFAULT_MAX_VERIFICATION_ARTIFACT_BYTES as _, renderCodeTraceCalibrationMarkdown as a, parseVerificationOutcome as b, createPublicBenchmarkRlmRunner as c, publicBenchmarkProtocolSha256 as d, loadPublicBenchmarkRows as f, selectPublicBenchmarkRows as g, publicBenchmarkSelectionReport as h, readAnalystBenchmarkArtifact as i, ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 as j, ANALYST_BENCHMARK_IMPLEMENTATION_DIGEST_ALGORITHM as k, CODE_TRACE_BENCH_ANALYST_PROMPT as l, publicBenchmarkDistributions as m, runAnalystBenchmarkCommand as n, summarizeCodeTraceCalibration as o, preparePublicAnalystBenchmark as p, normalizeBenchmarkLabel as q, renderAnalystBenchmarkMarkdown as r, compareAnalystRunners as s, ANALYST_BENCHMARK_HELP as t, createPublicBenchmarkDirectRunner as u, appendVerificationArtifactsToOtlp as v, ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES as w, adaptPublicBenchmarkFindings as x, loadCodeTraceVerificationArtifacts as y, renderAgentRxCalibrationMarkdown as z };
 
-//# sourceMappingURL=benchmark-command-CMqVqReF.js.map
+//# sourceMappingURL=benchmark-command-BKfjOBJ5.js.map