@tangle-network/agent-eval 0.143.0 → 0.144.1

package/dist/{benchmark-command-CSddvlq7.js → benchmark-command-95Zqd1e8.js}

@@ -1,12 +1,12 @@
 import { c as ValidationError, t as AgentEvalError } from "./errors-D-LKuDhb.js";
 import { s as resolveModelPricing } from "./metrics-C9YY1OcL.js";
 import { a as CostLedgerPersistenceError, i as CostLedger, n as CostCallConflictError, o as CostReceiptCaptureError, r as CostCeilingReachedError, s as CostReservationExceededError, t as CostAccountingIncompleteError } from "./cost-ledger-DMFxsLKr.js";
-import { c as callLlmJson, f as maximumChargeForLlmRequest, l as costReceiptFromLlm, r as LlmResponseError, t as LlmCallError, u as costReceiptFromLlmError } from "./llm-client-B3WXSH5Y.js";
+import { c as callLlmJson, r as LlmResponseError, t as LlmCallError } from "./llm-client-D3EoChAU.js";
 import { O as evidenceRefsFromRawFinding, h as TRACE_ANALYSIS_LIMITS, i as runTraceAnalyst } from "./kind-factory-Bvwe3pup.js";
 import { o as makeFinding, r as usageReceiptFromCostLedger } from "./usage-receipt-CgxMEBZq.js";
 import { i as hashCanonical, r as canonicalString } from "./canonical-D011XM8r.js";
-import { n as createRunCostLedger, r as fsCampaignStorage, t as acquireSingleRunLock } from "./single-run-lock-Do-8vBIJ.js";
-import { t as createDspyRlmTraceEngine } from "./dspy-rlm-engine-CJ4sN8Wx.js";
+import { M as resolveExternalOptimizerProcessLimits, f as runWithCleanup, n as createRunCostLedger, p as startExternalOptimizerModelProxy, r as fsCampaignStorage, t as acquireSingleRunLock } from "./single-run-lock-B-cFnnNn.js";
+import { t as createDspyRlmTraceEngine } from "./dspy-rlm-engine-BR7TtB0o.js";
 import { c as writeLedgerFileAtomically, s as withLedgerFileLock } from "./ledger-core-Dxz0Rkwa.js";
 import { E as pairedBootstrap } from "./statistics-ByxzSiOM.js";
 import { i as otlpTextToTraceAnalysisStore, r as createOtlpBufferTraceStore, t as DEFAULT_MAX_TRACE_FILE_BYTES } from "./store-otlp-D4I90_vR.js";
@@ -19,6 +19,7 @@ import { createHash, randomUUID } from "node:crypto";
 import { link, lstat, mkdir, open, readFile, readdir, realpath, stat, unlink } from "node:fs/promises";
 import { arch, platform } from "node:os";
 import { TextDecoder as TextDecoder$1 } from "node:util";
+import { pathToFileURL } from "node:url";
 //#region src/analyst/benchmark-dataset-utils.ts
 function normalizeBenchmarkLabel(value) {
 	const normalized = nonEmpty$1(value, "benchmark label").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "");
@@ -1070,8 +1071,34 @@ const artifactSchema = z.strictObject({
 			concurrency: positiveInteger$1,
 			rlmSamples: positiveInteger$1.optional(),
 			model: nonEmptyString,
+			modelOwnerCallRef: nonEmptyString.optional(),
 			maxOutputTokens: positiveInteger$1,
+			maxReasoningTokens: nonNegativeInteger.optional(),
+			maxModelRequestBytes: positiveInteger$1.optional(),
+			maxModelResponseBytes: positiveInteger$1.optional(),
+			modelRequestTimeoutMs: positiveInteger$1.optional(),
 			timeoutMs: positiveInteger$1,
+			pricing: z.strictObject({
+				inputUsdPerMillion: nonNegativeNumber,
+				cachedInputUsdPerMillion: nonNegativeNumber.optional(),
+				cacheWriteUsdPerMillion: nonNegativeNumber.optional(),
+				outputUsdPerMillion: nonNegativeNumber
+			}).optional(),
+			recursiveLimits: z.strictObject({
+				maxIterations: positiveInteger$1,
+				maxLlmCalls: positiveInteger$1,
+				maxToolCalls: positiveInteger$1,
+				maxOutputChars: positiveInteger$1,
+				maxModelRequests: positiveInteger$1.nullable(),
+				traceToolRequestBytes: positiveInteger$1,
+				traceToolResponseBytes: positiveInteger$1,
+				traceToolTimeoutMs: positiveInteger$1
+			}).optional(),
+			processLimits: z.strictObject({
+				maxInputBytes: positiveInteger$1,
+				maxResultBytes: positiveInteger$1,
+				maxOutputChars: positiveInteger$1
+			}).optional(),
 			maxCostUsd: nonNegativeNumber,
 			maxArtifactBytes: positiveInteger$1,
 			analystProtocolSha256: sha256,
@@ -1173,7 +1200,7 @@ const ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES = Object.freeze([
 	"package.json",
 	"pnpm-lock.yaml"
 ]);
-const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = "2867725e505a600f4f1f9d934fb797f5fb94a71c906577d05009aecc254f85f3";
+const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = "044bf5b84d7902c43e627dd302dad22332f8787aa5ae70b72f52fbf9db667d93";
 /** The published benchmark evidence was produced at this package version, by
 * the retired one-shot direct runner, before trace analysts moved to the
 * recursive DSPy RLM engine. Both evidence digests below are historical facts
@@ -1228,9 +1255,11 @@ const ANALYST_BENCHMARK_IMPLEMENTATION_FILES = Object.freeze([
 	"src/analyst/trace-tool-callback.ts",
 	"src/analyst/types.ts",
 	"src/analyst/usage-receipt.ts",
+	"src/campaign/external-optimizer-callback.ts",
 	"src/campaign/external-optimizer-contracts.ts",
 	"src/campaign/external-optimizer-http.ts",
 	"src/campaign/external-optimizer-model-proxy.ts",
+	"src/campaign/external-optimizer-process.ts",
 	"src/campaign/external-optimizer-resources.ts",
 	"src/campaign/external-optimizer-subprocess.ts",
 	"src/campaign/search-ledger-errors.ts",
@@ -1267,9 +1296,10 @@ const ANALYST_BENCHMARK_IMPLEMENTATION_FILES = Object.freeze([
 	"src/trace-analyst/types.ts",
 	"src/trace/attribute-vocabulary.ts",
 	"src/trace/otlp-attributes.ts",
-	"src/trace/raw-provider-sink.ts"
+	"src/trace/raw-provider-sink.ts",
+	"src/verdict-cache.ts"
 ]);
-const ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 = "b8b6d4f7a01f8341a0565159806f93ff9ca71c88322e63e4e52a86eed2c1d9c7";
+const ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 = "d317d9e7593ddc13f22a4a1e3d62d2fa1e4591395ef3984a0c7585a75794cf8d";
 function analystBenchmarkImplementationDigest() {
 	return ANALYST_BENCHMARK_IMPLEMENTATION_SHA256;
 }
@@ -2327,6 +2357,7 @@ async function prepareOutputLockPath(outDir) {
 	return `${directory}.lock`;
 }
 function createRunIdentity(config, prepared) {
+	const model = commandModelIdentity(config.model);
 	const caseDefinitions = prepared.cases.map((testCase) => ({
 		id: testCase.id,
 		clusterId: testCase.clusterId,
@@ -2343,8 +2374,7 @@ function createRunIdentity(config, prepared) {
 			datasetSplit: config.split,
 			model: {
 				id: config.model.model,
-				maxOutputTokens: config.model.maxOutputTokens,
-				timeoutMs: config.model.timeoutMs
+				...model
 			},
 			limit: config.limit,
 			seed: config.seed,
@@ -2369,6 +2399,36 @@ function createRunIdentity(config, prepared) {
 		}
 	};
 }
+function commandModelIdentity(config) {
+	const catalogPricing = resolveModelPricing(config.model);
+	const pricing = config.pricing ?? (catalogPricing ? {
+		inputUsdPerMillion: catalogPricing.input * 1e3,
+		outputUsdPerMillion: catalogPricing.output * 1e3
+	} : void 0);
+	if (!pricing) throw new Error(`benchmark model '${config.model}' has no recorded pricing`);
+	const recursive = config.dspyRlm;
+	return {
+		ownerCallRef: config.callRef,
+		maxOutputTokens: config.maxOutputTokens,
+		maxReasoningTokens: config.maxReasoningTokens ?? config.maxOutputTokens * 4,
+		maxRequestBytes: config.maxModelRequestBytes ?? 16 * 1024 * 1024,
+		maxResponseBytes: config.maxModelResponseBytes ?? 4 * 1024 * 1024,
+		requestTimeoutMs: config.modelRequestTimeoutMs ?? config.timeoutMs,
+		timeoutMs: config.timeoutMs,
+		pricing: { ...pricing },
+		recursiveLimits: {
+			maxIterations: recursive?.maxIterations ?? 14,
+			maxLlmCalls: recursive?.maxLlmCalls ?? 8,
+			maxToolCalls: recursive?.maxToolCalls ?? 80,
+			maxOutputChars: recursive?.maxOutputChars ?? 8e3,
+			maxModelRequests: recursive?.maxModelRequests ?? null,
+			traceToolRequestBytes: recursive?.traceToolRequestBytes ?? 1e6,
+			traceToolResponseBytes: recursive?.traceToolResponseBytes ?? 4e6,
+			traceToolTimeoutMs: recursive?.traceToolTimeoutMs ?? 6e4
+		},
+		processLimits: resolveExternalOptimizerProcessLimits(recursive?.runner?.limits)
+	};
+}
 function createLocalRunReceipt(config, paths) {
 	return {
 		kind: "agent-eval/analyst-benchmark-local-run",
@@ -2377,8 +2437,7 @@ function createLocalRunReceipt(config, paths) {
 			traceDir: resolve(config.traceDir),
 			...config.artifactDir ? { artifactDir: resolve(config.artifactDir) } : {},
 			outputDir: paths.directory,
-			baseUrl: config.model.baseUrl,
-			apiKeyEnvironment: config.apiKeyEnv
+			modelOwnerModule: config.modelOwnerModule
 		},
 		command: config.command,
 		environment: {
@@ -2450,7 +2509,7 @@ async function readAndValidateResumeFiles(paths, currentIdentity, currentIdentit
 		runIdentitySha256: currentIdentitySha256,
 		localIdentitySha256: currentLocalIdentitySha256
 	};
-	if (digestCanonical(value.local) !== currentLocalIdentitySha256 || canonicalJson(value.local) !== canonicalJson(localReceiptInput.local)) throw new Error("benchmark local paths or endpoint do not match the requested resume");
+	if (digestCanonical(value.local) !== currentLocalIdentitySha256 || canonicalJson(value.local) !== canonicalJson(localReceiptInput.local)) throw new Error("benchmark local paths or model-owner module do not match the requested resume");
 	if (localReceiptContent !== `${JSON.stringify(expectedLocalReceipt, null, 2)}\n`) throw new Error(`benchmark local run receipt does not exactly match: ${paths.localReceipt}`);
 	if (await readRegularFile(paths.initializationComplete, "benchmark initialization marker") !== renderInitializationComplete(manifest)) throw new Error(`benchmark initialization marker does not match the run manifest: ${paths.initializationComplete}`);
 	return manifest;
@@ -2996,8 +3055,16 @@ function assertCompletedArtifactMatchesRun(artifact, manifest, observations, pre
 			concurrency: config.concurrency,
 			...config.rlmSamples === void 0 ? {} : { rlmSamples: config.rlmSamples },
 			model: config.model.id,
+			modelOwnerCallRef: config.model.ownerCallRef,
 			maxOutputTokens: config.model.maxOutputTokens,
+			maxReasoningTokens: config.model.maxReasoningTokens,
+			maxModelRequestBytes: config.model.maxRequestBytes,
+			maxModelResponseBytes: config.model.maxResponseBytes,
+			modelRequestTimeoutMs: config.model.requestTimeoutMs,
 			timeoutMs: config.model.timeoutMs,
+			pricing: config.model.pricing,
+			recursiveLimits: config.model.recursiveLimits,
+			processLimits: config.model.processLimits,
 			maxCostUsd: config.maxCostUsd,
 			maxArtifactBytes: config.maxArtifactBytes,
 			analystProtocolSha256: config.analystProtocolSha256,
@@ -3014,7 +3081,7 @@ function assertCompletedArtifactMatchesRun(artifact, manifest, observations, pre
 		id: expectedDatasetId,
 		revision: config.datasetRevision,
 		split: config.datasetSplit
-	}) || provenance.caseCount !== inputs.selectedCaseIds.length || canonicalJson(provenance.runnerIds) !== canonicalJson(config.runnerIds) || provenance.repetitions !== config.repetitions || provenance.maxConcurrency !== Math.min(config.concurrency, expectedCount) || provenance.runnerOrderSeed !== config.seed || provenance.metadata?.model !== config.model.id || provenance.metadata?.rlmSamples !== config.rlmSamples || provenance.metadata?.outputAdapter !== expectedOutputAdapter || provenance.metadata?.caseSelection !== prepared.selection.method || provenance.metadata?.caseSelectionSeed !== config.seed || provenance.metadata?.selectionStratified !== prepared.selection.stratified || provenance.metadata?.protocolSha256 !== config.analystProtocolSha256 || provenance.metadata?.implementationSha256 !== config.implementationSha256 || provenance.metadata?.dependencyLockSha256 !== config.dependencyLockSha256 || provenance.metadata?.populationRepresentativenessProven !== false) throw new Error("completed benchmark result provenance does not match the run manifest");
+	}) || provenance.caseCount !== inputs.selectedCaseIds.length || canonicalJson(provenance.runnerIds) !== canonicalJson(config.runnerIds) || provenance.repetitions !== config.repetitions || provenance.maxConcurrency !== Math.min(config.concurrency, expectedCount) || provenance.runnerOrderSeed !== config.seed || provenance.metadata?.model !== config.model.id || provenance.metadata?.modelOwnerCallRef !== config.model.ownerCallRef || provenance.metadata?.rlmSamples !== config.rlmSamples || provenance.metadata?.outputAdapter !== expectedOutputAdapter || provenance.metadata?.caseSelection !== prepared.selection.method || provenance.metadata?.caseSelectionSeed !== config.seed || provenance.metadata?.selectionStratified !== prepared.selection.stratified || provenance.metadata?.protocolSha256 !== config.analystProtocolSha256 || provenance.metadata?.implementationSha256 !== config.implementationSha256 || provenance.metadata?.dependencyLockSha256 !== config.dependencyLockSha256 || provenance.metadata?.populationRepresentativenessProven !== false) throw new Error("completed benchmark result provenance does not match the run manifest");
 	const expectedSummaries = config.runnerIds.map((runnerId) => summarizeAnalystBenchmarkRunner(runnerId, observations.filter((observation) => observation.runnerId === runnerId)));
 	if (canonicalJson(artifact.result.summaries) !== canonicalJson(expectedSummaries)) throw new Error("completed benchmark summaries do not match durable observations");
 	const expectedComparisons = [compareAnalystRunners(artifact.result, {
@@ -3520,10 +3587,17 @@ function fileContext() {
 function createPublicBenchmarkDirectRunner(dataset, config) {
 	if (config.instructionsOverride) throw new Error("the direct runner executes only the stock protocol; an instructions override requires the dspy-rlm runner");
 	const model = requiredString(config.model, "model");
-	const baseUrl = requiredString(config.baseUrl, "baseUrl");
-	const apiKey = requiredString(config.apiKey, "apiKey");
+	const callRef = requiredString(config.callRef, "callRef");
+	if (typeof config.call !== "function") throw new TypeError("call must be a function");
+	if (typeof config.recordExecution !== "function") throw new TypeError("recordExecution must be a function");
 	const maxOutputTokens = positiveSafeInteger(config.maxOutputTokens, "maxOutputTokens");
 	const timeoutMs = positiveSafeInteger(config.timeoutMs, "timeoutMs");
+	const maxReasoningTokens = config.maxReasoningTokens ?? maxOutputTokens * 4;
+	const maxModelRequestBytes = config.maxModelRequestBytes ?? 16 * 1024 * 1024;
+	const maxModelResponseBytes = config.maxModelResponseBytes ?? 4 * 1024 * 1024;
+	const modelRequestTimeoutMs = config.modelRequestTimeoutMs ?? timeoutMs;
+	const pricing = config.pricing ?? pricingForModel$1(model);
+	const maxCostUsd = config.maxCostUsdPerAnalysis ?? 1;
 	const costLedger = config.costLedger ?? new CostLedger();
 	const durability = config.durability ? {
 		runIdentitySha256: requiredString(config.durability.runIdentitySha256, "durability.runIdentitySha256"),
@@ -3531,15 +3605,6 @@ function createPublicBenchmarkDirectRunner(dataset, config) {
 	} : void 0;
 	const actor = dataset === "agentrx" ? "agentrx-root-cause-localizer" : "codetracebench-step-localizer";
 	const outputAdapter = dataset === "agentrx" ? "agentrx-taxonomy-and-root-step" : "codetracebench-incorrect-block";
-	const llmOptions = {
-		baseUrl,
-		apiKey,
-		maximumAttempts: 1,
-		jsonSchemaTransport: "json-object",
-		jsonPayloadMode: "exact",
-		thinking: "disabled",
-		...config.fetchImpl ? { fetch: config.fetchImpl } : {}
-	};
 	return {
 		id: "direct",
 		async analyze(input, context) {
@@ -3557,7 +3622,8 @@ function createPublicBenchmarkDirectRunner(dataset, config) {
 			let modelMetadata = {
 				analysisMode: "direct-baseline",
 				outputAdapter,
-				protocolSha256: publicBenchmarkProtocolSha256(dataset)
+				protocolSha256: publicBenchmarkProtocolSha256(dataset),
+				callRef
 			};
 			try {
 				if (!input.traceStore) throw new Error(`${dataset} model runner requires a trace store`);
@@ -3575,7 +3641,7 @@ function createPublicBenchmarkDirectRunner(dataset, config) {
 					jsonMode: true,
 					thinking: "disabled",
 					maxTokens: maxOutputTokens,
-					timeoutMs
+					timeoutMs: modelRequestTimeoutMs
 				};
 				const cacheIdentity = durability ? {
 					runIdentitySha256: durability.runIdentitySha256,
@@ -3614,26 +3680,51 @@ function createPublicBenchmarkDirectRunner(dataset, config) {
 					};
 				} else {
 					assertNoSettledResponseWithoutCache(costLedger, callId);
-					let completedResult;
-					const paid = await costLedger.runPaidCall({
-						...callId ? { callId } : {},
-						channel: "analyst",
-						phase: "analyst.public-benchmark",
-						actor,
-						model,
-						signal: context.signal,
-						maximumCharge: maximumChargeForLlmRequest(request, llmOptions),
-						tags: costTags,
-						execute: async (signal, providerCallId) => {
+					const providerCallId = callId ?? `analyst-benchmark-${randomUUID()}`;
+					let modelProxy;
+					const completed = await runWithCleanup({
+						label: "public benchmark direct model resources",
+						run: async () => {
+							modelProxy = await startExternalOptimizerModelProxy({
+								call: config.call,
+								callRef,
+								recordExecution: config.recordExecution,
+								model,
+								budget: {
+									maxCostUsd,
+									maxRequests: 1,
+									maxRequestBytes: maxModelRequestBytes,
+									maxResponseBytes: maxModelResponseBytes,
+									maxOutputTokensPerRequest: maxOutputTokens,
+									maxReasoningTokensPerRequest: maxReasoningTokens,
+									pricing,
+									requestTimeoutMs: modelRequestTimeoutMs
+								},
+								costLedger,
+								channel: "analyst",
+								phase: "analyst.public-benchmark",
+								actor,
+								tags: costTags,
+								callId: providerCallId,
+								...context.signal ? { signal: context.signal } : {}
+							});
+							const llmOptions = {
+								baseUrl: modelProxy.baseUrl,
+								apiKey: modelProxy.apiKey,
+								maximumAttempts: 1,
+								jsonSchemaTransport: "json-object",
+								jsonPayloadMode: "exact",
+								thinking: "disabled"
+							};
 							try {
 								const completed = await callLlmJson(request, {
 									...llmOptions,
-									signal,
+									...context.signal ? { signal: context.signal } : {},
 									idempotencyKey: providerCallId
 								});
-								completedResult = completed.result;
 								const response = parsePublicBenchmarkModelResponse(dataset, completed.value);
 								const responseProducedAt = (/* @__PURE__ */ new Date()).toISOString();
+								const receipt = requiredSettledReceipt(costLedger, providerCallId);
 								if (cacheIdentity) writePublicBenchmarkResponseCache(durability.responseCacheDir, {
 									kind: "agent-eval/public-benchmark-model-response",
 									...cacheIdentity,
@@ -3646,42 +3737,49 @@ function createPublicBenchmarkDirectRunner(dataset, config) {
 										finishReason: completed.result.finishReason ?? null,
 										producedAt: responseProducedAt
 									},
-									receipt: costReceiptFromLlm(completed.result)
+									receipt: cacheReceiptInput(receipt)
 								});
+								modelProxy.assertExecutionComplete();
 								return {
 									...completed,
 									response,
-									producedAt: responseProducedAt
+									producedAt: responseProducedAt,
+									receipt
 								};
 							} catch (error) {
-								if (cacheIdentity) writePublicBenchmarkResponseCache(durability.responseCacheDir, {
-									kind: "agent-eval/public-benchmark-model-response",
-									...cacheIdentity,
-									callId: providerCallId,
-									status: "failed",
-									error: publicBenchmarkError(error, [apiKey]),
-									receipt: receiptForProviderFailure(error, completedResult, model)
-								});
+								const controlFailure = modelProxy.failures().find(isPaidCallControlError$1);
+								if (controlFailure) throw controlFailure;
+								const receipt = settledReceipt(costLedger, providerCallId);
+								if (cacheIdentity) {
+									if (receipt) writePublicBenchmarkResponseCache(durability.responseCacheDir, {
+										kind: "agent-eval/public-benchmark-model-response",
+										...cacheIdentity,
+										callId: providerCallId,
+										status: "failed",
+										error: publicBenchmarkError(error, []),
+										receipt: cacheReceiptInput(receipt)
+									});
+								}
 								throw error;
 							}
 						},
-						receipt: ({ result }) => costReceiptFromLlm(result),
-						receiptFromError: (error) => receiptForProviderFailure(error, completedResult, model)
+						cleanup: async () => {
+							await modelProxy?.close();
+						}
 					});
-					if (!paid.succeeded) throw paid.error;
-					const response = paid.value.response;
+					const response = completed.response;
 					rawPredictions = response.findings;
 					rejectedBlocks = response.rejectedBlocks;
-					providerModel = paid.value.result.model;
-					producedAt = paid.value.producedAt;
+					providerModel = completed.result.model;
+					producedAt = completed.producedAt;
 					modelMetadata = {
 						...modelMetadata,
 						responseSource: "provider",
 						report: response.report,
-						providerModel: paid.value.result.model,
-						providerDurationMs: paid.value.result.durationMs,
-						finishReason: paid.value.result.finishReason ?? null,
-						cost: costReceiptMetadata(paid.receipt)
+						providerModel: completed.result.model,
+						providerDurationMs: completed.result.durationMs,
+						finishReason: completed.result.finishReason ?? null,
+						cost: costReceiptMetadata(completed.receipt)
 					};
 				}
 				const converted = await publicBenchmarkPredictionsToFindings({
@@ -3725,7 +3823,7 @@ function createPublicBenchmarkDirectRunner(dataset, config) {
 						channel: "analyst",
 						tags: costTags
 					}),
-					error: publicBenchmarkError(error, [apiKey]),
+					error: publicBenchmarkError(error, []),
 					metadata: {
 						...modelMetadata,
 						rawPredictions,
@@ -3759,18 +3857,48 @@ function assertCacheReceiptMatches(cached, receipt) {
 function isPaidCallControlError$1(error) {
 	return error instanceof CostAccountingIncompleteError || error instanceof CostCallConflictError || error instanceof CostCeilingReachedError || error instanceof CostLedgerPersistenceError || error instanceof CostReceiptCaptureError || error instanceof CostReservationExceededError;
 }
-function receiptForProviderFailure(error, completedResult, model) {
-	if (completedResult) return costReceiptFromLlm(completedResult);
-	if (error instanceof Error) {
-		const captured = costReceiptFromLlmError(error);
-		if (captured) return captured;
-	}
+function settledReceipt(costLedger, callId) {
+	return costLedger.list().find((receipt) => receipt.callId === callId);
+}
+function requiredSettledReceipt(costLedger, callId) {
+	const receipt = settledReceipt(costLedger, callId);
+	if (!receipt) throw new CostAccountingIncompleteError(`caller-owned model call '${callId}' produced no cost receipt`);
+	return receipt;
+}
+function cacheReceiptInput(receipt) {
+	const usage = {
+		model: receipt.model,
+		inputTokens: receipt.inputTokens,
+		outputTokens: receipt.outputTokens,
+		...receipt.reasoningTokens === void 0 ? {} : { reasoningTokens: receipt.reasoningTokens },
+		...receipt.cachedTokens === void 0 ? {} : { cachedTokens: receipt.cachedTokens },
+		...receipt.cacheWriteTokens === void 0 ? {} : { cacheWriteTokens: receipt.cacheWriteTokens },
+		...receipt.usageUnknown === void 0 ? {} : { usageUnknown: receipt.usageUnknown }
+	};
+	if (receipt.costUnknown) return {
+		...usage,
+		costUnknown: true
+	};
+	if (receipt.actualCostUsd !== void 0) return {
+		...usage,
+		actualCostUsd: receipt.actualCostUsd
+	};
+	if (receipt.estimatedCostUsd !== void 0) return {
+		...usage,
+		estimatedCostUsd: receipt.estimatedCostUsd
+	};
+	if (receipt.pricing) return {
+		...usage,
+		customTokenPricing: {
+			inputUsdPerMillion: receipt.pricing.inputUsdPerThousand * 1e3,
+			...receipt.pricing.cachedInputUsdPerThousand === void 0 ? {} : { cachedInputUsdPerMillion: receipt.pricing.cachedInputUsdPerThousand * 1e3 },
+			...receipt.pricing.cacheWriteUsdPerThousand === void 0 ? {} : { cacheWriteUsdPerMillion: receipt.pricing.cacheWriteUsdPerThousand * 1e3 },
+			outputUsdPerMillion: receipt.pricing.outputUsdPerThousand * 1e3
+		}
+	};
 	return {
-		model,
-		inputTokens: 0,
-		outputTokens: 0,
-		costUnknown: true,
-		usageUnknown: true
+		...usage,
+		estimatedCostUsd: receipt.costUsd
 	};
 }
 function costReceiptMetadata(receipt) {
@@ -3792,6 +3920,14 @@ function costReceiptMetadata(receipt) {
 		estimatedCostUsd: null
 	};
 }
+function pricingForModel$1(model) {
+	const pricing = resolveModelPricing(model);
+	if (!pricing) throw new Error(`no pricing is configured for '${model}'; provide PublicAnalystBenchmarkModelConfig.pricing`);
+	return {
+		inputUsdPerMillion: pricing.input * 1e3,
+		outputUsdPerMillion: pricing.output * 1e3
+	};
+}
 const ModelSeveritySchema = z.enum([
 	"critical",
 	"high",
@@ -4111,13 +4247,24 @@ function createPublicBenchmarkRlmRunner(dataset, config) {
 	};
 	const pricing = config.pricing ?? pricingForModel(config.model);
 	const engine = createDspyRlmTraceEngine({
-		baseUrl: config.baseUrl,
-		apiKey: config.apiKey,
+		call: config.call,
+		callRef: config.callRef,
+		recordExecution: config.recordExecution,
 		model: config.model,
 		maxOutputTokens: config.maxOutputTokens,
 		timeoutMs: config.timeoutMs,
 		maxCostUsd: config.maxCostUsdPerAnalysis ?? 1,
 		pricing,
+		...config.maxReasoningTokens === void 0 ? {} : { maxReasoningTokens: config.maxReasoningTokens },
+		...config.maxModelRequestBytes === void 0 ? {} : { maxModelRequestBytes: config.maxModelRequestBytes },
+		...config.maxModelResponseBytes === void 0 ? {} : { maxModelResponseBytes: config.maxModelResponseBytes },
+		...config.modelRequestTimeoutMs === void 0 ? {} : { modelRequestTimeoutMs: config.modelRequestTimeoutMs },
+		...config.dspyRlm?.maxModelRequests === void 0 ? {} : { maxModelRequests: config.dspyRlm.maxModelRequests },
+		...config.dspyRlm?.traceToolRequestBytes === void 0 && config.dspyRlm?.traceToolResponseBytes === void 0 ? {} : { traceToolLimits: {
+			...config.dspyRlm?.traceToolRequestBytes === void 0 ? {} : { maxRequestBytes: config.dspyRlm.traceToolRequestBytes },
+			...config.dspyRlm?.traceToolResponseBytes === void 0 ? {} : { maxResponseBytes: config.dspyRlm.traceToolResponseBytes }
+		} },
+		...config.dspyRlm?.traceToolTimeoutMs === void 0 ? {} : { traceToolTimeoutMs: config.dspyRlm.traceToolTimeoutMs },
 		...config.dspyRlm?.runner ? { runner: config.dspyRlm.runner } : {}
 	});
 	const instructions = config.instructionsOverride?.text ?? publicBenchmarkRlmInstructions(dataset);
@@ -4328,7 +4475,7 @@ function createPublicBenchmarkRlmRunner(dataset, config) {
 				return {
 					findings: [],
 					usage,
-					error: publicBenchmarkError(error, [config.apiKey]),
+					error: publicBenchmarkError(error, []),
 					metadata: {
 						analysisMode: "recursive",
 						engine: "dspy-rlm",
@@ -4786,7 +4933,7 @@ async function runAnalystBenchmarkCommand(argv, env = process.env, dependencies
 		process.stdout.write(`${ANALYST_BENCHMARK_HELP}\n`);
 		return 0;
 	}
-	const config = parseCommandConfig(argv, env);
+	const config = await parseCommandConfig(argv, env, dependencies);
 	const outputLock = acquireSingleRunLock({ lockPath: await prepareOutputLockPath(config.outDir) });
 	try {
 		return await executeAnalystBenchmarkCommand(config, dependencies);
@@ -4869,6 +5016,7 @@ async function executeAnalystBenchmarkCommand(config, dependencies) {
 				},
 				metadata: {
 					model: config.model.model,
+					modelOwnerCallRef: config.model.callRef,
 					rlmSamples: config.rlmSamples,
 					outputAdapter: config.dataset === "agentrx" ? "agentrx-taxonomy-and-root-step" : "codetracebench-incorrect-block",
 					caseSelection: prepared.selection.method,
@@ -4920,8 +5068,16 @@ async function executeAnalystBenchmarkCommand(config, dependencies) {
 				concurrency: config.concurrency,
 				rlmSamples: config.rlmSamples,
 				model: config.model.model,
-				maxOutputTokens: config.model.maxOutputTokens,
-				timeoutMs: config.model.timeoutMs,
+				modelOwnerCallRef: manifest.identity.config.model.ownerCallRef,
+				maxOutputTokens: manifest.identity.config.model.maxOutputTokens,
+				maxReasoningTokens: manifest.identity.config.model.maxReasoningTokens,
+				maxModelRequestBytes: manifest.identity.config.model.maxRequestBytes,
+				maxModelResponseBytes: manifest.identity.config.model.maxResponseBytes,
+				modelRequestTimeoutMs: manifest.identity.config.model.requestTimeoutMs,
+				timeoutMs: manifest.identity.config.model.timeoutMs,
+				pricing: manifest.identity.config.model.pricing,
+				recursiveLimits: manifest.identity.config.model.recursiveLimits,
+				processLimits: manifest.identity.config.model.processLimits,
 				maxCostUsd: config.maxCostUsd,
 				maxArtifactBytes: config.maxArtifactBytes,
 				analystProtocolSha256: effectiveAnalystProtocolSha256(config.dataset, config.model.instructionsOverride),
@@ -4988,16 +5144,15 @@ Run the recursive DSPy trace analyst against public AgentRx or CodeTraceBench la
 Required:
   --dataset agentrx|codetracebench
   --analyst dspy-rlm|direct        Scored analyst. Default: dspy-rlm.
-                                   'direct' is the retired one-shot runner that
-                                   produced the published evidence.
+                                   'direct' is the one-shot comparison arm.
   --labels <dataset.json|dataset.jsonl>
   --trace-dir <one-trace-per-file OTLP JSONL directory>
   --artifact-dir <extracted artifact root>  Required for CodeTraceBench
   --out <new output directory>
   --revision <full 40- or 64-character hex digest>
   --split <dataset split>
-  --base-url <OpenAI-compatible /v1 URL>
-  --api-key-env <environment variable containing the bearer>
+  --model-owner-module <module>   Module exporting createModelExecutionOwner;
+                                   the owner keeps provider credentials and policy
   --model <provider model id>
   --limit <positive case count>
 
@@ -5015,6 +5170,22 @@ Controls:
                                    protocol to the override text, and
                                    result.json records instructionsOverrideSha256.
   --max-output-tokens <positive>   Model output limit per call. Default: 16384
+  --max-reasoning-tokens <integer> Reasoning-token limit per call. Default: 65536
+  --max-model-requests <positive>  Caller-owned model calls per analysis.
+                                   Default: max iterations + model calls + 1
+  --max-model-request-bytes <positive>   Default: 16777216
+  --max-model-response-bytes <positive>  Default: 4194304
+  --model-request-timeout-ms <positive>  Default: --timeout-ms
+  --max-iterations <positive>      Recursive iterations per analysis. Default: 14
+  --max-llm-calls <positive>       DSPy model calls per analysis. Default: 8
+  --max-tool-calls <positive>      Trace-tool calls per analysis. Default: 80
+  --max-analysis-output-chars <positive> Default: 8000
+  --trace-tool-request-bytes <positive>  Default: 1000000
+  --trace-tool-response-bytes <positive> Default: 4000000
+  --trace-tool-timeout-ms <positive>     Default: 60000
+  --max-process-input-bytes <positive>   Default: 67108864
+  --max-process-result-bytes <positive>  Default: 4194304
+  --max-process-output-chars <positive>  Default: 64000
   --python <executable>             Python with agent-eval-rpc[dspy]. Default: python
   --timeout-ms <positive>          Model analyst deadline per case. Default: 300000
   --max-cost-usd <positive>        Run-wide spend limit. Default: 5
@@ -5024,20 +5195,15 @@ Writes result.json with every observation, metric, usage field, error, compariso
 input digest, artifact digest, case distribution, selected case id, and explicit
 unknown cost. Limited deterministic-hash subsets are marked non-representative.
 Completed observations are fsynced to observations.jsonl. Shareable output is in
-result.json and report.md. Machine-local paths, endpoint, and command are isolated
-in run.local.json.
-The key is read from the named environment variable and is never written.`;
-function parseCommandConfig(argv, env) {
+result.json and report.md. Machine-local paths, execution-owner module, and command
+are isolated in run.local.json. Provider credentials never enter this command.`;
+async function parseCommandConfig(argv, env, dependencies) {
 	const flags = parseFlags(argv);
 	assertKnownFlags(flags);
 	const dataset = requiredFlag(flags, "dataset");
 	if (dataset !== "agentrx" && dataset !== "codetracebench") throw new Error("--dataset must be 'agentrx' or 'codetracebench'");
 	const artifactDir = flags.get("artifact-dir")?.trim();
 	if (dataset === "codetracebench" && !artifactDir) throw new Error("--artifact-dir is required for CodeTraceBench");
-	const apiKeyEnv = requiredFlag(flags, "api-key-env");
-	if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(apiKeyEnv)) throw new Error("--api-key-env must be a valid environment variable name");
-	const apiKey = env[apiKeyEnv]?.trim();
-	if (!apiKey) throw new Error(`--api-key-env points to an empty or missing variable: ${apiKeyEnv}`);
 	const maxCostUsd = positiveFiniteFlag(flags, "max-cost-usd", 5);
 	const python = flags.get("python")?.trim();
 	const analyst = flags.get("analyst")?.trim() ?? "dspy-rlm";
@@ -5048,6 +5214,16 @@ function parseCommandConfig(argv, env) {
 	if (instructionsFile && analyst !== "dspy-rlm") throw new Error("--instructions-file requires --analyst dspy-rlm");
 	const instructionsOverride = instructionsFile ? readAnalystInstructionsOverride(instructionsFile) : void 0;
 	if (rlmSamples > 1 && dataset !== "codetracebench") throw new Error("--rlm-samples above 1 requires --dataset codetracebench; step-level consensus is defined on its block grammar");
+	const model = requiredFlag(flags, "model");
+	const modelOwnerModule = requiredFlag(flags, "model-owner-module");
+	const owner = await (dependencies.loadModelExecutionOwner ?? loadModelExecutionOwner)(modelOwnerModule, {
+		model,
+		environment: Object.freeze({ ...env })
+	});
+	assertModelExecutionOwner(owner);
+	const pricing = owner.pricing ?? benchmarkModelPricing(model);
+	const maxOutputTokens = positiveFlag(flags, "max-output-tokens", 16384);
+	const timeoutMs = positiveFlag(flags, "timeout-ms", 3e5);
 	return {
 		dataset,
 		analyst,
@@ -5058,15 +5234,36 @@ function parseCommandConfig(argv, env) {
 		revision: immutableRevision(requiredFlag(flags, "revision")),
 		split: requiredFlag(flags, "split"),
 		model: {
-			baseUrl: openAiCompatibleBaseUrl(requiredFlag(flags, "base-url")),
-			apiKey,
-			model: requiredFlag(flags, "model"),
-			maxOutputTokens: positiveFlag(flags, "max-output-tokens", 16384),
-			timeoutMs: positiveFlag(flags, "timeout-ms", 3e5),
+			call: owner.call,
+			callRef: owner.callRef,
+			recordExecution: owner.recordExecution,
+			model,
+			maxOutputTokens,
+			timeoutMs,
+			maxReasoningTokens: nonNegativeFlag(flags, "max-reasoning-tokens", maxOutputTokens * 4),
+			maxModelRequestBytes: positiveFlag(flags, "max-model-request-bytes", 16 * 1024 * 1024),
+			maxModelResponseBytes: positiveFlag(flags, "max-model-response-bytes", 4 * 1024 * 1024),
+			modelRequestTimeoutMs: positiveFlag(flags, "model-request-timeout-ms", timeoutMs),
+			pricing,
 			maxCostUsdPerAnalysis: maxCostUsd,
 			...instructionsOverride ? { instructionsOverride } : {},
 			dspyRlm: {
-				...python ? { runner: { command: python } } : {},
+				runner: {
+					...python ? { command: python } : {},
+					limits: {
+						maxInputBytes: positiveFlag(flags, "max-process-input-bytes", 64 * 1024 * 1024),
+						maxResultBytes: positiveFlag(flags, "max-process-result-bytes", 4 * 1024 * 1024),
+						maxOutputChars: positiveFlag(flags, "max-process-output-chars", 64e3)
+					}
+				},
+				maxIterations: positiveFlag(flags, "max-iterations", 14),
+				maxLlmCalls: positiveFlag(flags, "max-llm-calls", 8),
+				maxToolCalls: positiveFlag(flags, "max-tool-calls", 80),
+				maxOutputChars: positiveFlag(flags, "max-analysis-output-chars", 8e3),
+				...flags.has("max-model-requests") ? { maxModelRequests: positiveFlag(flags, "max-model-requests") } : {},
+				traceToolRequestBytes: positiveFlag(flags, "trace-tool-request-bytes", 1e6),
+				traceToolResponseBytes: positiveFlag(flags, "trace-tool-response-bytes", 4e6),
+				traceToolTimeoutMs: positiveFlag(flags, "trace-tool-timeout-ms", 6e4),
 				samples: rlmSamples
 			}
 		},
@@ -5077,7 +5274,7 @@ function parseCommandConfig(argv, env) {
 		rlmSamples,
 		maxCostUsd,
 		maxArtifactBytes: positiveFlag(flags, "max-artifact-bytes", DEFAULT_MAX_VERIFICATION_ARTIFACT_BYTES),
-		apiKeyEnv,
+		modelOwnerModule,
 		command: `agent-eval analyst-benchmark ${argv.filter((argument) => argument !== "--resume").map(shellQuote).join(" ")}`,
 		resume: flags.has("resume")
 	};
@@ -5113,8 +5310,7 @@ const KNOWN_FLAGS = /* @__PURE__ */ new Set([
 	"out",
 	"revision",
 	"split",
-	"base-url",
-	"api-key-env",
+	"model-owner-module",
 	"model",
 	"limit",
 	"seed",
@@ -5123,6 +5319,21 @@ const KNOWN_FLAGS = /* @__PURE__ */ new Set([
 	"rlm-samples",
 	"instructions-file",
 	"max-output-tokens",
+	"max-reasoning-tokens",
+	"max-model-requests",
+	"max-model-request-bytes",
+	"max-model-response-bytes",
+	"model-request-timeout-ms",
+	"max-iterations",
+	"max-llm-calls",
+	"max-tool-calls",
+	"max-analysis-output-chars",
+	"trace-tool-request-bytes",
+	"trace-tool-response-bytes",
+	"trace-tool-timeout-ms",
+	"max-process-input-bytes",
+	"max-process-result-bytes",
+	"max-process-output-chars",
 	"python",
 	"timeout-ms",
 	"max-cost-usd",
@@ -5152,6 +5363,13 @@ function integerFlag(flags, name, defaultValue) {
 	if (!Number.isSafeInteger(value)) throw new Error(`--${name} must be a safe integer`);
 	return value;
 }
+function nonNegativeFlag(flags, name, defaultValue) {
+	const raw = flags.get(name);
+	if (raw === void 0) return defaultValue;
+	const value = Number(raw);
+	if (!Number.isSafeInteger(value) || value < 0) throw new Error(`--${name} must be a non-negative safe integer`);
+	return value;
+}
 function positiveFiniteFlag(flags, name, defaultValue) {
 	const raw = flags.get(name);
 	if (raw === void 0) return defaultValue;
@@ -5159,20 +5377,24 @@ function positiveFiniteFlag(flags, name, defaultValue) {
 	if (!Number.isFinite(value) || value <= 0) throw new Error(`--${name} must be a positive finite number`);
 	return value;
 }
-function openAiCompatibleBaseUrl(value) {
-	let parsed;
-	try {
-		parsed = new URL(value);
-	} catch {
-		throw new Error("--base-url must be an absolute HTTP or HTTPS URL");
-	}
-	if (parsed.protocol !== "http:" && parsed.protocol !== "https:" || parsed.username || parsed.password || parsed.search || parsed.hash) throw new Error("--base-url must use HTTP or HTTPS without credentials, query, or fragment");
-	if (parsed.protocol === "http:" && !isLoopbackHost(parsed.hostname)) throw new Error("--base-url must use HTTPS unless the endpoint is on the local machine");
-	return value;
+async function loadModelExecutionOwner(moduleRef, context) {
+	const imported = await (moduleRef.startsWith(".") || moduleRef.startsWith("/") ? import(pathToFileURL(resolve(moduleRef)).href) : import(moduleRef));
+	if (typeof imported.createModelExecutionOwner !== "function") throw new Error(`${moduleRef} must export createModelExecutionOwner({ model, environment })`);
+	return imported.createModelExecutionOwner(context);
+}
+function assertModelExecutionOwner(value) {
+	if (!value || typeof value !== "object") throw new Error("createModelExecutionOwner must return an object");
+	if (typeof value.call !== "function") throw new Error("model execution owner call must be a function");
+	if (typeof value.callRef !== "string" || !value.callRef.trim() || value.callRef !== value.callRef.trim()) throw new Error("model execution owner callRef must be trimmed and non-empty");
+	if (typeof value.recordExecution !== "function") throw new Error("model execution owner recordExecution must be a function");
 }
-function isLoopbackHost(hostname) {
-	const normalized = hostname.toLowerCase();
-	return normalized === "localhost" || normalized === "[::1]" || normalized === "::1" || /^127(?:\.\d{1,3}){3}$/.test(normalized);
+function benchmarkModelPricing(model) {
+	const pricing = resolveModelPricing(model);
+	if (!pricing) throw new Error(`model execution owner must supply pricing for uncatalogued model '${model}'`);
+	return {
+		inputUsdPerMillion: pricing.input * 1e3,
+		outputUsdPerMillion: pricing.output * 1e3
+	};
 }
 function immutableRevision(value) {
 	if (!/^(?:[a-fA-F0-9]{40}|[a-fA-F0-9]{64})$/.test(value)) throw new Error("--revision must be a full 40- or 64-character hexadecimal digest");
@@ -5247,4 +5469,4 @@ function shellQuote(value) {
 //#endregion
 export { normalizeAgentRxCategory as $, parseVerificationOutcome as A, analystBenchmarkDependencyLockDigest as B, MAX_INCORRECT_BLOCK_STEPS as C, DEFAULT_MAX_VERIFICATION_ARTIFACT_BYTES as D, publicBenchmarkSystemPrompt as E, ANALYST_BENCHMARK_EVIDENCE_IMPLEMENTATION_SHA256 as F, ANALYST_BENCHMARK_OBSERVATIONS_FILE as G, ANALYST_BENCHMARK_COST_LEDGER_FILE as H, ANALYST_BENCHMARK_EVIDENCE_PACKAGE_VERSION as I, summarizeAgentRxCalibration as J, AGENT_RX_UPSTREAM_REVISION as K, ANALYST_BENCHMARK_IMPLEMENTATION_DIGEST_ALGORITHM as L, ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES as M, ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 as N, appendVerificationArtifactsToOtlp as O, ANALYST_BENCHMARK_EVIDENCE_DEPENDENCY_LOCK_SHA256 as P, agentRxPredictionsToFindings as Q, ANALYST_BENCHMARK_IMPLEMENTATION_FILES as R, MAX_INCORRECT_BLOCKS as S, publicBenchmarkRlmInstructions as T, ANALYST_BENCHMARK_LOCAL_RECEIPT_FILE as U, analystBenchmarkImplementationDigest as V, ANALYST_BENCHMARK_MANIFEST_FILE as W, codeTracerPredictionsToFindings as X, codeTraceBenchCase as Y, agentRxBenchmarkCase as Z, compareAnalystRunners as _, preparePublicAnalystBenchmark as a, readAnalystInstructionsOverride as b, selectPublicBenchmarkRows as c, adaptPublicBenchmarkFindings as d, roundAgentRxStep as et, emptyPublicBenchmarkRunner as f, summarizeCodeTraceCalibration as g, renderCodeTraceCalibrationMarkdown as h, loadPublicBenchmarkRows as i, ANALYST_BENCHMARK_DEPENDENCY_LOCK_DIGEST_ALGORITHM as j, loadCodeTraceVerificationArtifacts as k, createPublicBenchmarkRlmRunner as l, readAnalystBenchmarkArtifact as m, runAnalystBenchmarkCommand as n, publicBenchmarkDistributions as o, expandCodeTraceFailureBlocks as p, renderAgentRxCalibrationMarkdown as q, renderAnalystBenchmarkMarkdown as r, publicBenchmarkSelectionReport as s, ANALYST_BENCHMARK_HELP as t, normalizeBenchmarkLabel as tt, createPublicBenchmarkDirectRunner as u, analystInstructionsOverrideFromText as v, publicBenchmarkProtocolSha256 as w, CODE_TRACE_BENCH_ANALYST_PROMPT as x, effectiveAnalystProtocolSha256 as y, ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 as z };
 
-//# sourceMappingURL=benchmark-command-CSddvlq7.js.map
+//# sourceMappingURL=benchmark-command-95Zqd1e8.js.map