@tangle-network/agent-eval 0.163.2 → 0.171.0

package/dist/{benchmark-command-CF-4GEWZ.js → benchmark-command-D8k3Gf0J.js}

@@ -1,17 +1,17 @@
 import { s as ValidationError, t as AgentEvalError } from "./errors-Dngq5h35.js";
-import { a as jsonDocument, i as hashCanonical, r as canonicalString } from "./canonical-IL-Bu-14.js";
+import { a as hashCanonical, i as compareCodeUnits, o as jsonDocument, r as canonicalString } from "./canonical-DPyQ_rpt.js";
 import { r as pairedBootstrap } from "./paired-tests-C8iCsioC.js";
 import { a as resolveModelPricing } from "./metrics-Qv-cpptD.js";
 import { a as CostLedgerPersistenceError, i as CostLedger, n as CostCallConflictError, o as CostReceiptCaptureError, r as CostCeilingReachedError, s as CostReservationExceededError, t as CostAccountingIncompleteError } from "./cost-ledger-B1qx30B4.js";
-import { c as writeLedgerFileAtomically, s as withLedgerFileLock } from "./ledger-core-BOzlRygb.js";
-import { b as createRunCostLedger, i as startExternalOptimizerModelProxy, r as runWithCleanup, v as resolveExternalOptimizerProcessLimits, x as fsCampaignStorage } from "./external-optimizer-subprocess-CqLMW3nh.js";
-import { n as makeFinding, o as usageReceiptFromCostLedger } from "./types-BI4fT3HN.js";
-import { C as RAW_FINDING_SCHEMA_PROMPT, T as evidenceRefsFromRawFinding, m as TRACE_ANALYSIS_LIMITS, r as runTraceAnalyst, w as RawAnalystFindingSchema } from "./kind-factory-DY8FdoXf.js";
+import { c as writeLedgerFileAtomically, s as withLedgerFileLock } from "./ledger-core-PIfjCbKn.js";
+import { S as fsCampaignStorage, i as startExternalOptimizerModelProxy, r as runWithCleanup, x as createRunCostLedger, y as resolveExternalOptimizerProcessLimits } from "./external-optimizer-subprocess-Cex8Da2i.js";
+import { r as makeFinding, s as usageReceiptFromCostLedger } from "./types-CiWITkGo.js";
+import { C as RAW_FINDING_SCHEMA_PROMPT, T as evidenceRefsFromRawFinding, m as TRACE_ANALYSIS_LIMITS, r as runTraceAnalyst, w as RawAnalystFindingSchema } from "./kind-factory-DMeEoMQZ.js";
 import { a as callLlmJson, r as LlmResponseError, t as LlmCallError } from "./llm-client-BFMRpmqb.js";
-import { t as createDspyRlmTraceEngine } from "./dspy-rlm-engine-DhA9qKIm.js";
-import { i as otlpTextToTraceAnalysisStore, r as createOtlpBufferTraceStore, t as DEFAULT_MAX_TRACE_FILE_BYTES } from "./store-otlp-C_Rq5I4D.js";
-import { i as summarizeAnalystBenchmarkRunner, n as runAnalystBenchmark, r as traceStoreEvidenceResolver } from "./benchmark-BhT16ep9.js";
-import { n as acquireSingleRunLock } from "./external-optimizer-process-BFmh36vW.js";
+import { t as createDspyRlmTraceEngine } from "./dspy-rlm-engine-CS3qcCEk.js";
+import { i as otlpTextToTraceAnalysisStore, r as createOtlpBufferTraceStore, t as DEFAULT_MAX_TRACE_FILE_BYTES } from "./store-otlp-CHjBvWQY.js";
+import { i as summarizeAnalystBenchmarkRunner, n as runAnalystBenchmark, r as traceStoreEvidenceResolver } from "./benchmark-C4wk_Sjr.js";
+import { n as acquireSingleRunLock } from "./external-optimizer-process-CQxylYeG.js";
 import { c as primeProtocolSha256, d as runPrimeExchange, f as decodeReplyRows, n as buildPrimePrompt, p as assertEqualDeclarativeTerms, t as analystUsageReceiptFromPrimeUsage, u as projectPrimeTrajectory } from "./prime-protocol-6tZTVsWm.js";
 import { createHash, randomUUID } from "node:crypto";
 import { constants, existsSync, lstatSync, readFileSync } from "node:fs";
@@ -625,6 +625,226 @@ function escapeCell$2(value) {
 	return value.replaceAll("|", "\\|").replaceAll("\n", " ");
 }
 //#endregion
+//#region src/analyst/benchmark-comparison.ts
+/**
+* Every metric a benchmark comparison reports, mapped to the direction that
+* is an improvement. This table is the only declaration of the vocabulary:
+* the type, the reporting order, the artifact schema's accepted values, and
+* each metric's direction all derive from it, so a metric cannot exist in one
+* of those four places and be missing from another.
+*/
+const ANALYST_COMPARISON_METRIC_DIRECTION = {
+	completion: "higher",
+	issueRecall: "higher",
+	findingPrecision: "higher",
+	f1: "higher",
+	criticalStepAccuracy: "higher",
+	citationCoverage: "higher",
+	citationExcerptCoverage: "higher",
+	citationLabelAgreement: "higher",
+	citationResolution: "higher",
+	trustedNegativeAccuracy: "higher",
+	latencyMs: "lower",
+	calls: "lower",
+	inputTokens: "lower",
+	outputTokens: "lower",
+	reasoningTokens: "lower",
+	cachedTokens: "lower",
+	cacheWriteTokens: "lower",
+	costUsd: "lower"
+};
+/** The vocabulary as a non-empty tuple, which is what `z.enum` accepts. Key
+*  order is the declaration order above, and it is the reporting order. */
+const ANALYST_COMPARISON_METRICS = Object.keys(ANALYST_COMPARISON_METRIC_DIRECTION);
+/** `'lower'` when a smaller value is the improvement. */
+function analystComparisonMetricDirection(metric) {
+	return ANALYST_COMPARISON_METRIC_DIRECTION[metric];
+}
+function compareAnalystRunners(result, options) {
+	const confidence = options.confidence ?? .95;
+	const resamples = options.resamples ?? 2e3;
+	assertComparisonControls(confidence, resamples);
+	const runnerIds = new Set(result.summaries.map((summary) => summary.runnerId));
+	if (!runnerIds.has(options.baselineRunnerId)) throw new TypeError(`unknown baseline analyst runner '${options.baselineRunnerId}'`);
+	if (!runnerIds.has(options.candidateRunnerId)) throw new TypeError(`unknown candidate analyst runner '${options.candidateRunnerId}'`);
+	if (options.baselineRunnerId === options.candidateRunnerId) throw new TypeError("baseline and candidate analyst runners must be different");
+	const baseline = observationsByCase(result.observations, options.baselineRunnerId);
+	const candidate = observationsByCase(result.observations, options.candidateRunnerId);
+	const populationRepresentativenessProven = result.provenance.metadata?.populationRepresentativenessProven === true;
+	const metrics = ANALYST_COMPARISON_METRICS.map((metric) => compareMetric({
+		metric,
+		baseline,
+		candidate,
+		confidence,
+		resamples,
+		seed: options.seed,
+		populationRepresentativenessProven
+	}));
+	return {
+		baselineRunnerId: options.baselineRunnerId,
+		candidateRunnerId: options.candidateRunnerId,
+		metrics
+	};
+}
+function compareMetric(options) {
+	const pairedCases = [];
+	let eligibleObservations = 0;
+	let pairedObservations = 0;
+	let baselineMissingObservations = 0;
+	let candidateMissingObservations = 0;
+	let asymmetricMissingObservations = 0;
+	const caseIds = /* @__PURE__ */ new Set([...options.baseline.keys(), ...options.candidate.keys()]);
+	for (const caseId of caseIds) {
+		const baselineByRepetition = new Map((options.baseline.get(caseId) ?? []).map((observation) => [observation.repetition, observation]));
+		const candidateByRepetition = new Map((options.candidate.get(caseId) ?? []).map((observation) => [observation.repetition, observation]));
+		const caseBefore = [];
+		const caseAfter = [];
+		let clusterId;
+		const repetitions = /* @__PURE__ */ new Set([...baselineByRepetition.keys(), ...candidateByRepetition.keys()]);
+		for (const repetition of repetitions) {
+			const baselineObservation = baselineByRepetition.get(repetition);
+			const candidateObservation = candidateByRepetition.get(repetition);
+			const identity = baselineObservation ?? candidateObservation;
+			if (!identity || !metricApplies(identity, options.metric)) continue;
+			if (baselineObservation && candidateObservation) assertSameCaseIdentity(baselineObservation, candidateObservation);
+			eligibleObservations += 1;
+			clusterId = identity.clusterId;
+			const baselineValue = baselineObservation ? metricValue(baselineObservation, options.metric) : null;
+			const candidateValue = candidateObservation ? metricValue(candidateObservation, options.metric) : null;
+			const baselineMissing = baselineValue === null;
+			const candidateMissing = candidateValue === null;
+			if (baselineMissing) baselineMissingObservations += 1;
+			if (candidateMissing) candidateMissingObservations += 1;
+			if (baselineMissing !== candidateMissing) asymmetricMissingObservations += 1;
+			if (baselineMissing || candidateMissing) continue;
+			caseBefore.push(baselineValue);
+			caseAfter.push(candidateValue);
+			pairedObservations += 1;
+		}
+		if (caseBefore.length === 0 || !clusterId) continue;
+		pairedCases.push({
+			clusterId,
+			baseline: mean$1(caseBefore),
+			candidate: mean$1(caseAfter)
+		});
+	}
+	const byCluster = /* @__PURE__ */ new Map();
+	for (const pairedCase of pairedCases) {
+		const rows = byCluster.get(pairedCase.clusterId) ?? [];
+		rows.push(pairedCase);
+		byCluster.set(pairedCase.clusterId, rows);
+	}
+	const before = [...byCluster.values()].map((rows) => mean$1(rows.map((row) => row.baseline)));
+	const after = [...byCluster.values()].map((rows) => mean$1(rows.map((row) => row.candidate)));
+	const interval = before.length === 0 ? null : pairedBootstrap(before, after, {
+		confidence: options.confidence,
+		resamples: options.resamples,
+		statistic: "mean",
+		seed: options.seed
+	});
+	const survivorOnly = pairedObservations < eligibleObservations;
+	const limitations = [];
+	if (!interval?.gateEligible) limitations.push("fewer-than-20-independent-clusters");
+	if (!options.populationRepresentativenessProven) limitations.push("population-representativeness-not-proven");
+	if (survivorOnly) limitations.push("missing-observations");
+	const comparison = {
+		metric: options.metric,
+		direction: analystComparisonMetricDirection(options.metric),
+		pairedCases: pairedCases.length,
+		pairedClusters: before.length,
+		eligibleObservations,
+		pairedObservations,
+		baselineMissingObservations,
+		candidateMissingObservations,
+		asymmetricMissingObservations,
+		survivorOnly,
+		baselineMean: before.length === 0 ? null : mean$1(before),
+		candidateMean: after.length === 0 ? null : mean$1(after),
+		meanDelta: interval?.mean ?? null,
+		intervalLow: interval?.low ?? null,
+		intervalHigh: interval?.high ?? null,
+		confidence: options.confidence,
+		resamples: options.resamples,
+		minimumSampleMet: interval?.gateEligible ?? false,
+		populationInferenceEligible: limitations.length === 0,
+		inferenceLimitations: limitations
+	};
+	assertValidComparison(comparison);
+	return comparison;
+}
+function observationsByCase(observations, runnerId) {
+	const byCase = /* @__PURE__ */ new Map();
+	for (const observation of observations) {
+		if (observation.runnerId !== runnerId) continue;
+		const rows = byCase.get(observation.caseId) ?? [];
+		rows.push(observation);
+		byCase.set(observation.caseId, rows);
+	}
+	return byCase;
+}
+function assertSameCaseIdentity(baseline, candidate) {
+	if (baseline.clusterId !== candidate.clusterId || baseline.labelState !== candidate.labelState) throw new Error(`analyst comparison case identity differs for '${baseline.caseId}' repetition ${baseline.repetition}`);
+}
+function metricApplies(observation, metric) {
+	if (metric === "trustedNegativeAccuracy") return observation.labelState === "trusted-negative";
+	if (metric === "issueRecall" || metric === "findingPrecision" || metric === "f1") return observation.labelState === "positive";
+	if (metric === "criticalStepAccuracy") return observation.labelState === "positive" && observation.score.criticalStepAccuracy !== null;
+	return true;
+}
+function metricValue(observation, metric) {
+	if (metric === "completion") return observation.error ? 0 : 1;
+	if (metric === "latencyMs") return observation.latencyMs;
+	if (metric === "trustedNegativeAccuracy") {
+		if (observation.error) return 0;
+		return observation.score.predictionOnLabelEmptyCase ? 0 : 1;
+	}
+	if (observation.error && (metric === "issueRecall" || metric === "findingPrecision" || metric === "f1" || metric === "criticalStepAccuracy")) return 0;
+	if (observation.error && (metric === "citationCoverage" || metric === "citationExcerptCoverage" || metric === "citationLabelAgreement" || metric === "citationResolution")) return null;
+	if (metric === "issueRecall") return observation.score.issueRecall;
+	if (metric === "findingPrecision") return observation.score.findingPrecision;
+	if (metric === "f1") return observation.score.f1;
+	if (metric === "criticalStepAccuracy") return observation.score.criticalStepAccuracy;
+	if (metric === "citationCoverage") return observation.score.citationCoverage;
+	if (metric === "citationExcerptCoverage") return observation.score.citationExcerptCoverage;
+	if (metric === "citationLabelAgreement") return observation.score.citationLabelAgreement;
+	if (metric === "citationResolution") return observation.evidenceResolution?.validity ?? null;
+	if (metric === "calls") return observation.usage?.calls ?? null;
+	if (metric === "inputTokens") return observation.usage?.tokens?.input ?? null;
+	if (metric === "outputTokens") return observation.usage?.tokens?.output ?? null;
+	if (metric === "reasoningTokens") return observation.usage?.tokens?.reasoning ?? null;
+	if (metric === "cachedTokens") return observation.usage?.tokens?.cached ?? null;
+	if (metric === "cacheWriteTokens") return observation.usage?.tokens?.cacheWrite ?? null;
+	if (observation.usage?.cost.kind === "uncaptured") return null;
+	return observation.usage?.cost.usd ?? null;
+}
+function mean$1(values) {
+	return values.reduce((sum, value) => sum + value, 0) / values.length;
+}
+function assertComparisonControls(confidence, resamples) {
+	if (!Number.isSafeInteger(resamples) || resamples <= 0 || resamples > 1e6) throw new Error(`compareAnalystRunners: resamples must be a positive safe integer no greater than 1000000, got ${String(resamples)}`);
+	if (!Number.isFinite(confidence) || confidence <= 0 || confidence >= 1) throw new Error(`compareAnalystRunners: confidence must be a finite number in (0,1), got ${String(confidence)}`);
+}
+function assertValidComparison(comparison) {
+	if ([
+		"pairedCases",
+		"pairedClusters",
+		"eligibleObservations",
+		"pairedObservations",
+		"baselineMissingObservations",
+		"candidateMissingObservations",
+		"asymmetricMissingObservations",
+		"confidence",
+		"resamples"
+	].some((field) => !Number.isFinite(comparison[field])) || [
+		"baselineMean",
+		"candidateMean",
+		"meanDelta",
+		"intervalLow",
+		"intervalHigh"
+	].some((field) => comparison[field] !== null && !Number.isFinite(comparison[field]))) throw new Error(`compareAnalystRunners: ${comparison.metric} produced non-finite comparison output`);
+	if (comparison.intervalLow !== null && comparison.intervalHigh !== null && comparison.intervalLow > comparison.intervalHigh) throw new Error(`compareAnalystRunners: ${comparison.metric} produced an invalid confidence interval`);
+}
+//#endregion
 //#region src/analyst/benchmark-command-validation.ts
 const nonEmptyString = z.string().refine((value) => value.trim().length > 0, { message: "must be a non-empty string" });
 const safeInteger$1 = z.number().refine(Number.isSafeInteger, { message: "must be a safe integer" });
@@ -863,26 +1083,7 @@ const resultSchema = z.strictObject({
 	summaries: z.array(summarySchema)
 });
 const comparisonMetricSchema = z.strictObject({
-	metric: z.enum([
-		"completion",
-		"issueRecall",
-		"findingPrecision",
-		"f1",
-		"criticalStepAccuracy",
-		"citationCoverage",
-		"citationExcerptCoverage",
-		"citationLabelAgreement",
-		"citationResolution",
-		"trustedNegativeAccuracy",
-		"latencyMs",
-		"calls",
-		"inputTokens",
-		"outputTokens",
-		"reasoningTokens",
-		"cachedTokens",
-		"cacheWriteTokens",
-		"costUsd"
-	]),
+	metric: z.enum(ANALYST_COMPARISON_METRICS),
 	direction: z.enum(["higher", "lower"]),
 	pairedCases: nonNegativeInteger,
 	pairedClusters: nonNegativeInteger,
@@ -1211,7 +1412,7 @@ const ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES = Object.freeze([
 	"package.json",
 	"pnpm-lock.yaml"
 ]);
-const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = "a9eb17beac6195c9b33de3c85ba4315d9e20a84082dccc363156c8f1a7ef623d";
+const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = "43f2e94849ab4d777ac8c0c358c6c4f0687129c322c0565fb5cc8583922a433d";
 const ANALYST_BENCHMARK_EVIDENCE_DEPENDENCY_LOCK_SHA256 = "1e03f2daed356d60316aabefb407ec1e437ac94d408d61eea4ae096e9c6fbb5b";
 const ANALYST_BENCHMARK_EVIDENCE_IMPLEMENTATION_SHA256 = "4dba263b6256a30d56c7fdb2d992d3a953c0035d731f359b704db806f68f75ac";
 const ANALYST_BENCHMARK_IMPLEMENTATION_FILES = Object.freeze([
@@ -1324,7 +1525,7 @@ const ANALYST_BENCHMARK_IMPLEMENTATION_FILES = Object.freeze([
 	"src/trace/raw-provider-sink.ts",
 	"src/verdict-cache.ts"
 ]);
-const ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 = "f2ba2bf8477be995b305d945a7a5b443b2980f1b9f6756d5cf7759d6fcf708ff";
+const ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 = "917ad7bb523495505a8da1e48f9acefe25970c946f401fe4d38dbac4e1c43aad";
 function analystBenchmarkImplementationDigest() {
 	return ANALYST_BENCHMARK_IMPLEMENTATION_SHA256;
 }
@@ -2019,7 +2220,7 @@ async function artifactCandidates(caseDirectory) {
 		if (seen.has(entry.path)) return false;
 		seen.add(entry.path);
 		return true;
-	}).sort((left, right) => artifactRoleOrder(left.role) - artifactRoleOrder(right.role) || left.relativePath.localeCompare(right.relativePath));
+	}).sort((left, right) => artifactRoleOrder(left.role) - artifactRoleOrder(right.role) || compareCodeUnits(left.relativePath, right.relativePath));
 }
 async function firstExisting(caseDirectory, candidates) {
 	for (const relativePath of candidates) {
@@ -2731,222 +2932,6 @@ function isNodeError(error, code) {
 	return error instanceof Error && "code" in error && error.code === code;
 }
 //#endregion
-//#region src/analyst/benchmark-comparison.ts
-function compareAnalystRunners(result, options) {
-	const confidence = options.confidence ?? .95;
-	const resamples = options.resamples ?? 2e3;
-	assertComparisonControls(confidence, resamples);
-	const runnerIds = new Set(result.summaries.map((summary) => summary.runnerId));
-	if (!runnerIds.has(options.baselineRunnerId)) throw new TypeError(`unknown baseline analyst runner '${options.baselineRunnerId}'`);
-	if (!runnerIds.has(options.candidateRunnerId)) throw new TypeError(`unknown candidate analyst runner '${options.candidateRunnerId}'`);
-	if (options.baselineRunnerId === options.candidateRunnerId) throw new TypeError("baseline and candidate analyst runners must be different");
-	const baseline = observationsByCase(result.observations, options.baselineRunnerId);
-	const candidate = observationsByCase(result.observations, options.candidateRunnerId);
-	const populationRepresentativenessProven = result.provenance.metadata?.populationRepresentativenessProven === true;
-	const metrics = METRICS.map((metric) => compareMetric({
-		metric,
-		baseline,
-		candidate,
-		confidence,
-		resamples,
-		seed: options.seed,
-		populationRepresentativenessProven
-	}));
-	return {
-		baselineRunnerId: options.baselineRunnerId,
-		candidateRunnerId: options.candidateRunnerId,
-		metrics
-	};
-}
-function compareMetric(options) {
-	const pairedCases = [];
-	let eligibleObservations = 0;
-	let pairedObservations = 0;
-	let baselineMissingObservations = 0;
-	let candidateMissingObservations = 0;
-	let asymmetricMissingObservations = 0;
-	const caseIds = /* @__PURE__ */ new Set([...options.baseline.keys(), ...options.candidate.keys()]);
-	for (const caseId of caseIds) {
-		const baselineByRepetition = new Map((options.baseline.get(caseId) ?? []).map((observation) => [observation.repetition, observation]));
-		const candidateByRepetition = new Map((options.candidate.get(caseId) ?? []).map((observation) => [observation.repetition, observation]));
-		const caseBefore = [];
-		const caseAfter = [];
-		let clusterId;
-		const repetitions = /* @__PURE__ */ new Set([...baselineByRepetition.keys(), ...candidateByRepetition.keys()]);
-		for (const repetition of repetitions) {
-			const baselineObservation = baselineByRepetition.get(repetition);
-			const candidateObservation = candidateByRepetition.get(repetition);
-			const identity = baselineObservation ?? candidateObservation;
-			if (!identity || !metricApplies(identity, options.metric)) continue;
-			if (baselineObservation && candidateObservation) assertSameCaseIdentity(baselineObservation, candidateObservation);
-			eligibleObservations += 1;
-			clusterId = identity.clusterId;
-			const baselineValue = baselineObservation ? metricValue(baselineObservation, options.metric) : null;
-			const candidateValue = candidateObservation ? metricValue(candidateObservation, options.metric) : null;
-			const baselineMissing = baselineValue === null;
-			const candidateMissing = candidateValue === null;
-			if (baselineMissing) baselineMissingObservations += 1;
-			if (candidateMissing) candidateMissingObservations += 1;
-			if (baselineMissing !== candidateMissing) asymmetricMissingObservations += 1;
-			if (baselineMissing || candidateMissing) continue;
-			caseBefore.push(baselineValue);
-			caseAfter.push(candidateValue);
-			pairedObservations += 1;
-		}
-		if (caseBefore.length === 0 || !clusterId) continue;
-		pairedCases.push({
-			clusterId,
-			baseline: mean$1(caseBefore),
-			candidate: mean$1(caseAfter)
-		});
-	}
-	const byCluster = /* @__PURE__ */ new Map();
-	for (const pairedCase of pairedCases) {
-		const rows = byCluster.get(pairedCase.clusterId) ?? [];
-		rows.push(pairedCase);
-		byCluster.set(pairedCase.clusterId, rows);
-	}
-	const before = [...byCluster.values()].map((rows) => mean$1(rows.map((row) => row.baseline)));
-	const after = [...byCluster.values()].map((rows) => mean$1(rows.map((row) => row.candidate)));
-	const interval = before.length === 0 ? null : pairedBootstrap(before, after, {
-		confidence: options.confidence,
-		resamples: options.resamples,
-		statistic: "mean",
-		seed: options.seed
-	});
-	const survivorOnly = pairedObservations < eligibleObservations;
-	const limitations = [];
-	if (!interval?.gateEligible) limitations.push("fewer-than-20-independent-clusters");
-	if (!options.populationRepresentativenessProven) limitations.push("population-representativeness-not-proven");
-	if (survivorOnly) limitations.push("missing-observations");
-	const comparison = {
-		metric: options.metric,
-		direction: LOWER_IS_BETTER.has(options.metric) ? "lower" : "higher",
-		pairedCases: pairedCases.length,
-		pairedClusters: before.length,
-		eligibleObservations,
-		pairedObservations,
-		baselineMissingObservations,
-		candidateMissingObservations,
-		asymmetricMissingObservations,
-		survivorOnly,
-		baselineMean: before.length === 0 ? null : mean$1(before),
-		candidateMean: after.length === 0 ? null : mean$1(after),
-		meanDelta: interval?.mean ?? null,
-		intervalLow: interval?.low ?? null,
-		intervalHigh: interval?.high ?? null,
-		confidence: options.confidence,
-		resamples: options.resamples,
-		minimumSampleMet: interval?.gateEligible ?? false,
-		populationInferenceEligible: limitations.length === 0,
-		inferenceLimitations: limitations
-	};
-	assertValidComparison(comparison);
-	return comparison;
-}
-const METRICS = [
-	"completion",
-	"issueRecall",
-	"findingPrecision",
-	"f1",
-	"criticalStepAccuracy",
-	"citationCoverage",
-	"citationExcerptCoverage",
-	"citationLabelAgreement",
-	"citationResolution",
-	"trustedNegativeAccuracy",
-	"latencyMs",
-	"calls",
-	"inputTokens",
-	"outputTokens",
-	"reasoningTokens",
-	"cachedTokens",
-	"cacheWriteTokens",
-	"costUsd"
-];
-const LOWER_IS_BETTER = /* @__PURE__ */ new Set([
-	"latencyMs",
-	"calls",
-	"inputTokens",
-	"outputTokens",
-	"reasoningTokens",
-	"cachedTokens",
-	"cacheWriteTokens",
-	"costUsd"
-]);
-function observationsByCase(observations, runnerId) {
-	const byCase = /* @__PURE__ */ new Map();
-	for (const observation of observations) {
-		if (observation.runnerId !== runnerId) continue;
-		const rows = byCase.get(observation.caseId) ?? [];
-		rows.push(observation);
-		byCase.set(observation.caseId, rows);
-	}
-	return byCase;
-}
-function assertSameCaseIdentity(baseline, candidate) {
-	if (baseline.clusterId !== candidate.clusterId || baseline.labelState !== candidate.labelState) throw new Error(`analyst comparison case identity differs for '${baseline.caseId}' repetition ${baseline.repetition}`);
-}
-function metricApplies(observation, metric) {
-	if (metric === "trustedNegativeAccuracy") return observation.labelState === "trusted-negative";
-	if (metric === "issueRecall" || metric === "findingPrecision" || metric === "f1") return observation.labelState === "positive";
-	if (metric === "criticalStepAccuracy") return observation.labelState === "positive" && observation.score.criticalStepAccuracy !== null;
-	return true;
-}
-function metricValue(observation, metric) {
-	if (metric === "completion") return observation.error ? 0 : 1;
-	if (metric === "latencyMs") return observation.latencyMs;
-	if (metric === "trustedNegativeAccuracy") {
-		if (observation.error) return 0;
-		return observation.score.predictionOnLabelEmptyCase ? 0 : 1;
-	}
-	if (observation.error && (metric === "issueRecall" || metric === "findingPrecision" || metric === "f1" || metric === "criticalStepAccuracy")) return 0;
-	if (observation.error && (metric === "citationCoverage" || metric === "citationExcerptCoverage" || metric === "citationLabelAgreement" || metric === "citationResolution")) return null;
-	if (metric === "issueRecall") return observation.score.issueRecall;
-	if (metric === "findingPrecision") return observation.score.findingPrecision;
-	if (metric === "f1") return observation.score.f1;
-	if (metric === "criticalStepAccuracy") return observation.score.criticalStepAccuracy;
-	if (metric === "citationCoverage") return observation.score.citationCoverage;
-	if (metric === "citationExcerptCoverage") return observation.score.citationExcerptCoverage;
-	if (metric === "citationLabelAgreement") return observation.score.citationLabelAgreement;
-	if (metric === "citationResolution") return observation.evidenceResolution?.validity ?? null;
-	if (metric === "calls") return observation.usage?.calls ?? null;
-	if (metric === "inputTokens") return observation.usage?.tokens?.input ?? null;
-	if (metric === "outputTokens") return observation.usage?.tokens?.output ?? null;
-	if (metric === "reasoningTokens") return observation.usage?.tokens?.reasoning ?? null;
-	if (metric === "cachedTokens") return observation.usage?.tokens?.cached ?? null;
-	if (metric === "cacheWriteTokens") return observation.usage?.tokens?.cacheWrite ?? null;
-	if (observation.usage?.cost.kind === "uncaptured") return null;
-	return observation.usage?.cost.usd ?? null;
-}
-function mean$1(values) {
-	return values.reduce((sum, value) => sum + value, 0) / values.length;
-}
-function assertComparisonControls(confidence, resamples) {
-	if (!Number.isSafeInteger(resamples) || resamples <= 0 || resamples > 1e6) throw new Error(`compareAnalystRunners: resamples must be a positive safe integer no greater than 1000000, got ${String(resamples)}`);
-	if (!Number.isFinite(confidence) || confidence <= 0 || confidence >= 1) throw new Error(`compareAnalystRunners: confidence must be a finite number in (0,1), got ${String(confidence)}`);
-}
-function assertValidComparison(comparison) {
-	if ([
-		"pairedCases",
-		"pairedClusters",
-		"eligibleObservations",
-		"pairedObservations",
-		"baselineMissingObservations",
-		"candidateMissingObservations",
-		"asymmetricMissingObservations",
-		"confidence",
-		"resamples"
-	].some((field) => !Number.isFinite(comparison[field])) || [
-		"baselineMean",
-		"candidateMean",
-		"meanDelta",
-		"intervalLow",
-		"intervalHigh"
-	].some((field) => comparison[field] !== null && !Number.isFinite(comparison[field]))) throw new Error(`compareAnalystRunners: ${comparison.metric} produced non-finite comparison output`);
-	if (comparison.intervalLow !== null && comparison.intervalHigh !== null && comparison.intervalLow > comparison.intervalHigh) throw new Error(`compareAnalystRunners: ${comparison.metric} produced an invalid confidence interval`);
-}
-//#endregion
 //#region src/analyst/benchmark-public-calibration.ts
 function summarizeCodeTraceCalibration(result) {
 	return {
@@ -4940,7 +4925,7 @@ function selectPublicBenchmarkRows(dataset, rows, options) {
 		if (byId.has(id)) throw new Error(`public analyst benchmark dataset repeats trajectory id '${id}'`);
 		byId.set(id, row);
 	}
-	return [...byId].sort(([left], [right]) => selectionKey(options.seed, left).localeCompare(selectionKey(options.seed, right)) || left.localeCompare(right)).slice(0, Math.min(options.limit, byId.size)).map(([, row]) => row);
+	return [...byId].sort(([left], [right]) => compareCodeUnits(selectionKey(options.seed, left), selectionKey(options.seed, right)) || compareCodeUnits(left, right)).slice(0, Math.min(options.limit, byId.size)).map(([, row]) => row);
 }
 function publicBenchmarkDistributions(dataset, rows) {
 	const values = {
@@ -6364,6 +6349,6 @@ function shellQuote(value) {
 	return /^[a-zA-Z0-9_./:@%+=,-]+$/.test(value) ? value : `'${value.replaceAll("'", "'\\''")}'`;
 }
 //#endregion
-export { ANALYST_BENCHMARK_COST_LEDGER_FILE as $, analystInstructionsOverrideFromText as A, appendVerificationArtifactsToOtlp as B, adaptPublicBenchmarkFindings as C, renderCodeTraceCalibrationMarkdown as D, readAnalystBenchmarkArtifact as E, MAX_INCORRECT_BLOCK_STEPS as F, ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 as G, parseVerificationOutcome as H, publicBenchmarkProtocolSha256 as I, ANALYST_BENCHMARK_IMPLEMENTATION_DIGEST_ALGORITHM as J, ANALYST_BENCHMARK_EVIDENCE_DEPENDENCY_LOCK_SHA256 as K, publicBenchmarkRlmInstructions as L, readAnalystInstructionsOverride as M, CODE_TRACE_BENCH_ANALYST_PROMPT as N, summarizeCodeTraceCalibration as O, MAX_INCORRECT_BLOCKS as P, analystBenchmarkImplementationDigest as Q, publicBenchmarkSystemPrompt as R, analystDefinitionProtocolSha256 as S, expandCodeTraceFailureBlocks as T, ANALYST_BENCHMARK_DEPENDENCY_LOCK_DIGEST_ALGORITHM as U, loadCodeTraceVerificationArtifacts as V, ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES as W, ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 as X, ANALYST_BENCHMARK_IMPLEMENTATION_FILES as Y, analystBenchmarkDependencyLockDigest as Z, createPublicBenchmarkDirectRunner as _, primeCodeTraceAnalystDefinition as a, summarizeAgentRxCalibration as at, AnalystExpressivenessError as b, loadPublicBenchmarkRows as c, agentRxBenchmarkCase as ct, publicBenchmarkSelectionReport as d, roundAgentRxStep as dt, ANALYST_BENCHMARK_LOCAL_RECEIPT_FILE as et, selectPublicBenchmarkRows as f, normalizeBenchmarkLabel as ft, runReplVariableAnalystDefinition as g, rlmEngineLimits as h, primeAnalystProtocolSha256 as i, renderAgentRxCalibrationMarkdown as it, effectiveAnalystProtocolSha256 as j, compareAnalystRunners as k, preparePublicAnalystBenchmark as l, agentRxPredictionsToFindings as lt, publicRlmAnalystDefinition as m, renderAnalystBenchmarkMarkdown as n, ANALYST_BENCHMARK_OBSERVATIONS_FILE as nt, runInlineAnalystDefinition as o, codeTraceBenchCase as ot, createPublicBenchmarkRlmRunner as p, ANALYST_BENCHMARK_EVIDENCE_IMPLEMENTATION_SHA256 as q, createPrimeBenchmarkRunner as r, AGENT_RX_UPSTREAM_REVISION as rt, nodeHttpPrimeBridgeTransport as s, codeTracerPredictionsToFindings as st, runAnalystBenchmarkCommand as t, ANALYST_BENCHMARK_MANIFEST_FILE as tt, publicBenchmarkDistributions as u, normalizeAgentRxCategory as ut, publicDirectAnalystDefinition as v, emptyPublicBenchmarkRunner as w, analystDefinitionAsymmetries as x, runChunkedAnalystDefinition as y, DEFAULT_MAX_VERIFICATION_ARTIFACT_BYTES as z };
+export { ANALYST_BENCHMARK_LOCAL_RECEIPT_FILE as $, effectiveAnalystProtocolSha256 as A, loadCodeTraceVerificationArtifacts as B, adaptPublicBenchmarkFindings as C, renderCodeTraceCalibrationMarkdown as D, readAnalystBenchmarkArtifact as E, publicBenchmarkProtocolSha256 as F, ANALYST_BENCHMARK_EVIDENCE_DEPENDENCY_LOCK_SHA256 as G, ANALYST_BENCHMARK_DEPENDENCY_LOCK_DIGEST_ALGORITHM as H, publicBenchmarkRlmInstructions as I, ANALYST_BENCHMARK_IMPLEMENTATION_FILES as J, ANALYST_BENCHMARK_EVIDENCE_IMPLEMENTATION_SHA256 as K, publicBenchmarkSystemPrompt as L, CODE_TRACE_BENCH_ANALYST_PROMPT as M, MAX_INCORRECT_BLOCKS as N, summarizeCodeTraceCalibration as O, MAX_INCORRECT_BLOCK_STEPS as P, ANALYST_BENCHMARK_COST_LEDGER_FILE as Q, DEFAULT_MAX_VERIFICATION_ARTIFACT_BYTES as R, analystDefinitionProtocolSha256 as S, expandCodeTraceFailureBlocks as T, ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES as U, parseVerificationOutcome as V, ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 as W, analystBenchmarkDependencyLockDigest as X, ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 as Y, analystBenchmarkImplementationDigest as Z, createPublicBenchmarkDirectRunner as _, primeCodeTraceAnalystDefinition as a, summarizeAgentRxCalibration as at, AnalystExpressivenessError as b, loadPublicBenchmarkRows as c, agentRxBenchmarkCase as ct, publicBenchmarkSelectionReport as d, roundAgentRxStep as dt, ANALYST_BENCHMARK_MANIFEST_FILE as et, selectPublicBenchmarkRows as f, normalizeBenchmarkLabel as ft, runReplVariableAnalystDefinition as g, rlmEngineLimits as h, primeAnalystProtocolSha256 as i, renderAgentRxCalibrationMarkdown as it, readAnalystInstructionsOverride as j, analystInstructionsOverrideFromText as k, preparePublicAnalystBenchmark as l, agentRxPredictionsToFindings as lt, publicRlmAnalystDefinition as m, renderAnalystBenchmarkMarkdown as n, compareAnalystRunners as nt, runInlineAnalystDefinition as o, codeTraceBenchCase as ot, createPublicBenchmarkRlmRunner as p, ANALYST_BENCHMARK_IMPLEMENTATION_DIGEST_ALGORITHM as q, createPrimeBenchmarkRunner as r, AGENT_RX_UPSTREAM_REVISION as rt, nodeHttpPrimeBridgeTransport as s, codeTracerPredictionsToFindings as st, runAnalystBenchmarkCommand as t, ANALYST_BENCHMARK_OBSERVATIONS_FILE as tt, publicBenchmarkDistributions as u, normalizeAgentRxCategory as ut, publicDirectAnalystDefinition as v, emptyPublicBenchmarkRunner as w, analystDefinitionAsymmetries as x, runChunkedAnalystDefinition as y, appendVerificationArtifactsToOtlp as z };
 
-//# sourceMappingURL=benchmark-command-CF-4GEWZ.js.map
+//# sourceMappingURL=benchmark-command-D8k3Gf0J.js.map