@tangle-network/agent-eval 0.186.2 → 0.187.1

package/dist/trajectory-replay/index.js

@@ -4,7 +4,7 @@ import { t as TraceEmitter } from "../emitter-DeQHiDMm.js";
 import { t as runCounterfactual } from "../counterfactual-Bjq1mlUu.js";
 import { n as InMemoryTraceStore } from "../store-DNe_Uv1Q.js";
 import { t as packageVersion } from "../package-version-D7lQHt_-.js";
-import { a as assertReplayableTrajectory, c as deriveFailureSignature, d as isRecordedTimeout, f as isSubmitAction, g as unreadableExitCount, h as parseRecordedReturncode, i as TIMEOUT_OBSERVATION_MARKER, l as finalRecordedOutcome, m as parseObservationOutput, n as FORMAT_ERROR_OBSERVATION_PREFIX, o as classifyObservation, p as isSubmitOnlyAction, r as SUBMIT_ACTION_SIGNATURE, s as decodeRecordedTurns, t as wrapActionForExec, u as isElidedField } from "../exec-D9WpA2p-.js";
+import { a as assertReplayableTrajectory, c as deriveFailureSignature, d as isRecordedTimeout, f as isSubmitAction, g as unreadableExitCount, h as parseRecordedReturncode, i as TIMEOUT_OBSERVATION_MARKER, l as finalRecordedOutcome, m as parseObservationOutput, n as FORMAT_ERROR_OBSERVATION_PREFIX, o as classifyObservation, p as isSubmitOnlyAction, r as SUBMIT_ACTION_SIGNATURE, s as decodeRecordedTurns, t as wrapActionForExec, u as isElidedField } from "../exec-ChJ6d5if.js";
 import { createHash } from "node:crypto";
 import { appendFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
@@ -198,7 +198,7 @@ function enumerateReplayableCases(corpora) {
 				missingGoldId = goldId;
 				break;
 			}
-			if (isSubmitAction(step.action)) {
+			if (step.action.trim() === "submit" || isSubmitOnlyAction(step.action)) {
 				submitGoldsSkipped += 1;
 				continue;
 			}
@@ -223,12 +223,22 @@ function enumerateReplayableCases(corpora) {
 			});
 			continue;
 		}
+		const recordedReturncodeAtK = parseRecordedReturncode(target.observation);
+		if (recordedReturncodeAtK === null) {
+			excluded.push({
+				corpus: corpus.name,
+				trajId,
+				reason: "no-recorded-returncode-at-k",
+				detail: `k=${target.step_id}; arm A cannot confirm execution without a recorded returncode`
+			});
+			continue;
+		}
 		replayable.push({
 			...resources,
 			goldIncorrectSteps: gold,
 			k: target.step_id,
 			submitGoldsSkipped,
-			recordedReturncodeAtK: parseRecordedReturncode(target.observation)
+			recordedReturncodeAtK
 		});
 	}
 	return {
@@ -831,7 +841,7 @@ async function replayVerify(options) {
 	if (target.step_id !== options.at) throw new Error(`trajectory-replay: steps[${index}].step_id=${target.step_id} != at ${options.at}; steps.json must be ordered with 1-based contiguous step_ids`);
 	const caseId = options.caseId ?? options.stepsPath;
 	const recordedReturncode = parseRecordedReturncode(target.observation);
-	const signature = options.signature ?? deriveFailureSignature(target.observation);
+	const signature = options.signature === void 0 ? deriveFailureSignature(target.observation) : options.signature.trim() || null;
 	const signatureBasis = signature ? "returncode+output-substring" : "returncode-only";
 	const runnerOptions = {
 		cwd: options.cwd,
@@ -853,7 +863,9 @@ async function replayVerify(options) {
 	const armAMs = Date.now() - armAStart;
 	const armAExec = requireExec(armARunner, "arm A");
 	const armAOutput = `${armAExec.stdout}\n${armAExec.stderr}`;
-	const failureSignatureMatch = recordedReturncode !== null && armAExec.exitCode === recordedReturncode && (signature ? armAOutput.includes(signature) : true);
+	const returncodeMatch = recordedReturncode !== null && armAExec.exitCode === recordedReturncode;
+	const failureSignatureMatch = returncodeMatch && signature !== null && armAOutput.includes(signature);
+	const failureReproduced = recordedReturncode !== 0 && returncodeMatch && (signature === null || failureSignatureMatch);
 	let armBRunner = null;
 	let armBExec = null;
 	let armBRunId = null;
@@ -909,7 +921,7 @@ async function replayVerify(options) {
 			exitCode: armBExec.exitCode,
 			wallMs: armBExec.wallMs,
 			prefix: requirePrefix(armBRunner, "arm B"),
-			failureVanished: armBExec.exitCode === 0 && (signature && armBOutput ? !armBOutput.includes(signature) : true)
+			failureVanished: requirePrefix(armBRunner, "arm B").prefixWithinTolerance && armBExec.exitCode === 0 && (signature && armBOutput ? !armBOutput.includes(signature) : true)
 		} : null,
 		timings: {
 			armAMs,
@@ -926,14 +938,15 @@ async function replayVerify(options) {
 	if (armAPrefix.prefixUnknownExpectations > 0) assumptions.push(`${armAPrefix.prefixUnknownExpectations} prefix step(s) carry no recorded returncode — agreement with the recording is unestablished there`);
 	if (options.prefixLimit !== void 0 && options.prefixLimit < index) assumptions.push(`prefix truncated at ${options.prefixLimit} of ${index} recorded steps — state fidelity at k is weakened`);
 	if (signatureBasis === "returncode-only") assumptions.push("no output substring derived from the recording — reproduction rests on the returncode alone");
-	const valid = core.prefixWithinTolerance && failureSignatureMatch && (core.armB === null || core.armB.failureVanished);
+	if (core.armB !== null && !core.armB.prefix.prefixWithinTolerance) assumptions.push("arm B prefix diverged beyond tolerance — the corrected step ran from a different state");
+	const valid = core.prefixWithinTolerance && failureReproduced && (core.armB === null || core.armB.failureVanished);
 	const verdict = {
 		...core,
 		valid,
 		score: valid ? 1 : 0,
 		scores: {
 			prefixFidelity: 1 - core.prefixDivergencePct / 100,
-			armAReproduced: failureSignatureMatch ? 1 : 0,
+			armAReproduced: failureReproduced ? 1 : 0,
 			...core.armB === null ? {} : { armBFailureVanished: core.armB.failureVanished ? 1 : 0 }
 		},
 		certification: {
@@ -1037,8 +1050,9 @@ function renderReport(verdict, target, armA, armB) {
 *
 * Headline metrics:
 *   replayability rate — fraction of replayable cases where the prefix
-*     replays within the divergence tolerance AND arm A reproduces the
-*     recorded returncode at k;
+*     replays within the divergence tolerance AND arm A reproduces a
+*     recorded failure at k; a zero exit without an error signature is not
+*     evidence of failure;
 *   prefix fidelity — executed prefix steps and the share of them that did
 *     not confirm the recording, split by kind. A corpus whose recordings
 *     carry no returncodes shows up here as unknown-expectation steps, never
@@ -1097,7 +1111,7 @@ async function executeArmB(replayCase, fixCommand, backend, signature, stepTimeo
 		prefixExecuted: prefix.prefixExecuted,
 		prefixDivergences: prefix.prefixDivergences.length,
 		prefixDivergencePct: prefix.prefixDivergencePct,
-		failureVanished: exec.exitCode === 0 && (signature ? !output.includes(signature) : true),
+		failureVanished: prefix.prefixWithinTolerance && exec.exitCode === 0 && (signature ? !output.includes(signature) : true),
 		stdout: exec.stdout,
 		stderr: exec.stderr
 	};
@@ -1205,7 +1219,7 @@ async function runReplayBatch(options) {
 				armAExit: verdict.armA.exitCode,
 				armAReturncodeMatch: returncodeMatch,
 				armASignatureMatch: verdict.armA.failureSignatureMatch,
-				replayed: verdict.prefixWithinTolerance && returncodeMatch,
+				replayed: verdict.valid,
 				fix: null,
 				wallMs: Date.now() - caseStart
 			};
@@ -1557,7 +1571,7 @@ function renderBatchReport(report) {
 	lines.push("## Headline");
 	lines.push("");
 	const fidelity = headline.prefixFidelity;
-	lines.push(`- **Replayability rate: ${pct(headline.replayabilityRate.value)}** (${headline.replayabilityRate.numerator}/${headline.replayabilityRate.denominator} replayable cases where the prefix replayed within ${fidelity.tolerancePct}% divergence AND arm A reproduced the recorded returncode at the gold step k).`);
+	lines.push(`- **Replayability rate: ${pct(headline.replayabilityRate.value)}** (${headline.replayabilityRate.numerator}/${headline.replayabilityRate.denominator} replayable cases where the prefix replayed within ${fidelity.tolerancePct}% divergence AND arm A reproduced an observed failure at the gold step k).`);
 	lines.push(`- Signature-strict rate: ${pct(headline.signatureStrictRate.value)} (${headline.signatureStrictRate.numerator}/${headline.signatureStrictRate.denominator}; additionally requires the recorded error substring in arm A output).`);
 	lines.push(`- **Prefix divergence: ${fidelity.divergencePct === null ? "—" : `${fidelity.divergencePct}%`}** (${fidelity.divergentSteps}/${fidelity.executedSteps} executed prefix steps did not confirm the recording: ${fidelity.returncodeMismatches} returncode mismatches, ${fidelity.unknownExpectations} with no recorded returncode to check). ${fidelity.casesWithinTolerance}/${fidelity.casesExecuted} executed cases are within the ${fidelity.tolerancePct}% tolerance.`);
 	if (headline.fixFlipRate) lines.push(`- **Fix-flip rate: ${pct(headline.fixFlipRate.value)}** (${headline.fixFlipRate.numerator}/${headline.fixFlipRate.denominator} arm-B-executed cases where the generated fix made the failure vanish).`);
@@ -1671,7 +1685,11 @@ function resolveFindingInvocation(finding, corpora) {
 		const resolution = resolveCaseResources(corpus, finding.trajId);
 		if (resolution.resolved) {
 			const at = subject.firstStep;
-			if (!resolution.resources.steps.find((s) => s.step_id === at)) throw new Error(`trajectory-replay: finding step ${at} is outside ${finding.trajId} (${resolution.resources.steps.length} steps)`);
+			const step = resolution.resources.steps.find((s) => s.step_id === at);
+			if (!step) throw new Error(`trajectory-replay: finding step ${at} is outside ${finding.trajId} (${resolution.resources.steps.length} steps)`);
+			const recordedReturncode = parseRecordedReturncode(step.observation);
+			if (recordedReturncode === null) throw new Error(`trajectory-replay: finding step ${at} in ${finding.trajId} has no recorded returncode`);
+			if (recordedReturncode === 0) throw new Error(`trajectory-replay: finding step ${at} in ${finding.trajId} exited 0; shell replay cannot verify its semantic error`);
 			return {
 				resources: resolution.resources,
 				subject,
@@ -1876,14 +1894,14 @@ function resolveFindingReplayability(finding, source) {
 	};
 }
 /**
-* Arm A reproduced on a prefix the recording confirmed → the fix flipping it
+* Arm A reproduced an observed failure on a confirmed prefix → a fix flipping it
 * beats plain reproduction; anything else diverged. A proof standing on a
 * prefix outside the divergence tolerance is divergent no matter what arm A
 * did: the state it ran against is not the recorded state.
 */
 function classifyVerdict(verdict) {
 	if (!verdict.prefixWithinTolerance) return "divergent";
-	if (!verdict.armA.failureSignatureMatch) return "divergent";
+	if (verdict.scores?.armAReproduced !== 1) return "divergent";
 	if (verdict.armB?.failureVanished) return "fix-flipped";
 	return "reproduced";
 }
@@ -1898,6 +1916,7 @@ function receiptExecution(verdict) {
 			command: verdict.armA.command,
 			exitCode: verdict.armA.exitCode,
 			wallMs: verdict.armA.wallMs,
+			reproduced: verdict.scores?.armAReproduced === 1,
 			failureSignatureMatch: verdict.armA.failureSignatureMatch
 		},
 		armB: verdict.armB ? {