@tangle-network/agent-eval 0.216.0 → 0.217.1

package/dist/trajectory-replay/index.js

@@ -1,16 +1,109 @@
-import { t as mulberry32 } from "../random-Dn5fPWkt.js";
-import { t as certificationEvidenceDigest } from "../verdict-DhPoXApk.js";
+import { o as NotFoundError, s as ValidationError } from "../errors-BcOs5MKW.js";
+import { t as mulberry32 } from "../random-DDDcC7YU.js";
+import { t as certificationEvidenceDigest } from "../verdict-BGQmxQ8A.js";
 import { t as TraceEmitter } from "../emitter-DbMXUIVP.js";
-import { t as runCounterfactual } from "../counterfactual-DtMWvn7D.js";
 import { n as InMemoryTraceStore } from "../store-BhNuLdJw.js";
 import { t as packageVersion } from "../package-version-D7lQHt_-.js";
-import { a as assertReplayableTrajectory, c as deriveFailureSignature, d as isRecordedTimeout, f as isSubmitAction, g as unreadableExitCount, h as parseRecordedReturncode, i as TIMEOUT_OBSERVATION_MARKER, l as finalRecordedOutcome, m as parseObservationOutput, n as FORMAT_ERROR_OBSERVATION_PREFIX, o as classifyObservation, p as isSubmitOnlyAction, r as SUBMIT_ACTION_SIGNATURE, s as decodeRecordedTurns, t as wrapActionForExec, u as isElidedField } from "../exec-ChJ6d5if.js";
+import { t as buildTrajectory } from "../trajectory-D_7rLrvE.js";
+import { a as classifyObservation, c as finalRecordedOutcome, d as isSubmitOnlyAction, f as parseObservationOutput, o as decodeRecordedTurns, p as parseRecordedReturncode, s as deriveFailureSignature, t as wrapActionForExec, u as isSubmitAction } from "../exec-CpbtOXVt.js";
 import { createHash } from "node:crypto";
 import { appendFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { tmpdir } from "node:os";
 import { execFile } from "node:child_process";
 import { promisify } from "node:util";
+//#region src/counterfactual.ts
+/**
+* Counterfactual replay — "what would have happened if we'd changed
+* exactly one thing at turn N?"
+*
+* The framework does NOT drive the agent — it sets up the replay
+* context (prior spans, prior state, mutation spec) and records the
+* resulting divergence. Consumers supply an `executeFrom(ctx)` callback
+* that runs their agent starting from turn N with the mutation applied.
+*
+* Counterfactual runs are recorded as a new Run with `layer='meta'` and
+* `parentRunId = originalRunId`, so downstream diff + correlation
+* pipelines see them natively.
+*/
+async function runCounterfactual(store, originalRunId, mutation, runner) {
+	const originalRun = await store.getRun(originalRunId);
+	if (!originalRun) throw new NotFoundError(`counterfactual: run ${originalRunId} not found`);
+	const trajectory = await buildTrajectory(store, originalRunId);
+	if (mutation.at < 0 || mutation.at >= trajectory.steps.length) throw new ValidationError(`counterfactual: mutation.at=${mutation.at} out of range [0, ${trajectory.steps.length})`);
+	const targetStep = trajectory.steps[mutation.at];
+	const mutatedStep = applyMutation(targetStep, mutation);
+	const cfEmitter = new TraceEmitter(store);
+	await cfEmitter.startRun({
+		scenarioId: originalRun.scenarioId,
+		variantId: originalRun.variantId ? `${originalRun.variantId}+cf:${mutation.kind}@${mutation.at}` : `cf:${mutation.kind}@${mutation.at}`,
+		projectId: originalRun.projectId,
+		parentRunId: originalRunId,
+		layer: "meta",
+		tags: {
+			counterfactual: "true",
+			mutationKind: mutation.kind,
+			mutationAt: String(mutation.at)
+		}
+	});
+	await runner.executeFrom({
+		originalRunId,
+		originalTrajectory: trajectory,
+		prefix: trajectory.steps.slice(0, mutation.at),
+		mutation,
+		mutatedStep
+	}, cfEmitter);
+	const counterfactual = await store.getRun(cfEmitter.runId);
+	const delta = {
+		originalOutcomeScore: originalRun.outcome?.score ?? null,
+		counterfactualOutcomeScore: counterfactual?.outcome?.score ?? null,
+		deltaScore: originalRun.outcome?.score !== void 0 && counterfactual?.outcome?.score !== void 0 ? counterfactual.outcome.score - originalRun.outcome.score : null
+	};
+	return {
+		counterfactualRunId: cfEmitter.runId,
+		originalRunId,
+		mutation,
+		delta
+	};
+}
+function applyMutation(step, mutation) {
+	if (mutation.kind === "swap-model" && step.span.kind === "llm") {
+		const llm = step.span;
+		return {
+			...step,
+			span: {
+				...llm,
+				model: mutation.newModel
+			}
+		};
+	}
+	if (mutation.kind === "swap-tool-result" && step.span.kind === "tool") {
+		const tool = step.span;
+		return {
+			...step,
+			span: {
+				...tool,
+				result: mutation.newResult
+			}
+		};
+	}
+	if (mutation.kind === "inject-system-message" && step.span.kind === "llm") {
+		const llm = step.span;
+		return {
+			...step,
+			span: {
+				...llm,
+				messages: [{
+					role: "system",
+					content: mutation.content
+				}, ...llm.messages]
+			}
+		};
+	}
+	if (mutation.kind === "custom") return mutation.apply(step);
+	return step;
+}
+//#endregion
 //#region src/trajectory-replay/corpus.ts
 /**
 * Corpus enumeration for batch replay verification.
@@ -27,20 +120,6 @@ import { promisify } from "node:util";
 * is returned with a machine-readable reason — the batch report surfaces the
 * full exclusion table, never a silently shrunk denominator.
 */
-/** Parses `name=<labelsPath>::<preparedDir>` (paths may contain `=`, not `::`). */
-function parseCorpusFlag(value) {
-	const eq = value.indexOf("=");
-	if (eq <= 0) throw new Error(`--corpus must be name=<labels>::<prepared>, got: ${value}`);
-	const name = value.slice(0, eq);
-	const rest = value.slice(eq + 1);
-	const sep = rest.indexOf("::");
-	if (sep <= 0 || sep === rest.length - 2) throw new Error(`--corpus must be name=<labels>::<prepared>, got: ${value}`);
-	return {
-		name,
-		labelsPath: rest.slice(0, sep),
-		preparedDir: rest.slice(sep + 2)
-	};
-}
 function readLabelEntries(labelsPath) {
 	const parsed = JSON.parse(readFileSync(labelsPath, "utf8"));
 	if (!Array.isArray(parsed)) throw new Error(`${labelsPath} is not a JSON array of label entries`);
@@ -2116,44 +2195,7 @@ async function verifyFindings(findings, options) {
 	}, null, 2)}\n`);
 	return run;
 }
-function verdictCell(verification) {
-	switch (verification.verified) {
-		case "reproduced": return "**VERIFIED** — reproduced";
-		case "fix-flipped": return "**VERIFIED** — fix-flipped";
-		case "divergent": return "DIVERGENT — recorded failure did not reproduce";
-		case "not-replayable": return `UNVERIFIABLE — ${verification.reason ?? "no reason recorded"}`;
-	}
-}
-/** Markdown section an analysis report appends when finding verification ran. */
-function renderVerifiedFindingsSection(run) {
-	const lines = ["## Verified findings (executed replay)", ""];
-	const total = run.verifications.length;
-	lines.push(`${total} finding(s) → ${run.counts.reproduced} reproduced, ${run.counts["fix-flipped"]} fix-flipped, ${run.counts.divergent} divergent, ${run.counts["not-replayable"]} not replayable (${run.executions} execution(s); findings accusing the same step share one proof).`);
-	lines.push("");
-	lines.push("| Finding | Subject | Step | Verdict | Receipt |");
-	lines.push("|---|---|---:|---|---|");
-	for (const verification of run.verifications) {
-		const shared = verification.deduplicated_with ? " (shared proof)" : "";
-		lines.push(`| \`${verification.finding_id ?? "—"}\` | \`${verification.subject ?? "—"}\` | ${verification.step ?? "—"} | ${verdictCell(verification)} | \`${verification.receipt}\`${shared} |`);
-	}
-	lines.push("");
-	lines.push("VERIFIED = the accused step was re-executed after replaying the trajectory prefix, and the recorded failure signature reproduced (fix-flipped: a corrected command additionally made it vanish). Each receipt directory carries receipt.json and, when executed, replay-verdict.json + report.md with real stdout/stderr.");
-	lines.push("");
-	return lines.join("\n");
-}
-/**
-* Accepts the two shapes findings travel in: a bare JSON array of analyst
-* findings, or an object with a `findings` array (e.g. an extracted
-* `observations[n]` from a result.json).
-*/
-function readFindingsFile(path) {
-	const parsed = JSON.parse(readFileSync(path, "utf8"));
-	const array = Array.isArray(parsed) ? parsed : parsed && typeof parsed === "object" && Array.isArray(parsed.findings) ? parsed.findings : null;
-	if (!array) throw new Error(`${path} is neither a findings array nor an object with a findings array`);
-	for (const entry of array) if (!entry || typeof entry !== "object") throw new Error(`${path}: every finding must be an object, got ${JSON.stringify(entry)}`);
-	return array;
-}
 //#endregion
-export { FORMAT_ERROR_OBSERVATION_PREFIX, PREFIX_DIVERGENCE_TOLERANCE_PCT, SUBMIT_ACTION_SIGNATURE, SandboxCounterfactualRunner, TIMEOUT_OBSERVATION_MARKER, assertReplayableTrajectory, buildFixPrompt, buildRetryFixPrompt, classifyObservation, classifyPrefixStep, classifyVerdict, clipText, countScriptCommands, decodeRecordedTurns, deriveFailureSignature, derivedImageTag, dockerImagePreparer, enumerateReplayableCases, extractFixCommand, finalRecordedOutcome, findingReplayStep, findingTrajectoryId, generateFixCommand, goldIncorrectSteps, ingestRecordedTrajectory, isElidedField, isRecordedTimeout, isSubmitAction, isSubmitOnlyAction, parseCorpusFlag, parseIncorrectStepsSubject, parseObservationOutput, parseRecordedReturncode, readFindingsFile, renderVerifiedFindingsSection, replayVerify, replayVerifyFinding, resolveCaseResources, resolveFindingInvocation, resolveFindingReplayability, runFixLoop, runReplayBatch, seededSample, summarizePrefixReplay, unreadableExitCount, verifyFindings, wrapActionForExec };
+export { PREFIX_DIVERGENCE_TOLERANCE_PCT, classifyObservation, classifyVerdict, decodeRecordedTurns, dockerImagePreparer, finalRecordedOutcome, replayVerify, replayVerifyFinding, runReplayBatch, seededSample, verifyFindings };
 
 //# sourceMappingURL=index.js.map