@tangle-network/agent-eval 0.186.1 → 0.187.0

package/dist/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "@tangle-network/agent-eval — wire protocol",
-    "version": "0.186.1",
+    "version": "0.187.0",
     "description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.1.0. Bumps on breaking changes to request/response schemas.",
     "contact": {
       "name": "Tangle Network",

package/dist/{steps-CiNVJry_.d.ts → steps-Bg5-KVx1.d.ts}

@@ -46,9 +46,9 @@ declare function parseRecordedReturncode(observation: string | null): number | n
  *  the tags are absent. */
 declare function parseObservationOutput(observation: string | null): string;
 /**
- * Stable failure-signature candidate: the first line of the recorded output
- * that contains the word "error". Null when no such line exists — a verdict
- * then falls back to returncode-only matching and says so.
+ * Stable failure-signature candidate: the first error line, or a shell
+ * command-not-found suffix. Null when neither exists — a verdict then falls
+ * back to returncode-only matching and says so.
  * Pass an explicit signature to override (compiler quote glyphs vary with
  * locale, so a hand-picked ASCII substring is often more robust).
  */
@@ -198,4 +198,4 @@ declare function unreadableExitCount(steps: readonly RecordedTrajectoryStep[]):
 declare function assertReplayableTrajectory(decoded: DecodedTrajectory): void;
 //#endregion
 export { isSubmitOnlyAction as _, RecordedTrajectoryStep as a, unreadableExitCount as b, TIMEOUT_OBSERVATION_MARKER as c, decodeRecordedTurns as d, deriveFailureSignature as f, isSubmitAction as g, isRecordedTimeout as h, RecordedObservationKind as i, assertReplayableTrajectory as l, isElidedField as m, FORMAT_ERROR_OBSERVATION_PREFIX as n, RecordedTrajectoryTurn as o, finalRecordedOutcome as p, RecordedFinalOutcome as r, SUBMIT_ACTION_SIGNATURE as s, DecodedTrajectory as t, classifyObservation as u, parseObservationOutput as v, parseRecordedReturncode as y };
-//# sourceMappingURL=steps-CiNVJry_.d.ts.map
+//# sourceMappingURL=steps-Bg5-KVx1.d.ts.map

package/dist/{steps-CiNVJry_.d.ts.map → steps-Bg5-KVx1.d.ts.map}

@@ -1 +1 @@
-{"version":3,"file":"steps-CiNVJry_.d.ts","names":[],"sources":["../src/trajectory-replay/steps.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;UAmCiB;;WAEN;WACA;;WAEA;;;iBAIK,wBAAwB;;;iBAQxB,uBAAuB;;;;;;;;iBAavB,uBAAuB;;;;;cAW1B;iBAEG,eAAe;;;;;;;;;;iBAaf,mBAAmB;iBAwBnB,cAAc;;cAOjB;;cAGA;;;;;;;;;;;KAaD;iBAQI,oBAAoB,6BAA6B;;;;iBAYjD,kBAAkB;;;;;;;UAYjB;WACN;WACA;WACA;aAA4B;;WAC5B;;UAGM;;WAEN,gBAAgB;;WAEhB;;;;;;;;;WASA;;WAEA;;WAEA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAmCK,oBAAoB,gBAAgB,2BAA2B;;;;;;;;;KAyCnE;WACG;WAA6B;;WAC7B;;WACA;WAA6B,QAAQ;;;;;;;;iBAQpC,qBACd,gBAAgB,2BACf;;;;;;;iBAiBa,oBAAoB,gBAAgB;;;;;;;;;iBAYpC,2BAA2B,SAAS"}
+{"version":3,"file":"steps-Bg5-KVx1.d.ts","names":[],"sources":["../src/trajectory-replay/steps.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;UAmCiB;;WAEN;WACA;;WAEA;;;iBAIK,wBAAwB;;;iBAQxB,uBAAuB;;;;;;;;iBAavB,uBAAuB;;;;;cAc1B;iBAEG,eAAe;;;;;;;;;;iBAaf,mBAAmB;iBAwBnB,cAAc;;cAOjB;;cAGA;;;;;;;;;;;KAaD;iBAQI,oBAAoB,6BAA6B;;;;iBAYjD,kBAAkB;;;;;;;UAYjB;WACN;WACA;WACA;aAA4B;;WAC5B;;UAGM;;WAEN,gBAAgB;;WAEhB;;;;;;;;;WASA;;WAEA;;WAEA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAmCK,oBAAoB,gBAAgB,2BAA2B;;;;;;;;;KAyCnE;WACG;WAA6B;;WAC7B;;WACA;WAA6B,QAAQ;;;;;;;;iBAQpC,qBACd,gBAAgB,2BACf;;;;;;;iBAiBa,oBAAoB,gBAAgB;;;;;;;;;iBAYpC,2BAA2B,SAAS"}

package/dist/trace-repair/index.d.ts

@@ -4,7 +4,7 @@ import { u as RunTokenUsage } from "../run-record-D9wS35fK.js";
 import { t as DefaultVerdict } from "../verdict-E4eRNf7-.js";
 import { a as TraceAnalystLimits, n as TraceAnalysisEngine } from "../engine-DNlKhIm0.js";
 import { t as PrimeBridgeTransport } from "../prime-bridge-transport-6feEglLf.js";
-import { a as RecordedTrajectoryStep, h as isRecordedTimeout } from "../steps-CiNVJry_.js";
+import { a as RecordedTrajectoryStep, h as isRecordedTimeout } from "../steps-Bg5-KVx1.js";
 //#region src/trace-repair/mini-swe-scaffold.d.ts
 /** Whole-line marker that ends a run. The first output line must equal it and the command must exit 0. */
 declare const SUBMIT_SENTINEL = "COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT";

package/dist/trace-repair/index.js

@@ -6,7 +6,7 @@ import { t as certificationEvidenceDigest } from "../verdict-DhPoXApk.js";
 import { t as packageVersion } from "../package-version-D7lQHt_-.js";
 import { t as assertEqualDeclarativeTerms } from "../equal-terms-Blx9Zk5I.js";
 import { d as runPrimeExchange, n as buildPrimePrompt, t as analystUsageReceiptFromPrimeUsage } from "../prime-protocol-DcAK9pC8.js";
-import { c as deriveFailureSignature, d as isRecordedTimeout, h as parseRecordedReturncode, i as TIMEOUT_OBSERVATION_MARKER, n as FORMAT_ERROR_OBSERVATION_PREFIX, r as SUBMIT_ACTION_SIGNATURE, t as wrapActionForExec } from "../exec-D9WpA2p-.js";
+import { c as deriveFailureSignature, d as isRecordedTimeout, h as parseRecordedReturncode, i as TIMEOUT_OBSERVATION_MARKER, n as FORMAT_ERROR_OBSERVATION_PREFIX, r as SUBMIT_ACTION_SIGNATURE, t as wrapActionForExec } from "../exec-ChJ6d5if.js";
 import { createHash } from "node:crypto";
 import { constants } from "node:os";
 import { spawn } from "node:child_process";

package/dist/trajectory-replay/index.d.ts

@@ -2,7 +2,7 @@ import { s as TraceStore } from "../store-BErPvYBr.js";
 import { t as DefaultVerdict } from "../verdict-E4eRNf7-.js";
 import { i as TraceEmitter } from "../emitter-Cs0egaFd.js";
 import { i as CounterfactualRunner, t as CounterfactualContext } from "../counterfactual-CLgrwhkY.js";
-import { _ as isSubmitOnlyAction, a as RecordedTrajectoryStep, b as unreadableExitCount, c as TIMEOUT_OBSERVATION_MARKER, d as decodeRecordedTurns, f as deriveFailureSignature, g as isSubmitAction, h as isRecordedTimeout, i as RecordedObservationKind, l as assertReplayableTrajectory, m as isElidedField, n as FORMAT_ERROR_OBSERVATION_PREFIX, o as RecordedTrajectoryTurn, p as finalRecordedOutcome, r as RecordedFinalOutcome, s as SUBMIT_ACTION_SIGNATURE, t as DecodedTrajectory, u as classifyObservation, v as parseObservationOutput, y as parseRecordedReturncode } from "../steps-CiNVJry_.js";
+import { _ as isSubmitOnlyAction, a as RecordedTrajectoryStep, b as unreadableExitCount, c as TIMEOUT_OBSERVATION_MARKER, d as decodeRecordedTurns, f as deriveFailureSignature, g as isSubmitAction, h as isRecordedTimeout, i as RecordedObservationKind, l as assertReplayableTrajectory, m as isElidedField, n as FORMAT_ERROR_OBSERVATION_PREFIX, o as RecordedTrajectoryTurn, p as finalRecordedOutcome, r as RecordedFinalOutcome, s as SUBMIT_ACTION_SIGNATURE, t as DecodedTrajectory, u as classifyObservation, v as parseObservationOutput, y as parseRecordedReturncode } from "../steps-Bg5-KVx1.js";
 //#region src/trajectory-replay/corpus.d.ts
 interface CorpusSpec {
   readonly name: string;
@@ -11,7 +11,7 @@ interface CorpusSpec {
 }
 /** Parses `name=<labelsPath>::<preparedDir>` (paths may contain `=`, not `::`). */
 declare function parseCorpusFlag(value: string): CorpusSpec;
-type ReplayExclusionReason = 'no-swe-raw-trajectory' | 'ambiguous-swe-raw-trajectory' | 'unreadable-raw-trajectory' | 'no-docker-image' | 'no-gold-incorrect-step' | 'gold-only-submit-step' | 'missing-steps-json' | 'gold-step-outside-steps' | 'cwd-underivable';
+type ReplayExclusionReason = 'no-swe-raw-trajectory' | 'ambiguous-swe-raw-trajectory' | 'unreadable-raw-trajectory' | 'no-docker-image' | 'no-gold-incorrect-step' | 'gold-only-submit-step' | 'missing-steps-json' | 'gold-step-outside-steps' | 'cwd-underivable' | 'no-recorded-returncode-at-k';
 interface ExcludedCase {
   readonly corpus: string;
   readonly trajId: string;
@@ -36,7 +36,8 @@ interface ReplayableCase extends CaseResources {
   /** 1-based gold incorrect step ids, ascending. */
   readonly goldIncorrectSteps: readonly number[];
   /** k — the first gold incorrect step that is a real mid-trajectory action;
-   *  submit-command golds are skipped (see SUBMIT_ACTION_SIGNATURE). */
+   *  submit-only golds are skipped. A step that also changes state stays in
+   *  the trajectory and is excluded below if its exit was not recorded. */
   readonly k: number;
   /** Gold steps before k skipped because their action is the submit command. */
   readonly submitGoldsSkipped: number;
@@ -358,8 +359,8 @@ interface ReplayBatchCaseRow {
   readonly armAExit: number | null;
   readonly armAReturncodeMatch: boolean;
   readonly armASignatureMatch: boolean;
-  /** Headline predicate: prefix divergence within tolerance AND arm A
-   *  reproduced the recorded returncode at k. */
+  /** Headline predicate: prefix within tolerance and arm A reproduced an
+   *  observed failure at k. */
   readonly replayed: boolean;
   readonly fix: ReplayBatchFixResult | null;
   readonly wallMs: number;
@@ -377,7 +378,7 @@ interface ReplayBatchReport {
     readonly executed: number;
     readonly excludedByReason: Record<string, number>;
     /** Per-corpus submit-gold accounting: cases dropped because every gold is
-     *  the submit command, and golds skipped inside still-replayable cases. */
+     *  a pure submit, and such golds skipped inside replayable cases. */
     readonly submitGoldsByCorpus: Record<string, {
       submitOnlyCases: number;
       goldsSkippedWithinReplayable: number;
@@ -413,8 +414,7 @@ interface ReplayBatchReport {
       denominator: number;
       value: number | null;
     } | null;
-    /** Fix-flip restricted to cases whose recorded returncode at k is nonzero —
-     *  real recorded failures, where "the failure vanished" is not vacuous. */
+    /** Fix-flip restricted to cases whose recorded returncode at k is nonzero. */
     readonly fixFlipRateNonzeroRc: {
       numerator: number;
       denominator: number;
@@ -564,9 +564,9 @@ interface ReplayArmVerdict {
   prefix: PrefixReplayResult;
 }
 /** Extends the substrate verdict spine: `valid` = the claim the caller asked
- *  to verify held under execution — arm A reproduced the recorded failure on
+ *  to verify held under execution — arm A reproduced a recorded failure on
  *  an in-tolerance prefix, and, when arm B ran, the corrected step made the
- *  failure vanish. `score` is the same bit as a number; the sub-measurements
+ *  failure vanish on another in-tolerance prefix. `score` is the same bit as a number; the sub-measurements
  *  live in `scores`. Certified as `'replication'`: a re-execution from the
  *  pinned image, deterministic given the pins. */
 interface ReplayVerdict extends DefaultVerdict {
@@ -717,7 +717,7 @@ interface VerifyFindingsOptions {
   readonly onProgress?: (message: string) => void;
 }
 /**
- * Arm A reproduced on a prefix the recording confirmed → the fix flipping it
+ * Arm A reproduced an observed failure on a confirmed prefix → a fix flipping it
  * beats plain reproduction; anything else diverged. A proof standing on a
  * prefix outside the divergence tolerance is divergent no matter what arm A
  * did: the state it ran against is not the recorded state.

package/dist/trajectory-replay/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","names":[],"sources":["../../src/trajectory-replay/corpus.ts","../../src/trajectory-replay/exec.ts","../../src/trajectory-replay/fix.ts","../../src/trajectory-replay/fix-loop.ts","../../src/trajectory-replay/image-preparer.ts","../../src/trajectory-replay/batch.ts","../../src/trajectory-replay/verify.ts","../../src/trajectory-replay/findings.ts","../../src/trajectory-replay/wire.ts"],"mappings":";;;;;;UAyBiB;WACN;WACA;WACA;;;iBAIK,gBAAgB,gBAAgB;KAYpC;UAWK;WACN;WACA;WACA,QAAQ;WACR;;KAGC;;UAGK;WACN;WACA;WACA;WACA,gBAAgB;WAChB;WACA;WACA;WACA,WAAW;;WAEX;;UAGM,uBAAuB;;WAE7B;;;WAGA;;WAEA;;WAEA;;UAGD;WACC;WACA;aAAuC;;;iBAGlC,iBAAiB,qBAAqB;iBAWtC,mBAAmB,OAAO;KAuD9B;WACG;WAAyB,WAAW;;WACpC;WAA0B,QAAQ;WAAgC;;;;;;iBAMjE,qBAAqB,QAAQ,YAAY,iBAAiB;UAiEzD;WACN,YAAY;WACZ,UAAU;WACV;;;;;;;;;;iBAWK,yBAAyB,kBAAkB,eAAe;;;;;;;;;;;;UC9OzD;EACf;EACA;EACA;;UAGe;EACf,KAAK,iBAAiB,oBAAoB,QAAQ;EAClD,SAAS;;UAGM;;EAEf,QAAQ,QAAQ;;;;;KAMN,4BAA4B,kBAAkB;;;;;;;iBAQ1C,kBAAkB,gBAAgB;;;UCrBjC;WACN;WACA;;KAGC;WACG;WAA0B;IAAS;IAAiB,OAAO;;;WAC3D;WAA2B;;UAEzB;EACf,SAAS,gBAAgB,eAAe,QAAQ;;UAGjC;WACN;WACA,gBAAgB;;WAEhB;;WAEA;;;iBAIK,SAAS,cAAc;iBAuCvB,eAAe,OAAO;EAAmB;EAAgB;;;UAiBxD;WACN;;WAEA;WACA;WACA;WACA;WACA;;;;;;;iBA4BK,oBACd,OAAO,gBACP,wBAAwB,oBACxB;EACG;EAAgB;;;iBAwBL,oBAAoB;;iBAMpB,kBAAkB;KAQtB;WACG;WAA0B;IAAS;IAAiB,OAAO;;;WAC3D;WAA2B;;iBAEpB,mBACpB,QAAQ,sBACR,OAAO,iBACN,QAAQ;;;;UCvJM;WACN;WACA;WACA;;WAEA;WACA;WACA;WACA;;;KAIC,kBAAkB,iBAAiB,oBAAoB,QAAQ;UAE1D;;WAEN;;WAEA;;WAEA;WACA,cAAc;;UAGR;WACN;WACA;WACA;WACA;aAAkB;aAA+B;;WACjD;WACA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;;UAGM;WACN;WACA;WACA,mBAAmB;;WAEnB;WACA;;;WAGA;WACA;WACA;;;;WAIA;;iBAcW,WACpB,QAAQ,sBACR,OAAO,gBACP,UAAU,gBACV,SAAS,iBACR,QAAQ;;;;;;;;;;;;KCpFC;WAEG;WACA;IAAS;IAAsB;IAAiB;;;WAEhD;WAA2B;;UAEzB;EACf,OAAO,eAAe,cAAc,QAAQ;;UAG7B;WACN;WACA;;iBAGK,gBAAgB,eAAe;;;;;;;iBAW/B,oBAAoB,UAAS,6BAAkC;;;UCO9D;WACN,kBAAkB;WAClB;;;;WAIA;;WAEA;WACA,YAAY;WACZ;;WAEA;WACA;;WAEA;WACA;;WAEA;;WAEA;;WAEA,WAAW;;WAEX,gBAAgB;WAChB,cAAc;;UAGR;;WAEN;WACA;WACA;WACA;;;WAGA,OAAO;WACP;WACA;WACA;WACA;WACA;WACA;;WAEA,mBAAmB;;;WAGnB;;UAGM;WACN;WACA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;;WAEA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;;WAEA;WACA;;;WAGA;WACA;WACA;WACA;;;WAGA;WACA,KAAK;WACL;;UAGM;WACN;WACA;IAAoB;IAAc;IAAoB;;WACtD;aACE;aACA;aACA;aACA,kBAAkB;;;aAGlB,qBAAqB;MAE1B;MAAyB;;;WAGtB;aACE;MAAqB;MAAmB;MAAqB;;aAC7D;MAAuB;MAAmB;MAAqB;;;;;aAI/D;eACE;eACA;eACA;eACA;;eAEA;eACA;eACA;eACA;;aAEF;MAAe;MAAmB;MAAqB;;;;aAGvD;MACP;MACA;MACA;;;;aAIO;MACP;MACA;MACA;;;aAGO,gBAAgB;;WAElB;aACE;aACA;aACA;aACA;aACA;;;;aAIA;;WAEF;IAAqB;IAAgB;IAAgB;IAAgB;;WACrE;IAAyB;IAAgB;IAAgB;IAAe;;WACxE,gBAAgB;;iBAGX,aAAa,GAAG,gBAAgB,KAAK,cAAc,eAAe,IAAI;iBA+EhE,eAAe,SAAS,qBAAqB,QAAQ;iBAuhB3D,kBAAkB,QAAQ;;;UC9vBzB;EACf;EACA;;;;;;;iBAQoB,yBACpB,OAAO,YACP,gBAAgB,0BAChB,iBACC,QAAQ;;KA6BC;;;;;UAMK;EACf;EACA,MAAM;;EAEN;EACA;;UAGe;EACf;;EAEA,mBAAmB;;EAEnB;EACA;EACA;;;EAGA;;EAEA;EACA;;;;cAKW;;;;;iBAMG,mBACd,cACA,mCACA,qBACC;;iBAWa,sBACd,wBACA,mBAAmB,oBACnB,iBACC;UAwBc;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;;;EAGA;EACA,cAAc;;;;;;;;cAsBH,uCAAuC;mBAK/B;mBACA;EALnB,YAAY;EACZ,SAAS;EAET,YACmB,SAAS,mBACT,SAAS;EAGtB,YAAY,KAAK,uBAAuB,SAAS,eAAe;;UAyEvD;EACf;EACA;;EAEA;;EAEA;EACA;EACA;EACA;;EAEA;EACA;EACA;;EAEA;;EAEA,SAAS;EACT,cAAc;;UAGC;EACf;EACA;EACA;EACA,QAAQ;;;;;;;;UASO,sBAAsB;EACrC;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,mBAAmB;;EAEnB;EACA;EACA;EACA;EACA;EACA,MAAM;IAAqB;;EAC3B,OAAO;IAAqB;;EAC5B;IAAW;IAAgB;IAAuB;;EAClD;IAAU;IAAkB;IAAc;;;iBAUtB,aAAa,SAAS,sBAAsB,QAAQ;;;UC5TzD;WACN;WACA;WACA;WACA;WACA;WACA;aACE;aACA;aACA;;WAEF,WAAW,SAAS;;;;;;;;iBAWf,kBAAkB,SAAS;;iBAgB3B,oBAAoB,SAAS;;;;;;;;;;KAoBjC;WAEG;WACA;WACA;WACA;WACA;;WAGA;WACA,kBAAkB;WAClB,WAAW;;UAGT;WACN;WACA;;WAEA;WACA;;WAEA;WACA;WACA;;KAGC;WACG;WAA2B,UAAU;;WACrC;WAA4B;;;;;;;iBAiC3B,4BACd,SAAS,mBACT,QAAQ,sBACP;KAmFS;UAMK;WACN;WACA;WACA;;WAEA;WACA,UAAU;;WAEV;;WAEA;WACA;;WAEA;;;;;;;;UASM,0BAA0B;WAChC;WACA,wBAAwB;WACxB,QAAQ,SAAS,OAAO;;WAExB;;UAGM;WACN,QAAQ;;WAER;;WAEA;WACA;WACA;;WAEA,gBAAgB;;;WAGhB,kBAAkB;WAClB,cAAc;;;;;;;;iBAST,gBAAgB,SAAS,gBAAgB;;;;;;;iBAuGnC,eACpB,mBAAmB,qBACnB,SAAS,wBACR,QAAQ;;iBAwNK,8BAA8B,KAAK;;;;;;iBAmCnC,iBAAiB,eAAe;;;UCtoB/B;WACN;WACA;WACA;WACA;;;iBAMK,2BAA2B,kBAAkB;UAW5C;WACN;WACA;;UAGM;WACN,WAAW;WACX,SAAS;;WAET;;;;;;;;iBASK,yBACd,SAAS,sBACT,kBAAkB,eACjB;UA4Bc;WACN,kBAAkB;WAClB;;WAEA,YAAY;;WAEZ;WACA;WACA;;WAEA,gBAAgB;WAChB,cAAc;;UAGR;WACN,YAAY;WACZ;WACA,SAAS;;;;;;;iBAQE,oBACpB,SAAS,sBACT,SAAS,uBACR,QAAQ"}
+{"version":3,"file":"index.d.ts","names":[],"sources":["../../src/trajectory-replay/corpus.ts","../../src/trajectory-replay/exec.ts","../../src/trajectory-replay/fix.ts","../../src/trajectory-replay/fix-loop.ts","../../src/trajectory-replay/image-preparer.ts","../../src/trajectory-replay/batch.ts","../../src/trajectory-replay/verify.ts","../../src/trajectory-replay/findings.ts","../../src/trajectory-replay/wire.ts"],"mappings":";;;;;;UAyBiB;WACN;WACA;WACA;;;iBAIK,gBAAgB,gBAAgB;KAYpC;UAYK;WACN;WACA;WACA,QAAQ;WACR;;KAGC;;UAGK;WACN;WACA;WACA;WACA,gBAAgB;WAChB;WACA;WACA;WACA,WAAW;;WAEX;;UAGM,uBAAuB;;WAE7B;;;;WAIA;;WAEA;;WAEA;;UAGD;WACC;WACA;aAAuC;;;iBAGlC,iBAAiB,qBAAqB;iBAWtC,mBAAmB,OAAO;KAuD9B;WACG;WAAyB,WAAW;;WACpC;WAA0B,QAAQ;WAAgC;;;;;;iBAMjE,qBAAqB,QAAQ,YAAY,iBAAiB;UAiEzD;WACN,YAAY;WACZ,UAAU;WACV;;;;;;;;;;iBAWK,yBAAyB,kBAAkB,eAAe;;;;;;;;;;;;UChPzD;EACf;EACA;EACA;;UAGe;EACf,KAAK,iBAAiB,oBAAoB,QAAQ;EAClD,SAAS;;UAGM;;EAEf,QAAQ,QAAQ;;;;;KAMN,4BAA4B,kBAAkB;;;;;;;iBAQ1C,kBAAkB,gBAAgB;;;UCrBjC;WACN;WACA;;KAGC;WACG;WAA0B;IAAS;IAAiB,OAAO;;;WAC3D;WAA2B;;UAEzB;EACf,SAAS,gBAAgB,eAAe,QAAQ;;UAGjC;WACN;WACA,gBAAgB;;WAEhB;;WAEA;;;iBAIK,SAAS,cAAc;iBAuCvB,eAAe,OAAO;EAAmB;EAAgB;;;UAiBxD;WACN;;WAEA;WACA;WACA;WACA;WACA;;;;;;;iBA4BK,oBACd,OAAO,gBACP,wBAAwB,oBACxB;EACG;EAAgB;;;iBAwBL,oBAAoB;;iBAMpB,kBAAkB;KAQtB;WACG;WAA0B;IAAS;IAAiB,OAAO;;;WAC3D;WAA2B;;iBAEpB,mBACpB,QAAQ,sBACR,OAAO,iBACN,QAAQ;;;;UCvJM;WACN;WACA;WACA;;WAEA;WACA;WACA;WACA;;;KAIC,kBAAkB,iBAAiB,oBAAoB,QAAQ;UAE1D;;WAEN;;WAEA;;WAEA;WACA,cAAc;;UAGR;WACN;WACA;WACA;WACA;aAAkB;aAA+B;;WACjD;WACA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;;UAGM;WACN;WACA;WACA,mBAAmB;;WAEnB;WACA;;;WAGA;WACA;WACA;;;;WAIA;;iBAcW,WACpB,QAAQ,sBACR,OAAO,gBACP,UAAU,gBACV,SAAS,iBACR,QAAQ;;;;;;;;;;;;KCpFC;WAEG;WACA;IAAS;IAAsB;IAAiB;;;WAEhD;WAA2B;;UAEzB;EACf,OAAO,eAAe,cAAc,QAAQ;;UAG7B;WACN;WACA;;iBAGK,gBAAgB,eAAe;;;;;;;iBAW/B,oBAAoB,UAAS,6BAAkC;;;UCQ9D;WACN,kBAAkB;WAClB;;;;WAIA;;WAEA;WACA,YAAY;WACZ;;WAEA;WACA;;WAEA;WACA;;WAEA;;WAEA;;WAEA,WAAW;;WAEX,gBAAgB;WAChB,cAAc;;UAGR;;WAEN;WACA;WACA;WACA;;;WAGA,OAAO;WACP;WACA;WACA;WACA;WACA;WACA;;WAEA,mBAAmB;;;WAGnB;;UAGM;WACN;WACA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;;WAEA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;;WAEA;WACA;;;WAGA;WACA;WACA;WACA;;;WAGA;WACA,KAAK;WACL;;UAGM;WACN;WACA;IAAoB;IAAc;IAAoB;;WACtD;aACE;aACA;aACA;aACA,kBAAkB;;;aAGlB,qBAAqB;MAE1B;MAAyB;;;WAGtB;aACE;MAAqB;MAAmB;MAAqB;;aAC7D;MAAuB;MAAmB;MAAqB;;;;;aAI/D;eACE;eACA;eACA;eACA;;eAEA;eACA;eACA;eACA;;aAEF;MAAe;MAAmB;MAAqB;;;aAEvD;MACP;MACA;MACA;;;;aAIO;MACP;MACA;MACA;;;aAGO,gBAAgB;;WAElB;aACE;aACA;aACA;aACA;aACA;;;;aAIA;;WAEF;IAAqB;IAAgB;IAAgB;IAAgB;;WACrE;IAAyB;IAAgB;IAAgB;IAAe;;WACxE,gBAAgB;;iBAGX,aAAa,GAAG,gBAAgB,KAAK,cAAc,eAAe,IAAI;iBAkFhE,eAAe,SAAS,qBAAqB,QAAQ;iBAuhB3D,kBAAkB,QAAQ;;;UCjwBzB;EACf;EACA;;;;;;;iBAQoB,yBACpB,OAAO,YACP,gBAAgB,0BAChB,iBACC,QAAQ;;KA6BC;;;;;UAMK;EACf;EACA,MAAM;;EAEN;EACA;;UAGe;EACf;;EAEA,mBAAmB;;EAEnB;EACA;EACA;;;EAGA;;EAEA;EACA;;;;cAKW;;;;;iBAMG,mBACd,cACA,mCACA,qBACC;;iBAWa,sBACd,wBACA,mBAAmB,oBACnB,iBACC;UAwBc;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;;;EAGA;EACA,cAAc;;;;;;;;cAsBH,uCAAuC;mBAK/B;mBACA;EALnB,YAAY;EACZ,SAAS;EAET,YACmB,SAAS,mBACT,SAAS;EAGtB,YAAY,KAAK,uBAAuB,SAAS,eAAe;;UAyEvD;EACf;EACA;;EAEA;;EAEA;EACA;EACA;EACA;;EAEA;EACA;EACA;;EAEA;;EAEA,SAAS;EACT,cAAc;;UAGC;EACf;EACA;EACA;EACA,QAAQ;;;;;;;;UASO,sBAAsB;EACrC;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,mBAAmB;;EAEnB;EACA;EACA;EACA;EACA;EACA,MAAM;IAAqB;;EAC3B,OAAO;IAAqB;;EAC5B;IAAW;IAAgB;IAAuB;;EAClD;IAAU;IAAkB;IAAc;;;iBAUtB,aAAa,SAAS,sBAAsB,QAAQ;;;UC5TzD;WACN;WACA;WACA;WACA;WACA;WACA;aACE;aACA;aACA;;WAEF,WAAW,SAAS;;;;;;;;iBAWf,kBAAkB,SAAS;;iBAgB3B,oBAAoB,SAAS;;;;;;;;;;KAoBjC;WAEG;WACA;WACA;WACA;WACA;;WAGA;WACA,kBAAkB;WAClB,WAAW;;UAGT;WACN;WACA;;WAEA;WACA;;WAEA;WACA;WACA;;KAGC;WACG;WAA2B,UAAU;;WACrC;WAA4B;;;;;;;iBAiC3B,4BACd,SAAS,mBACT,QAAQ,sBACP;KAmFS;UAMK;WACN;WACA;WACA;;WAEA;WACA,UAAU;;WAEV;;WAEA;WACA;;WAEA;;;;;;;;UASM,0BAA0B;WAChC;WACA,wBAAwB;WACxB,QAAQ,SAAS,OAAO;;WAExB;;UAGM;WACN,QAAQ;;WAER;;WAEA;WACA;WACA;;WAEA,gBAAgB;;;WAGhB,kBAAkB;WAClB,cAAc;;;;;;;;iBAST,gBAAgB,SAAS,gBAAgB;;;;;;;iBAyGnC,eACpB,mBAAmB,qBACnB,SAAS,wBACR,QAAQ;;iBAwNK,8BAA8B,KAAK;;;;;;iBAmCnC,iBAAiB,eAAe;;;UCvoB/B;WACN;WACA;WACA;WACA;;;iBAMK,2BAA2B,kBAAkB;UAW5C;WACN;WACA;;UAGM;WACN,WAAW;WACX,SAAS;;WAET;;;;;;;;iBASK,yBACd,SAAS,sBACT,kBAAkB,eACjB;UAuCc;WACN,kBAAkB;WAClB;;WAEA,YAAY;;WAEZ;WACA;WACA;;WAEA,gBAAgB;WAChB,cAAc;;UAGR;WACN,YAAY;WACZ;WACA,SAAS;;;;;;;iBAQE,oBACpB,SAAS,sBACT,SAAS,uBACR,QAAQ"}

package/dist/trajectory-replay/index.js

@@ -4,7 +4,7 @@ import { t as TraceEmitter } from "../emitter-DeQHiDMm.js";
 import { t as runCounterfactual } from "../counterfactual-Bjq1mlUu.js";
 import { n as InMemoryTraceStore } from "../store-DNe_Uv1Q.js";
 import { t as packageVersion } from "../package-version-D7lQHt_-.js";
-import { a as assertReplayableTrajectory, c as deriveFailureSignature, d as isRecordedTimeout, f as isSubmitAction, g as unreadableExitCount, h as parseRecordedReturncode, i as TIMEOUT_OBSERVATION_MARKER, l as finalRecordedOutcome, m as parseObservationOutput, n as FORMAT_ERROR_OBSERVATION_PREFIX, o as classifyObservation, p as isSubmitOnlyAction, r as SUBMIT_ACTION_SIGNATURE, s as decodeRecordedTurns, t as wrapActionForExec, u as isElidedField } from "../exec-D9WpA2p-.js";
+import { a as assertReplayableTrajectory, c as deriveFailureSignature, d as isRecordedTimeout, f as isSubmitAction, g as unreadableExitCount, h as parseRecordedReturncode, i as TIMEOUT_OBSERVATION_MARKER, l as finalRecordedOutcome, m as parseObservationOutput, n as FORMAT_ERROR_OBSERVATION_PREFIX, o as classifyObservation, p as isSubmitOnlyAction, r as SUBMIT_ACTION_SIGNATURE, s as decodeRecordedTurns, t as wrapActionForExec, u as isElidedField } from "../exec-ChJ6d5if.js";
 import { createHash } from "node:crypto";
 import { appendFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
@@ -198,7 +198,7 @@ function enumerateReplayableCases(corpora) {
 				missingGoldId = goldId;
 				break;
 			}
-			if (isSubmitAction(step.action)) {
+			if (step.action.trim() === "submit" || isSubmitOnlyAction(step.action)) {
 				submitGoldsSkipped += 1;
 				continue;
 			}
@@ -223,12 +223,22 @@ function enumerateReplayableCases(corpora) {
 			});
 			continue;
 		}
+		const recordedReturncodeAtK = parseRecordedReturncode(target.observation);
+		if (recordedReturncodeAtK === null) {
+			excluded.push({
+				corpus: corpus.name,
+				trajId,
+				reason: "no-recorded-returncode-at-k",
+				detail: `k=${target.step_id}; arm A cannot confirm execution without a recorded returncode`
+			});
+			continue;
+		}
 		replayable.push({
 			...resources,
 			goldIncorrectSteps: gold,
 			k: target.step_id,
 			submitGoldsSkipped,
-			recordedReturncodeAtK: parseRecordedReturncode(target.observation)
+			recordedReturncodeAtK
 		});
 	}
 	return {
@@ -831,7 +841,7 @@ async function replayVerify(options) {
 	if (target.step_id !== options.at) throw new Error(`trajectory-replay: steps[${index}].step_id=${target.step_id} != at ${options.at}; steps.json must be ordered with 1-based contiguous step_ids`);
 	const caseId = options.caseId ?? options.stepsPath;
 	const recordedReturncode = parseRecordedReturncode(target.observation);
-	const signature = options.signature ?? deriveFailureSignature(target.observation);
+	const signature = options.signature === void 0 ? deriveFailureSignature(target.observation) : options.signature.trim() || null;
 	const signatureBasis = signature ? "returncode+output-substring" : "returncode-only";
 	const runnerOptions = {
 		cwd: options.cwd,
@@ -853,7 +863,9 @@ async function replayVerify(options) {
 	const armAMs = Date.now() - armAStart;
 	const armAExec = requireExec(armARunner, "arm A");
 	const armAOutput = `${armAExec.stdout}\n${armAExec.stderr}`;
-	const failureSignatureMatch = recordedReturncode !== null && armAExec.exitCode === recordedReturncode && (signature ? armAOutput.includes(signature) : true);
+	const returncodeMatch = recordedReturncode !== null && armAExec.exitCode === recordedReturncode;
+	const failureSignatureMatch = returncodeMatch && signature !== null && armAOutput.includes(signature);
+	const failureReproduced = recordedReturncode !== 0 && returncodeMatch && (signature === null || failureSignatureMatch);
 	let armBRunner = null;
 	let armBExec = null;
 	let armBRunId = null;
@@ -909,7 +921,7 @@ async function replayVerify(options) {
 			exitCode: armBExec.exitCode,
 			wallMs: armBExec.wallMs,
 			prefix: requirePrefix(armBRunner, "arm B"),
-			failureVanished: armBExec.exitCode === 0 && (signature && armBOutput ? !armBOutput.includes(signature) : true)
+			failureVanished: requirePrefix(armBRunner, "arm B").prefixWithinTolerance && armBExec.exitCode === 0 && (signature && armBOutput ? !armBOutput.includes(signature) : true)
 		} : null,
 		timings: {
 			armAMs,
@@ -926,14 +938,15 @@ async function replayVerify(options) {
 	if (armAPrefix.prefixUnknownExpectations > 0) assumptions.push(`${armAPrefix.prefixUnknownExpectations} prefix step(s) carry no recorded returncode — agreement with the recording is unestablished there`);
 	if (options.prefixLimit !== void 0 && options.prefixLimit < index) assumptions.push(`prefix truncated at ${options.prefixLimit} of ${index} recorded steps — state fidelity at k is weakened`);
 	if (signatureBasis === "returncode-only") assumptions.push("no output substring derived from the recording — reproduction rests on the returncode alone");
-	const valid = core.prefixWithinTolerance && failureSignatureMatch && (core.armB === null || core.armB.failureVanished);
+	if (core.armB !== null && !core.armB.prefix.prefixWithinTolerance) assumptions.push("arm B prefix diverged beyond tolerance — the corrected step ran from a different state");
+	const valid = core.prefixWithinTolerance && failureReproduced && (core.armB === null || core.armB.failureVanished);
 	const verdict = {
 		...core,
 		valid,
 		score: valid ? 1 : 0,
 		scores: {
 			prefixFidelity: 1 - core.prefixDivergencePct / 100,
-			armAReproduced: failureSignatureMatch ? 1 : 0,
+			armAReproduced: failureReproduced ? 1 : 0,
 			...core.armB === null ? {} : { armBFailureVanished: core.armB.failureVanished ? 1 : 0 }
 		},
 		certification: {
@@ -1037,8 +1050,9 @@ function renderReport(verdict, target, armA, armB) {
 *
 * Headline metrics:
 *   replayability rate — fraction of replayable cases where the prefix
-*     replays within the divergence tolerance AND arm A reproduces the
-*     recorded returncode at k;
+*     replays within the divergence tolerance AND arm A reproduces a
+*     recorded failure at k; a zero exit without an error signature is not
+*     evidence of failure;
 *   prefix fidelity — executed prefix steps and the share of them that did
 *     not confirm the recording, split by kind. A corpus whose recordings
 *     carry no returncodes shows up here as unknown-expectation steps, never
@@ -1097,7 +1111,7 @@ async function executeArmB(replayCase, fixCommand, backend, signature, stepTimeo
 		prefixExecuted: prefix.prefixExecuted,
 		prefixDivergences: prefix.prefixDivergences.length,
 		prefixDivergencePct: prefix.prefixDivergencePct,
-		failureVanished: exec.exitCode === 0 && (signature ? !output.includes(signature) : true),
+		failureVanished: prefix.prefixWithinTolerance && exec.exitCode === 0 && (signature ? !output.includes(signature) : true),
 		stdout: exec.stdout,
 		stderr: exec.stderr
 	};
@@ -1205,7 +1219,7 @@ async function runReplayBatch(options) {
 				armAExit: verdict.armA.exitCode,
 				armAReturncodeMatch: returncodeMatch,
 				armASignatureMatch: verdict.armA.failureSignatureMatch,
-				replayed: verdict.prefixWithinTolerance && returncodeMatch,
+				replayed: verdict.valid,
 				fix: null,
 				wallMs: Date.now() - caseStart
 			};
@@ -1557,7 +1571,7 @@ function renderBatchReport(report) {
 	lines.push("## Headline");
 	lines.push("");
 	const fidelity = headline.prefixFidelity;
-	lines.push(`- **Replayability rate: ${pct(headline.replayabilityRate.value)}** (${headline.replayabilityRate.numerator}/${headline.replayabilityRate.denominator} replayable cases where the prefix replayed within ${fidelity.tolerancePct}% divergence AND arm A reproduced the recorded returncode at the gold step k).`);
+	lines.push(`- **Replayability rate: ${pct(headline.replayabilityRate.value)}** (${headline.replayabilityRate.numerator}/${headline.replayabilityRate.denominator} replayable cases where the prefix replayed within ${fidelity.tolerancePct}% divergence AND arm A reproduced an observed failure at the gold step k).`);
 	lines.push(`- Signature-strict rate: ${pct(headline.signatureStrictRate.value)} (${headline.signatureStrictRate.numerator}/${headline.signatureStrictRate.denominator}; additionally requires the recorded error substring in arm A output).`);
 	lines.push(`- **Prefix divergence: ${fidelity.divergencePct === null ? "—" : `${fidelity.divergencePct}%`}** (${fidelity.divergentSteps}/${fidelity.executedSteps} executed prefix steps did not confirm the recording: ${fidelity.returncodeMismatches} returncode mismatches, ${fidelity.unknownExpectations} with no recorded returncode to check). ${fidelity.casesWithinTolerance}/${fidelity.casesExecuted} executed cases are within the ${fidelity.tolerancePct}% tolerance.`);
 	if (headline.fixFlipRate) lines.push(`- **Fix-flip rate: ${pct(headline.fixFlipRate.value)}** (${headline.fixFlipRate.numerator}/${headline.fixFlipRate.denominator} arm-B-executed cases where the generated fix made the failure vanish).`);
@@ -1671,7 +1685,11 @@ function resolveFindingInvocation(finding, corpora) {
 		const resolution = resolveCaseResources(corpus, finding.trajId);
 		if (resolution.resolved) {
 			const at = subject.firstStep;
-			if (!resolution.resources.steps.find((s) => s.step_id === at)) throw new Error(`trajectory-replay: finding step ${at} is outside ${finding.trajId} (${resolution.resources.steps.length} steps)`);
+			const step = resolution.resources.steps.find((s) => s.step_id === at);
+			if (!step) throw new Error(`trajectory-replay: finding step ${at} is outside ${finding.trajId} (${resolution.resources.steps.length} steps)`);
+			const recordedReturncode = parseRecordedReturncode(step.observation);
+			if (recordedReturncode === null) throw new Error(`trajectory-replay: finding step ${at} in ${finding.trajId} has no recorded returncode`);
+			if (recordedReturncode === 0) throw new Error(`trajectory-replay: finding step ${at} in ${finding.trajId} exited 0; shell replay cannot verify its semantic error`);
 			return {
 				resources: resolution.resources,
 				subject,
@@ -1876,14 +1894,14 @@ function resolveFindingReplayability(finding, source) {
 	};
 }
 /**
-* Arm A reproduced on a prefix the recording confirmed → the fix flipping it
+* Arm A reproduced an observed failure on a confirmed prefix → a fix flipping it
 * beats plain reproduction; anything else diverged. A proof standing on a
 * prefix outside the divergence tolerance is divergent no matter what arm A
 * did: the state it ran against is not the recorded state.
 */
 function classifyVerdict(verdict) {
 	if (!verdict.prefixWithinTolerance) return "divergent";
-	if (!verdict.armA.failureSignatureMatch) return "divergent";
+	if (verdict.scores?.armAReproduced !== 1) return "divergent";
 	if (verdict.armB?.failureVanished) return "fix-flipped";
 	return "reproduced";
 }
@@ -1898,6 +1916,7 @@ function receiptExecution(verdict) {
 			command: verdict.armA.command,
 			exitCode: verdict.armA.exitCode,
 			wallMs: verdict.armA.wallMs,
+			reproduced: verdict.scores?.armAReproduced === 1,
 			failureSignatureMatch: verdict.armA.failureSignatureMatch
 		},
 		armB: verdict.armB ? {