@tangle-network/traces 0.11.0 → 0.11.3

package/dist/index.js

@@ -15,6 +15,7 @@ import {
   CopilotAdapter,
   DEFAULT_HARNESS,
   EMPTY_SPAN_RUN_SNAPSHOT,
+  EXPORT_RULE,
   EmptySessionError,
   FactoryAdapter,
   ForgeAdapter,
@@ -31,6 +32,8 @@ import {
   RULE,
   SPAWN_JOURNAL_FILE,
   STATUS_MARK,
+  SUBMIT_ACTION_SIGNATURE,
+  SandboxCounterfactualRunner,
   SessionWorkflowError,
   TRACES_REDACTION_RULES,
   ZERO_ROLL,
@@ -38,9 +41,12 @@ import {
   alreadyUploaded,
   analyzeAdoption,
   analyzeLiveBatch,
+  analyzeLoopConvergence,
   analyzeReactions,
   analyzeSpans,
+  analyzeSteeringChain,
   applyRedactor,
+  buildFixPrompt,
   buildPolicyEvidenceRecord,
   buildSessionIndex,
   buildSessionIndexFromRows,
@@ -51,6 +57,8 @@ import {
   classifyLiveActions,
   classifyReaction,
   claudeActor,
+  cmdReplayVerify,
+  cmdReplayVerifyBatch,
   codexActor,
   collectPolicyEvidence,
   collectSessionIndex,
@@ -58,29 +66,41 @@ import {
   collectSessionWorkflow,
   commandAnalyzer,
   commandRedactor,
+  conformanceOfSpans,
   connectors,
   countSkillRunsJsonl,
   cwdMatchesSelection,
   defaultTraceLiveAnalysts,
+  deriveFailureSignature,
+  derivedImageTag,
   describeSessionRelationship,
+  dockerImagePreparer,
+  enumerateReplayableCases,
   equivalentGitCwds,
   executeUpload,
   exportTraceEvidenceFile,
   exportTraceEvidenceRows,
   exportTraceEvidenceText,
+  extractFixCommand,
   fileRunContextSupervisorRunReader,
   findSpanSourceFiles,
   forEachTreeNode,
+  generateFixCommand,
+  goldIncorrectSteps,
   haloAnalyzer,
   hodoscopeAnalyzer,
   importCodeTraceBench,
+  incompleteInputsNote,
   indexSessionIdsByTrace,
   inferRole,
+  inferSpanKind,
+  ingestCodeTraceBenchSteps,
   inspectSessionIndex,
   int,
   isChatTrajectory,
   isChatTrajectoryMessage,
   isFileRunContextDir,
+  isSubmitAction,
   knownHarnesses,
   listAdapters,
   loadState,
@@ -91,8 +111,14 @@ import {
   mergeTracesConfig,
   ms,
   normalizeRemote,
+  otlpRowToSpan,
   outboundHash,
+  parseCorpusFlag,
   parseIsoToEpochMs,
+  parseObservationOutput,
+  parseRecordedReturncode,
+  parseReplayBatchArgs,
+  parseReplayVerifyArgs,
   parseSession,
   parseSince,
   planTraceAgenticRoute,
@@ -101,6 +127,8 @@ import {
   readFileRunContext,
   readJsonFile,
   readJsonl,
+  readLabelEntries,
+  readOtlpInput,
   readRunContextJournal,
   readRunContextSnapshot,
   readRunWatchSnapshot,
@@ -108,11 +136,19 @@ import {
   readSpanRunSnapshot,
   recordSessionCorruption,
   redactSpans,
+  renderBatchReport,
+  renderConformance,
   renderInspectionReport,
   renderRunContextSnapshot,
   renderRunWatchSnapshot,
   renderSpanRunSnapshot,
+  renderValidation,
+  replayBatchUsage,
+  replayVerify,
+  replayVerifyUsage,
   resolveAdapter,
+  resolveCaseResources,
+  resolveOtlpInputFiles,
   resolveRepoAttrs,
   resolveRunContextDir,
   resolveRunWatchTarget,
@@ -120,13 +156,16 @@ import {
   runCommand,
   runExternalAnalyzers,
   runPipelines,
+  runReplayBatch,
   runTotals,
   runTraceImprovement,
   runTraceInvestigation,
   runTraceStoreInvestigation,
+  sandboxReplayBackend,
   saveReport,
   saveState,
   scanSessions,
+  seededSample,
   selectAdapters,
   serializeInspectionReport,
   serializePolicyEvidence,
@@ -136,6 +175,7 @@ import {
   sessionHash,
   sessionIdFromAttributes,
   sessionIntegrityAttributes,
+  skippedAnalyses,
   span,
   spanRunTotals,
   stampRepoAttrs,
@@ -145,6 +185,7 @@ import {
   streamSessions,
   summarizeOtlpExecution,
   summarizeSpanExecution,
+  summarizeSpanStructure,
   takeJsonl,
   textIsSynthetic,
   toOpenInferenceSpan,
@@ -154,13 +195,16 @@ import {
   totalAgenticFailureMessage,
   traceAgenticKinds,
   traceStreamEventsFromSpans,
+  unavailableCapabilities,
   uploadKey,
   usd,
   userPromptSpan,
   validateOtlpSpans,
+  validationExitCode,
   watchRunTarget,
   watchSessions,
   wrap,
+  wrapActionForExec,
   writeHodoscopeInput,
   writeInspectionReportFile,
   writeOtlpFile,
@@ -168,8 +212,9 @@ import {
   writeSessionIndexFile,
   writeTraceEvidenceExportFile,
   writeTraceImprovementArtifacts,
+  zaiChatCaller,
   zeroPricedCost
-} from "./chunk-JCI5YXNT.js";
+} from "./chunk-CK5F5TWO.js";
 
 // src/collect.ts
 async function collectSessions(opts = {}) {
@@ -185,6 +230,79 @@ async function collectSessions(opts = {}) {
   return out;
 }
 
+// src/replay-wire.ts
+var SUBJECT_PATTERN = /^incorrect-steps-(\d+)-(\d+)-(escaped|unescaped)-consequence-(\d+)$/;
+function parseIncorrectStepsSubject(subject) {
+  const match = SUBJECT_PATTERN.exec(subject);
+  if (!match) return null;
+  return {
+    firstStep: Number(match[1]),
+    lastStep: Number(match[2]),
+    escapeStatus: match[3],
+    consequenceStep: Number(match[4])
+  };
+}
+function resolveFindingInvocation(finding, corpora) {
+  const subject = parseIncorrectStepsSubject(finding.subject);
+  if (!subject) {
+    throw new Error(
+      `replay-wire: subject '${finding.subject}' is not incorrect-steps-<first>-<last>-<escaped|unescaped>-consequence-<step>`
+    );
+  }
+  const failures = [];
+  for (const corpus of corpora) {
+    const resolution = resolveCaseResources(corpus, finding.trajId);
+    if (resolution.resolved) {
+      const at = subject.firstStep;
+      const step = resolution.resources.steps.find((s) => s.step_id === at);
+      if (!step) {
+        throw new Error(
+          `replay-wire: finding step ${at} is outside ${finding.trajId} (${resolution.resources.steps.length} steps)`
+        );
+      }
+      return { resources: resolution.resources, subject, at };
+    }
+    failures.push(`${corpus.name}: ${resolution.reason}`);
+  }
+  throw new Error(
+    `replay-wire: trajectory ${finding.trajId} is not replayable in any corpus \u2014 ${failures.join("; ")}`
+  );
+}
+async function replayVerifyFinding(finding, options) {
+  if (options.fixCaller && options.fixCommand !== void 0) {
+    throw new Error("replay-wire: pass fixCaller or fixCommand, not both");
+  }
+  const invocation = resolveFindingInvocation(finding, options.corpora);
+  let fixCommand = options.fixCommand ?? null;
+  if (options.fixCaller) {
+    const generated = await generateFixCommand(options.fixCaller, {
+      taskStatement: invocation.resources.taskStatement,
+      steps: invocation.resources.steps,
+      k: invocation.at
+    });
+    if (!generated.succeeded) {
+      throw new Error(`replay-wire: fix generation failed \u2014 ${generated.error}`);
+    }
+    fixCommand = generated.value.command;
+  }
+  const verdict = await replayVerify({
+    stepsPath: invocation.resources.stepsPath,
+    image: invocation.resources.image,
+    at: invocation.at,
+    fixCommand: fixCommand ?? void 0,
+    cwd: invocation.resources.cwd,
+    out: options.out,
+    caseId: invocation.resources.trajId,
+    stepTimeoutMs: options.stepTimeoutMs ?? invocation.resources.recordedStepTimeoutMs ?? void 0,
+    prefixLimit: options.prefixLimit,
+    backend: options.backend,
+    apiKey: options.apiKey,
+    baseUrl: options.baseUrl,
+    onProgress: options.onProgress
+  });
+  return { invocation, fixCommand, verdict };
+}
+
 // src/index.ts
 import {
   AnalystRegistry,
@@ -212,6 +330,7 @@ export {
   DEFAULT_HARNESS,
   DEFAULT_REDACTION_RULES,
   EMPTY_SPAN_RUN_SNAPSHOT,
+  EXPORT_RULE,
   EmptySessionError,
   FactoryAdapter,
   ForgeAdapter,
@@ -228,6 +347,8 @@ export {
   RULE,
   SPAWN_JOURNAL_FILE,
   STATUS_MARK,
+  SUBMIT_ACTION_SIGNATURE,
+  SandboxCounterfactualRunner,
   SessionWorkflowError,
   TRACES_REDACTION_RULES,
   ZERO_ROLL,
@@ -235,10 +356,13 @@ export {
   alreadyUploaded,
   analyzeAdoption,
   analyzeLiveBatch,
+  analyzeLoopConvergence,
   analyzeReactions,
   analyzeSpans,
+  analyzeSteeringChain,
   applyRedactor,
   buildDefaultAnalystRegistry,
+  buildFixPrompt,
   buildPolicyEvidenceRecord,
   buildSessionIndex,
   buildSessionIndexFromRows,
@@ -249,6 +373,8 @@ export {
   classifyLiveActions,
   classifyReaction,
   claudeActor,
+  cmdReplayVerify,
+  cmdReplayVerifyBatch,
   codexActor,
   collectPolicyEvidence,
   collectSessionIndex,
@@ -257,31 +383,43 @@ export {
   collectSessions,
   commandAnalyzer,
   commandRedactor,
+  conformanceOfSpans,
   connectors,
   countSkillRunsJsonl,
   createHostedClient,
   cwdMatchesSelection,
   defaultTraceLiveAnalysts,
+  deriveFailureSignature,
+  derivedImageTag,
   describeSessionRelationship,
+  dockerImagePreparer,
+  enumerateReplayableCases,
   equivalentGitCwds,
   executeUpload,
   exportTraceEvidenceFile,
   exportTraceEvidenceRows,
   exportTraceEvidenceText,
+  extractFixCommand,
   fileRunContextSupervisorRunReader,
   findSpanSourceFiles,
   forEachTreeNode,
+  generateFixCommand,
+  goldIncorrectSteps,
   haloAnalyzer,
   hodoscopeAnalyzer,
   hostedClientFromEnv,
   importCodeTraceBench,
+  incompleteInputsNote,
   indexSessionIdsByTrace,
   inferRole,
+  inferSpanKind,
+  ingestCodeTraceBenchSteps,
   inspectSessionIndex,
   int,
   isChatTrajectory,
   isChatTrajectoryMessage,
   isFileRunContextDir,
+  isSubmitAction,
   knownHarnesses,
   listAdapters,
   loadState,
@@ -293,8 +431,15 @@ export {
   mergeTracesConfig,
   ms,
   normalizeRemote,
+  otlpRowToSpan,
   outboundHash,
+  parseCorpusFlag,
+  parseIncorrectStepsSubject,
   parseIsoToEpochMs,
+  parseObservationOutput,
+  parseRecordedReturncode,
+  parseReplayBatchArgs,
+  parseReplayVerifyArgs,
   parseSession,
   parseSince,
   planTraceAgenticRoute,
@@ -303,6 +448,8 @@ export {
   readFileRunContext,
   readJsonFile,
   readJsonl,
+  readLabelEntries,
+  readOtlpInput,
   readRunContextJournal,
   readRunContextSnapshot,
   readRunWatchSnapshot,
@@ -312,11 +459,21 @@ export {
   redactSpans,
   redactString,
   redactValue,
+  renderBatchReport,
+  renderConformance,
   renderInspectionReport,
   renderRunContextSnapshot,
   renderRunWatchSnapshot,
   renderSpanRunSnapshot,
+  renderValidation,
+  replayBatchUsage,
+  replayVerify,
+  replayVerifyFinding,
+  replayVerifyUsage,
   resolveAdapter,
+  resolveCaseResources,
+  resolveFindingInvocation,
+  resolveOtlpInputFiles,
   resolveRepoAttrs,
   resolveRunContextDir,
   resolveRunWatchTarget,
@@ -324,13 +481,16 @@ export {
   runCommand,
   runExternalAnalyzers,
   runPipelines,
+  runReplayBatch,
   runTotals,
   runTraceImprovement,
   runTraceInvestigation,
   runTraceStoreInvestigation,
+  sandboxReplayBackend,
   saveReport,
   saveState,
   scanSessions,
+  seededSample,
   selectAdapters,
   serializeInspectionReport,
   serializePolicyEvidence,
@@ -340,6 +500,7 @@ export {
   sessionHash,
   sessionIdFromAttributes,
   sessionIntegrityAttributes,
+  skippedAnalyses,
   span,
   spanRunTotals,
   stampRepoAttrs,
@@ -349,6 +510,7 @@ export {
   streamSessions,
   summarizeOtlpExecution,
   summarizeSpanExecution,
+  summarizeSpanStructure,
   takeJsonl,
   textIsSynthetic,
   toOpenInferenceSpan,
@@ -358,13 +520,16 @@ export {
   totalAgenticFailureMessage,
   traceAgenticKinds,
   traceStreamEventsFromSpans,
+  unavailableCapabilities,
   uploadKey,
   usd,
   userPromptSpan,
   validateOtlpSpans,
+  validationExitCode,
   watchRunTarget,
   watchSessions,
   wrap,
+  wrapActionForExec,
   writeHodoscopeInput,
   writeInspectionReportFile,
   writeOtlpFile,
@@ -372,5 +537,6 @@ export {
   writeSessionIndexFile,
   writeTraceEvidenceExportFile,
   writeTraceImprovementArtifacts,
+  zaiChatCaller,
   zeroPricedCost
 };

package/docs/replay-batch-20260802.md

@@ -0,0 +1,76 @@
+# Replay-verify batch — first measured replayability and fix-flip rates (2026-08-02)
+
+Provenance: labels shas in `~/bench-cache/ctb-20260801/replay-batch/label-shas.txt`; docker 29.6.1; node v24.16.0; orchestrator :4095 + SDK adapter :4097; seed 17; fix model glm-5.2 via z.ai direct (16k cap, one doubled retry on length-empty); command recorded in `run2-20260802.log` header.
+Corpora: dev-32, holdout-1, holdout-2, split3 (all spent mini-SWE splits; submit-only golds excluded per the split3 positional-degeneracy finding).
+
+## Headline (n=22 replayable of 133 label entries)
+
+| Rate | Value |
+| --- | --- |
+| Replayability (prefix ≤10% divergence AND arm A reproduces recorded returncode at the gold step) | 16/22 = 72.7% |
+| Strict signature match (returncode + output substring) | 13/22 = 59.1% |
+| Fix-flip (one-shot glm-5.2 corrected command makes the failure vanish) | 9/11 = 81.8% |
+| Fix-flip, nonzero-returncode subset | 7/9 = 77.8% |
+
+Exclusions (101 of 133, all named): 65 no-swe-raw-trajectory, 21 no-docker-image, 21 gold-only-submit-step, 4 no-gold-incorrect-step.
+LLM: 16 calls, 5 residual failures (long-reasoning cases exceeding the doubled cap), 41,878 prompt + 51,312 completion tokens (~$0.14).
+
+
+Generated 2026-08-02T00:55:37.309Z.
+
+## Headline
+
+- **Replayability rate: 72.7%** (16/22 replayable cases where the prefix replayed with ≤10% returncode divergence AND arm A reproduced the recorded returncode at the gold step k).
+- Signature-strict rate: 59.1% (13/22; additionally requires the recorded error substring in arm A output).
+- **Fix-flip rate: 81.8%** (9/11 arm-B-executed cases where the generated fix made the failure vanish).
+- Fix-flip rate on recorded-rc≠0 cases: 77.8% (7/9; real recorded failures — a gold step recorded with rc 0 flips vacuously).
+
+## Enumeration
+
+133 label entries across 4 corpora → 22 replayable (SWE-style docker image + ≥1 gold incorrect step), 22 executed.
+
+| exclusion reason | count |
+| --- | --- |
+| no-swe-raw-trajectory | 65 |
+| no-docker-image | 21 |
+| gold-only-submit-step | 21 |
+| no-gold-incorrect-step | 4 |
+
+Submit-command golds are never counterfactual targets (a gold on the submit step marks a bad submit decision, not a failed command):
+
+| corpus | cases excluded (all golds = submit) | golds skipped within replayable cases |
+| --- | --- | --- |
+| dev-32 | 1 | 0 |
+| holdout-1 | 2 | 0 |
+| split3 | 18 | 0 |
+
+## Per-case results
+
+| corpus | trajectory | k | rc@k | prefix | div | div% | armA exit | rc match | sig match | replayed | fix | armB exit | vanished | wall s |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+holdout-1 | miniswe-OpenAI__GPT-5-clap-rs__clap-3179-107b… | 33 | 0 | 32 | 4 | 12.5 | 0 | yes | yes | no | — | — | — | 10.6
+holdout-1 | miniswe-OpenAI__GPT-5-clap-rs__clap-3421-8c92… | 23 | 2 | 22 | 1 | 4.5 | 2 | yes | no | **yes** | llm-failed | — | — | 7.2
+holdout-1 | miniswe-OpenAI__GPT-5-clap-rs__clap-3975-5f1a… | 33 | 101 | 32 | 1 | 3.1 | 127 | no | no | no | — | — | — | 13.0
+holdout-1 | miniswe-OpenAI__GPT-5-clap-rs__clap-4248-85bd… | 23 | 2 | 22 | 0 | 0 | 2 | yes | no | **yes** | generated | 0 | **yes** | 35.3
+holdout-2 | miniswe-OpenAI__GPT-5-instance_ansible__ansib… | 7 | 127 | 6 | 0 | 0 | 127 | yes | yes | **yes** | generated | 0 | **yes** | 20.0
+holdout-2 | miniswe-OpenAI__GPT-5-instance_ansible__ansib… | 13 | 0 | 12 | 0 | 0 | 0 | yes | yes | **yes** | llm-failed | — | — | 13.5
+holdout-2 | miniswe-OpenAI__GPT-5-instance_element-hq__el… | 17 | 2 | 16 | 0 | 0 | 2 | yes | no | **yes** | llm-failed | — | — | 25.9
+holdout-2 | miniswe-OpenAI__GPT-5-instance_internetarchiv… | 10 | 0 | 9 | 0 | 0 | 0 | yes | yes | **yes** | generated | 0 | **yes** | 4.8
+holdout-2 | miniswe-OpenAI__GPT-5-instance_qutebrowser__q… | 15 | 127 | 14 | 0 | 0 | 127 | yes | yes | **yes** | generated | 0 | **yes** | 4.1
+holdout-2 | miniswe-OpenAI__GPT-5-instance_qutebrowser__q… | 18 | 127 | 17 | 0 | 0 | 127 | yes | yes | **yes** | generated | 0 | **yes** | 4.1
+holdout-2 | miniswe-OpenAI__GPT-5-ponylang__ponyc-1057-56… | 35 | 127 | 34 | 0 | 0 | 127 | yes | yes | **yes** | generated | 0 | **yes** | 4.4
+holdout-2 | miniswe-OpenAI__GPT-5-ponylang__ponyc-2532-98… | 12 | 1 | 11 | 0 | 0 | 1 | yes | yes | **yes** | generated | 0 | **yes** | 4.4
+holdout-2 | miniswe-OpenAI__GPT-5-sveltejs__svelte-10608-… | 23 | 127 | 22 | 0 | 0 | 127 | yes | yes | **yes** | generated | 0 | **yes** | 4.7
+split3 | miniswe-OpenAI__GPT-5-fasterxml__jackson-data… | 18 | 127 | 17 | 0 | 0 | 127 | yes | yes | **yes** | generated | 127 | no | 5.7
+split3 | miniswe-OpenAI__GPT-5-instance_ansible__ansib… | 21 | 0 | 20 | 1 | 5 | 2 | no | no | no | — | — | — | 4.8
+split3 | miniswe-OpenAI__GPT-5-instance_element-hq__el… | 21 | 0 | 20 | 0 | 0 | 2 | no | no | no | — | — | — | 6.3
+split3 | miniswe-OpenAI__GPT-5-nushell__nushell-13357-… | 28 | 127 | 27 | 0 | 0 | 127 | yes | yes | **yes** | generated | 127 | no | 5.9
+split3 | miniswe-OpenAI__GPT-5-ponylang__ponyc-2205-2a… | 27 | 127 | 26 | 0 | 0 | 0 | no | no | no | — | — | — | 5.4
+split3 | miniswe-OpenAI__GPT-5-ponylang__ponyc-3973-60… | 20 | 0 | 19 | 0 | 0 | 127 | no | no | no | — | — | — | 5.8
+split3 | miniswe-OpenAI__GPT-5-simdjson__simdjson-2016… | 24 | 0 | 23 | 0 | 0 | 0 | yes | yes | **yes** | llm-failed | — | — | 5.2
+split3 | miniswe-OpenAI__GPT-5-sveltejs__svelte-11913-… | 19 | 0 | 18 | 0 | 0 | 0 | yes | yes | **yes** | generated | 0 | **yes** | 41.2
+split3 | miniswe-OpenAI__GPT-5-sveltejs__svelte-9962-5… | 21 | 127 | 20 | 0 | 0 | 127 | yes | yes | **yes** | llm-failed | — | — | 4.3
+
+## LLM fix generation
+
+Model glm-5.2: 16 calls (5 failed), 41878 prompt + 51312 completion tokens.

package/docs/replay-verify.md

@@ -0,0 +1,105 @@
+# replay-verify — executed proofs for trajectory findings
+
+An analyst finding today is a cited claim: "step 37 is where the run went wrong."
+`traces replay-verify` upgrades that claim to an executed proof: it replays the trajectory prefix inside a real sandbox on the trajectory's own docker image, then runs two counterfactual arms at the error-critical step k:
+
+- **Arm A** re-executes the recorded step k and checks the recorded failure reproduces — same returncode plus a stable output substring (the failure signature).
+- **Arm B** executes a corrected step k and checks the failure vanishes (exit 0 and the signature absent).
+
+Each arm gets a fresh sandbox with its own prefix replay, so arm B is never contaminated by arm A's side effects.
+Under the hood each arm is an agent-eval `runCounterfactual` meta-run (`layer='meta'`, `parentRunId` = the ingested trajectory run), with the sandbox-backed `CounterfactualRunner` from `src/replay-verify.ts` supplying the `executeFrom` callback that scaffold leaves to consumers.
+
+## Usage
+
+```sh
+traces replay-verify \
+  --steps normalized/<traj_id>/steps.json \
+  --image <docker_config.base_image> \
+  --at 37 \
+  --cwd /home \
+  --fix-command "sed -i '145d' /home/zstd/programs/zstdcli.c && make -C /home/zstd zstd -j2 V=1" \
+  --signature 'undeclared (first use in this function)' \
+  --out ./replay-out \
+  --base-url http://127.0.0.1:4097 --api-key-env SANDBOX_API_KEY
+```
+
+Outputs `replay-verdict.json` (machine-readable: prefix divergences, both arms, timings, run ids) and `report.md` (human-readable with real stdout/stderr excerpts).
+`traces replay-verify --help` lists every flag.
+
+## Execution semantics
+
+Every step runs the way mini-SWE ran it: a fresh `/bin/sh` subshell from a fixed workdir.
+The action is base64-piped into `sh` after `cd`-ing to `--cwd`, so arbitrary quoting in recorded actions survives, and the reported exit code is the action's own.
+
+Prefix divergences (recorded returncode ≠ replayed exit) are recorded per step, never abort the replay, and are surfaced in the verdict.
+A high divergence rate is a finding about replayability, not a failure of the tool.
+
+## Honest limits
+
+- **Only SWE-style trajectories are replayable.** The docker image comes from the raw trajectory's `info.docker_config.base_image`; terminal-bench-style tasks that need external compose peers (databases, services) cannot be replayed with a single image.
+- **Commands never run as root.** The sandbox platform pins every customer command to the configured non-root subprocess identity (uid 1000 by default) — a deliberate security invariant with no override. Trajectories recorded as root (mini-SWE's default) will hit permission errors on root-owned trees. Fix by deriving a replay image that chowns the working tree to the sandbox uid:
+
+  ```dockerfile
+  FROM mswebench/facebook_m_zstd:pr-1733
+  RUN chown -R 1000:1000 /home
+  ```
+
+  This preserves the recorded failure semantics for build/edit workflows (they depend on writability, not on uid 0). The verdict records the image actually used.
+- **Copy-on-write forks are firecracker-only.** `box.branch()` could replay the prefix once and fork at k; on the docker driver each arm replays the prefix in its own fresh sandbox instead. Wall-time cost is one extra prefix replay per arm.
+- **Environment drift shows up as divergence.** The sandbox mounts a workspace at `/home/agent` and injects platform env; recorded observations that depend on exact `ls` output or env contents can differ while returncodes still match. Only returncodes drive the divergence count.
+- **The failure signature is derived, not assumed.** Default: the first recorded-output line containing "error". Compiler quote glyphs vary with locale, so prefer an explicit `--signature` with an ASCII-stable substring.
+
+## Batch mode — replayability and fix-flip rates
+
+`traces replay-verify-batch` runs the proof at corpus scale over gold-labeled CodeTraceBench corpora and measures two headline numbers nobody had before:
+
+- **Replayability rate** — fraction of replayable cases where the prefix replays with ≤10% returncode divergence AND arm A reproduces the recorded returncode at the gold step k (k = the first gold incorrect step from `incorrect_stages[].incorrect_step_ids` that is a real mid-trajectory action; see the submit-step rule below).
+- **Fix-flip rate** — fraction of arm-B-executed cases where a generated corrected command made the failure vanish.
+
+```sh
+traces replay-verify-batch \
+  --corpus dev-32=<labels.json>::<prepared-dir> \
+  --corpus holdout-1=<labels.json>::<prepared-dir> \
+  --out ./replay-batch-out \
+  --fix generate --max-fix-cases 30 --seed 17 \
+  --base-url http://127.0.0.1:4097 --api-key-env SANDBOX_API_KEY \
+  --fix-api-key-env ZAI_GLM_API_KEY
+```
+
+A corpus is a gold label file plus a prepared directory (`normalized/<traj_id>/steps.json` + `extracted/<traj_id>/swe_raw/**`). Replayable = the raw trajectory carries `info.docker_config.base_image` AND the labels mark ≥1 gold incorrect step that is a real mid-trajectory action. The batch:
+
+- reports the true replayable count and a per-reason exclusion table (`--enumerate-only` prints both without touching a sandbox);
+- **skips submit-command golds when choosing k.** A gold step whose action is the agent's end-of-run submit (`COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`) marks a bad submit *decision*, not a failed command — there is no executable failure to reproduce, so it is never a counterfactual target. k advances to the first non-submit gold; a case whose golds are all submit steps is excluded as `gold-only-submit-step`. The report carries a per-corpus table of both counts (this matters most for split3, whose gold labels concentrate on the submit decision);
+- derives the working directory from the recorded run config (`info.config.environment.cwd`), falling back to `docker_config.cwd`, then to the first `pwd` observation;
+- builds and caches a uid-1000 derived image per (base image, cwd) pair — `ctb-replay:<hash>-uid1000` — because the sandbox platform pins commands to a non-root identity;
+- runs every case serially (image pulls contend on disk and registry bandwidth) and records pull/build failures as report rows, never silent skips;
+- uses the recorded per-step timeout (`info.config.environment.timeout`) unless `--step-timeout` overrides it.
+
+Fix generation (`--fix generate`) runs ONE chat completion per arm-A-reproduced case against an OpenAI-compatible endpoint (default: glm-5.2 on z.ai). The prompt carries the gold step's action and observation, ±3 surrounding steps, and the task statement; the reply must be a single corrected shell command. Cases beyond `--max-fix-cases` are excluded by a seeded random sample and marked `sampled-out` in the report. Arm B replays the prefix in its own fresh sandbox and executes the corrected command; `failureVanished` = exit 0 with the failure signature absent.
+
+Honest-reporting notes baked into the report:
+
+- A gold step whose recorded observation carries **no returncode** can never satisfy "arm A reproduces the recorded returncode": it counts against the replayability rate and the per-case table says why.
+- A gold step recorded with **returncode 0** (the labeled mistake succeeded — a wrong-direction action, not a crash) reproduces trivially; for such cases `failureVanished` is vacuous, so read the headline's separate fix-flip rate on the recorded-rc≠0 subset.
+
+Outputs: `batch-report.json`, `batch-report.md` (headline + exclusion + pull-failure + per-case tables), `cases.jsonl` (incremental, crash-safe), and one directory per case with the full `replay-verify` artifacts plus `armB-result.json`.
+
+## Analyst wire — finding in, executed proof out
+
+`replayVerifyFinding` (exported from the package root, `src/replay-wire.ts`) is the entry point the analyst product calls:
+
+```ts
+import { replayVerifyFinding } from '@tangle-network/traces'
+
+const { invocation, fixCommand, verdict } = await replayVerifyFinding(
+  { trajId: 'miniswe-…-zstd-1733-786102c1', subject: 'incorrect-steps-37-37-unescaped-consequence-39' },
+  { corpora, out: './proof', fixCaller, baseUrl, apiKey },
+)
+```
+
+The subject grammar is the analyst benchmark's `incorrect-steps-<first>-<last>-<escaped|unescaped>-consequence-<step>`; `--at` is the finding's **first** incorrect step (the finding's claim, which may differ from the gold label). The wire resolves the trajectory across the given corpora, generates the arm-B fix through the same one-call generator (or accepts a pre-supplied `fixCommand`), and returns the full `ReplayVerdict`. It throws with a precise reason when the finding cannot be replayed (malformed subject, unknown trajectory, non-SWE case, step out of range) — the product surfaces that reason instead of a proof.
+
+## Orchestrator prerequisites
+
+replay-verify talks to a sandbox API (`--base-url`); in local development that is the sandbox SDK adapter in front of an orchestrator running the docker driver.
+Arbitrary docker images need the platform control runtime (node injected from a mounted Nix profile) because sidecar-less images rarely ship node: the orchestrator needs `NIX_PROFILE_PATH` set, and this CLI passes the `SIDECAR_PLATFORM_CONTROL_RUNTIME` container-env marker on create (the production sandbox API derives the same marker from `agent: false`).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/traces",
-  "version": "0.11.0",
+  "version": "0.11.3",
   "description": "Point it at your coding-agent session traces (Claude Code, Codex, OpenCode, Gemini, Pi, …) and get failure-mode + efficiency findings. CLI + SDK over the @tangle-network/agent-eval analyst suite — observe live sessions, run your own analysts, redact, and upload to the Tangle Intelligence Platform.",
   "type": "module",
   "license": "MIT",
@@ -47,9 +47,10 @@
     "node": ">=22"
   },
   "dependencies": {
-    "@tangle-network/agent-eval": "0.140.0",
-    "@tangle-network/agent-runtime": "0.116.0",
-    "@tangle-network/sandbox": "0.15.2"
+    "@tangle-network/agent-eval": "0.140.1",
+    "@tangle-network/agent-runtime": "0.121.0",
+    "@tangle-network/agent-trace-contract": "^1.0.2",
+    "@tangle-network/sandbox": "0.16.0"
   },
   "devDependencies": {
     "@types/node": "^22.0.0",
@@ -64,6 +65,7 @@
     "dev": "tsx src/cli.ts",
     "build": "tsup src/index.ts src/cli.ts --format esm --target node22 --clean --dts",
     "check:package": "node scripts/check-package-bin.mjs",
+    "check:source": "node scripts/check-source-text.mjs",
     "test": "vitest run",
     "test:watch": "vitest",
     "typecheck": "tsc --noEmit"