@tangle-network/traces 0.11.2 → 0.11.3

package/dist/index.js

@@ -32,6 +32,7 @@ import {
   RULE,
   SPAWN_JOURNAL_FILE,
   STATUS_MARK,
+  SUBMIT_ACTION_SIGNATURE,
   SandboxCounterfactualRunner,
   SessionWorkflowError,
   TRACES_REDACTION_RULES,
@@ -45,6 +46,7 @@ import {
   analyzeSpans,
   analyzeSteeringChain,
   applyRedactor,
+  buildFixPrompt,
   buildPolicyEvidenceRecord,
   buildSessionIndex,
   buildSessionIndexFromRows,
@@ -56,6 +58,7 @@ import {
   classifyReaction,
   claudeActor,
   cmdReplayVerify,
+  cmdReplayVerifyBatch,
   codexActor,
   collectPolicyEvidence,
   collectSessionIndex,
@@ -69,15 +72,21 @@ import {
   cwdMatchesSelection,
   defaultTraceLiveAnalysts,
   deriveFailureSignature,
+  derivedImageTag,
   describeSessionRelationship,
+  dockerImagePreparer,
+  enumerateReplayableCases,
   equivalentGitCwds,
   executeUpload,
   exportTraceEvidenceFile,
   exportTraceEvidenceRows,
   exportTraceEvidenceText,
+  extractFixCommand,
   fileRunContextSupervisorRunReader,
   findSpanSourceFiles,
   forEachTreeNode,
+  generateFixCommand,
+  goldIncorrectSteps,
   haloAnalyzer,
   hodoscopeAnalyzer,
   importCodeTraceBench,
@@ -91,6 +100,7 @@ import {
   isChatTrajectory,
   isChatTrajectoryMessage,
   isFileRunContextDir,
+  isSubmitAction,
   knownHarnesses,
   listAdapters,
   loadState,
@@ -103,9 +113,11 @@ import {
   normalizeRemote,
   otlpRowToSpan,
   outboundHash,
+  parseCorpusFlag,
   parseIsoToEpochMs,
   parseObservationOutput,
   parseRecordedReturncode,
+  parseReplayBatchArgs,
   parseReplayVerifyArgs,
   parseSession,
   parseSince,
@@ -115,6 +127,7 @@ import {
   readFileRunContext,
   readJsonFile,
   readJsonl,
+  readLabelEntries,
   readOtlpInput,
   readRunContextJournal,
   readRunContextSnapshot,
@@ -123,15 +136,18 @@ import {
   readSpanRunSnapshot,
   recordSessionCorruption,
   redactSpans,
+  renderBatchReport,
   renderConformance,
   renderInspectionReport,
   renderRunContextSnapshot,
   renderRunWatchSnapshot,
   renderSpanRunSnapshot,
   renderValidation,
+  replayBatchUsage,
   replayVerify,
   replayVerifyUsage,
   resolveAdapter,
+  resolveCaseResources,
   resolveOtlpInputFiles,
   resolveRepoAttrs,
   resolveRunContextDir,
@@ -140,6 +156,7 @@ import {
   runCommand,
   runExternalAnalyzers,
   runPipelines,
+  runReplayBatch,
   runTotals,
   runTraceImprovement,
   runTraceInvestigation,
@@ -148,6 +165,7 @@ import {
   saveReport,
   saveState,
   scanSessions,
+  seededSample,
   selectAdapters,
   serializeInspectionReport,
   serializePolicyEvidence,
@@ -194,8 +212,9 @@ import {
   writeSessionIndexFile,
   writeTraceEvidenceExportFile,
   writeTraceImprovementArtifacts,
+  zaiChatCaller,
   zeroPricedCost
-} from "./chunk-UY3M7XUF.js";
+} from "./chunk-CK5F5TWO.js";
 
 // src/collect.ts
 async function collectSessions(opts = {}) {
@@ -211,6 +230,79 @@ async function collectSessions(opts = {}) {
   return out;
 }
 
+// src/replay-wire.ts
+var SUBJECT_PATTERN = /^incorrect-steps-(\d+)-(\d+)-(escaped|unescaped)-consequence-(\d+)$/;
+function parseIncorrectStepsSubject(subject) {
+  const match = SUBJECT_PATTERN.exec(subject);
+  if (!match) return null;
+  return {
+    firstStep: Number(match[1]),
+    lastStep: Number(match[2]),
+    escapeStatus: match[3],
+    consequenceStep: Number(match[4])
+  };
+}
+function resolveFindingInvocation(finding, corpora) {
+  const subject = parseIncorrectStepsSubject(finding.subject);
+  if (!subject) {
+    throw new Error(
+      `replay-wire: subject '${finding.subject}' is not incorrect-steps-<first>-<last>-<escaped|unescaped>-consequence-<step>`
+    );
+  }
+  const failures = [];
+  for (const corpus of corpora) {
+    const resolution = resolveCaseResources(corpus, finding.trajId);
+    if (resolution.resolved) {
+      const at = subject.firstStep;
+      const step = resolution.resources.steps.find((s) => s.step_id === at);
+      if (!step) {
+        throw new Error(
+          `replay-wire: finding step ${at} is outside ${finding.trajId} (${resolution.resources.steps.length} steps)`
+        );
+      }
+      return { resources: resolution.resources, subject, at };
+    }
+    failures.push(`${corpus.name}: ${resolution.reason}`);
+  }
+  throw new Error(
+    `replay-wire: trajectory ${finding.trajId} is not replayable in any corpus \u2014 ${failures.join("; ")}`
+  );
+}
+async function replayVerifyFinding(finding, options) {
+  if (options.fixCaller && options.fixCommand !== void 0) {
+    throw new Error("replay-wire: pass fixCaller or fixCommand, not both");
+  }
+  const invocation = resolveFindingInvocation(finding, options.corpora);
+  let fixCommand = options.fixCommand ?? null;
+  if (options.fixCaller) {
+    const generated = await generateFixCommand(options.fixCaller, {
+      taskStatement: invocation.resources.taskStatement,
+      steps: invocation.resources.steps,
+      k: invocation.at
+    });
+    if (!generated.succeeded) {
+      throw new Error(`replay-wire: fix generation failed \u2014 ${generated.error}`);
+    }
+    fixCommand = generated.value.command;
+  }
+  const verdict = await replayVerify({
+    stepsPath: invocation.resources.stepsPath,
+    image: invocation.resources.image,
+    at: invocation.at,
+    fixCommand: fixCommand ?? void 0,
+    cwd: invocation.resources.cwd,
+    out: options.out,
+    caseId: invocation.resources.trajId,
+    stepTimeoutMs: options.stepTimeoutMs ?? invocation.resources.recordedStepTimeoutMs ?? void 0,
+    prefixLimit: options.prefixLimit,
+    backend: options.backend,
+    apiKey: options.apiKey,
+    baseUrl: options.baseUrl,
+    onProgress: options.onProgress
+  });
+  return { invocation, fixCommand, verdict };
+}
+
 // src/index.ts
 import {
   AnalystRegistry,
@@ -255,6 +347,7 @@ export {
   RULE,
   SPAWN_JOURNAL_FILE,
   STATUS_MARK,
+  SUBMIT_ACTION_SIGNATURE,
   SandboxCounterfactualRunner,
   SessionWorkflowError,
   TRACES_REDACTION_RULES,
@@ -269,6 +362,7 @@ export {
   analyzeSteeringChain,
   applyRedactor,
   buildDefaultAnalystRegistry,
+  buildFixPrompt,
   buildPolicyEvidenceRecord,
   buildSessionIndex,
   buildSessionIndexFromRows,
@@ -280,6 +374,7 @@ export {
   classifyReaction,
   claudeActor,
   cmdReplayVerify,
+  cmdReplayVerifyBatch,
   codexActor,
   collectPolicyEvidence,
   collectSessionIndex,
@@ -295,15 +390,21 @@ export {
   cwdMatchesSelection,
   defaultTraceLiveAnalysts,
   deriveFailureSignature,
+  derivedImageTag,
   describeSessionRelationship,
+  dockerImagePreparer,
+  enumerateReplayableCases,
   equivalentGitCwds,
   executeUpload,
   exportTraceEvidenceFile,
   exportTraceEvidenceRows,
   exportTraceEvidenceText,
+  extractFixCommand,
   fileRunContextSupervisorRunReader,
   findSpanSourceFiles,
   forEachTreeNode,
+  generateFixCommand,
+  goldIncorrectSteps,
   haloAnalyzer,
   hodoscopeAnalyzer,
   hostedClientFromEnv,
@@ -318,6 +419,7 @@ export {
   isChatTrajectory,
   isChatTrajectoryMessage,
   isFileRunContextDir,
+  isSubmitAction,
   knownHarnesses,
   listAdapters,
   loadState,
@@ -331,9 +433,12 @@ export {
   normalizeRemote,
   otlpRowToSpan,
   outboundHash,
+  parseCorpusFlag,
+  parseIncorrectStepsSubject,
   parseIsoToEpochMs,
   parseObservationOutput,
   parseRecordedReturncode,
+  parseReplayBatchArgs,
   parseReplayVerifyArgs,
   parseSession,
   parseSince,
@@ -343,6 +448,7 @@ export {
   readFileRunContext,
   readJsonFile,
   readJsonl,
+  readLabelEntries,
   readOtlpInput,
   readRunContextJournal,
   readRunContextSnapshot,
@@ -353,15 +459,20 @@ export {
   redactSpans,
   redactString,
   redactValue,
+  renderBatchReport,
   renderConformance,
   renderInspectionReport,
   renderRunContextSnapshot,
   renderRunWatchSnapshot,
   renderSpanRunSnapshot,
   renderValidation,
+  replayBatchUsage,
   replayVerify,
+  replayVerifyFinding,
   replayVerifyUsage,
   resolveAdapter,
+  resolveCaseResources,
+  resolveFindingInvocation,
   resolveOtlpInputFiles,
   resolveRepoAttrs,
   resolveRunContextDir,
@@ -370,6 +481,7 @@ export {
   runCommand,
   runExternalAnalyzers,
   runPipelines,
+  runReplayBatch,
   runTotals,
   runTraceImprovement,
   runTraceInvestigation,
@@ -378,6 +490,7 @@ export {
   saveReport,
   saveState,
   scanSessions,
+  seededSample,
   selectAdapters,
   serializeInspectionReport,
   serializePolicyEvidence,
@@ -424,5 +537,6 @@ export {
   writeSessionIndexFile,
   writeTraceEvidenceExportFile,
   writeTraceImprovementArtifacts,
+  zaiChatCaller,
   zeroPricedCost
 };

package/docs/replay-batch-20260802.md

@@ -0,0 +1,76 @@
+# Replay-verify batch — first measured replayability and fix-flip rates (2026-08-02)
+
+Provenance: labels shas in `~/bench-cache/ctb-20260801/replay-batch/label-shas.txt`; docker 29.6.1; node v24.16.0; orchestrator :4095 + SDK adapter :4097; seed 17; fix model glm-5.2 via z.ai direct (16k cap, one doubled retry on length-empty); command recorded in `run2-20260802.log` header.
+Corpora: dev-32, holdout-1, holdout-2, split3 (all spent mini-SWE splits; submit-only golds excluded per the split3 positional-degeneracy finding).
+
+## Headline (n=22 replayable of 133 label entries)
+
+| Rate | Value |
+| --- | --- |
+| Replayability (prefix ≤10% divergence AND arm A reproduces recorded returncode at the gold step) | 16/22 = 72.7% |
+| Strict signature match (returncode + output substring) | 13/22 = 59.1% |
+| Fix-flip (one-shot glm-5.2 corrected command makes the failure vanish) | 9/11 = 81.8% |
+| Fix-flip, nonzero-returncode subset | 7/9 = 77.8% |
+
+Exclusions (101 of 133, all named): 65 no-swe-raw-trajectory, 21 no-docker-image, 21 gold-only-submit-step, 4 no-gold-incorrect-step.
+LLM: 16 calls, 5 residual failures (long-reasoning cases exceeding the doubled cap), 41,878 prompt + 51,312 completion tokens (~$0.14).
+
+
+Generated 2026-08-02T00:55:37.309Z.
+
+## Headline
+
+- **Replayability rate: 72.7%** (16/22 replayable cases where the prefix replayed with ≤10% returncode divergence AND arm A reproduced the recorded returncode at the gold step k).
+- Signature-strict rate: 59.1% (13/22; additionally requires the recorded error substring in arm A output).
+- **Fix-flip rate: 81.8%** (9/11 arm-B-executed cases where the generated fix made the failure vanish).
+- Fix-flip rate on recorded-rc≠0 cases: 77.8% (7/9; real recorded failures — a gold step recorded with rc 0 flips vacuously).
+
+## Enumeration
+
+133 label entries across 4 corpora → 22 replayable (SWE-style docker image + ≥1 gold incorrect step), 22 executed.
+
+| exclusion reason | count |
+| --- | --- |
+| no-swe-raw-trajectory | 65 |
+| no-docker-image | 21 |
+| gold-only-submit-step | 21 |
+| no-gold-incorrect-step | 4 |
+
+Submit-command golds are never counterfactual targets (a gold on the submit step marks a bad submit decision, not a failed command):
+
+| corpus | cases excluded (all golds = submit) | golds skipped within replayable cases |
+| --- | --- | --- |
+| dev-32 | 1 | 0 |
+| holdout-1 | 2 | 0 |
+| split3 | 18 | 0 |
+
+## Per-case results
+
+| corpus | trajectory | k | rc@k | prefix | div | div% | armA exit | rc match | sig match | replayed | fix | armB exit | vanished | wall s |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+holdout-1 | miniswe-OpenAI__GPT-5-clap-rs__clap-3179-107b… | 33 | 0 | 32 | 4 | 12.5 | 0 | yes | yes | no | — | — | — | 10.6
+holdout-1 | miniswe-OpenAI__GPT-5-clap-rs__clap-3421-8c92… | 23 | 2 | 22 | 1 | 4.5 | 2 | yes | no | **yes** | llm-failed | — | — | 7.2
+holdout-1 | miniswe-OpenAI__GPT-5-clap-rs__clap-3975-5f1a… | 33 | 101 | 32 | 1 | 3.1 | 127 | no | no | no | — | — | — | 13.0
+holdout-1 | miniswe-OpenAI__GPT-5-clap-rs__clap-4248-85bd… | 23 | 2 | 22 | 0 | 0 | 2 | yes | no | **yes** | generated | 0 | **yes** | 35.3
+holdout-2 | miniswe-OpenAI__GPT-5-instance_ansible__ansib… | 7 | 127 | 6 | 0 | 0 | 127 | yes | yes | **yes** | generated | 0 | **yes** | 20.0
+holdout-2 | miniswe-OpenAI__GPT-5-instance_ansible__ansib… | 13 | 0 | 12 | 0 | 0 | 0 | yes | yes | **yes** | llm-failed | — | — | 13.5
+holdout-2 | miniswe-OpenAI__GPT-5-instance_element-hq__el… | 17 | 2 | 16 | 0 | 0 | 2 | yes | no | **yes** | llm-failed | — | — | 25.9
+holdout-2 | miniswe-OpenAI__GPT-5-instance_internetarchiv… | 10 | 0 | 9 | 0 | 0 | 0 | yes | yes | **yes** | generated | 0 | **yes** | 4.8
+holdout-2 | miniswe-OpenAI__GPT-5-instance_qutebrowser__q… | 15 | 127 | 14 | 0 | 0 | 127 | yes | yes | **yes** | generated | 0 | **yes** | 4.1
+holdout-2 | miniswe-OpenAI__GPT-5-instance_qutebrowser__q… | 18 | 127 | 17 | 0 | 0 | 127 | yes | yes | **yes** | generated | 0 | **yes** | 4.1
+holdout-2 | miniswe-OpenAI__GPT-5-ponylang__ponyc-1057-56… | 35 | 127 | 34 | 0 | 0 | 127 | yes | yes | **yes** | generated | 0 | **yes** | 4.4
+holdout-2 | miniswe-OpenAI__GPT-5-ponylang__ponyc-2532-98… | 12 | 1 | 11 | 0 | 0 | 1 | yes | yes | **yes** | generated | 0 | **yes** | 4.4
+holdout-2 | miniswe-OpenAI__GPT-5-sveltejs__svelte-10608-… | 23 | 127 | 22 | 0 | 0 | 127 | yes | yes | **yes** | generated | 0 | **yes** | 4.7
+split3 | miniswe-OpenAI__GPT-5-fasterxml__jackson-data… | 18 | 127 | 17 | 0 | 0 | 127 | yes | yes | **yes** | generated | 127 | no | 5.7
+split3 | miniswe-OpenAI__GPT-5-instance_ansible__ansib… | 21 | 0 | 20 | 1 | 5 | 2 | no | no | no | — | — | — | 4.8
+split3 | miniswe-OpenAI__GPT-5-instance_element-hq__el… | 21 | 0 | 20 | 0 | 0 | 2 | no | no | no | — | — | — | 6.3
+split3 | miniswe-OpenAI__GPT-5-nushell__nushell-13357-… | 28 | 127 | 27 | 0 | 0 | 127 | yes | yes | **yes** | generated | 127 | no | 5.9
+split3 | miniswe-OpenAI__GPT-5-ponylang__ponyc-2205-2a… | 27 | 127 | 26 | 0 | 0 | 0 | no | no | no | — | — | — | 5.4
+split3 | miniswe-OpenAI__GPT-5-ponylang__ponyc-3973-60… | 20 | 0 | 19 | 0 | 0 | 127 | no | no | no | — | — | — | 5.8
+split3 | miniswe-OpenAI__GPT-5-simdjson__simdjson-2016… | 24 | 0 | 23 | 0 | 0 | 0 | yes | yes | **yes** | llm-failed | — | — | 5.2
+split3 | miniswe-OpenAI__GPT-5-sveltejs__svelte-11913-… | 19 | 0 | 18 | 0 | 0 | 0 | yes | yes | **yes** | generated | 0 | **yes** | 41.2
+split3 | miniswe-OpenAI__GPT-5-sveltejs__svelte-9962-5… | 21 | 127 | 20 | 0 | 0 | 127 | yes | yes | **yes** | llm-failed | — | — | 4.3
+
+## LLM fix generation
+
+Model glm-5.2: 16 calls (5 failed), 41878 prompt + 51312 completion tokens.

package/docs/replay-verify.md

@@ -49,6 +49,56 @@ A high divergence rate is a finding about replayability, not a failure of the to
 - **Environment drift shows up as divergence.** The sandbox mounts a workspace at `/home/agent` and injects platform env; recorded observations that depend on exact `ls` output or env contents can differ while returncodes still match. Only returncodes drive the divergence count.
 - **The failure signature is derived, not assumed.** Default: the first recorded-output line containing "error". Compiler quote glyphs vary with locale, so prefer an explicit `--signature` with an ASCII-stable substring.
 
+## Batch mode — replayability and fix-flip rates
+
+`traces replay-verify-batch` runs the proof at corpus scale over gold-labeled CodeTraceBench corpora and measures two headline numbers nobody had before:
+
+- **Replayability rate** — fraction of replayable cases where the prefix replays with ≤10% returncode divergence AND arm A reproduces the recorded returncode at the gold step k (k = the first gold incorrect step from `incorrect_stages[].incorrect_step_ids` that is a real mid-trajectory action; see the submit-step rule below).
+- **Fix-flip rate** — fraction of arm-B-executed cases where a generated corrected command made the failure vanish.
+
+```sh
+traces replay-verify-batch \
+  --corpus dev-32=<labels.json>::<prepared-dir> \
+  --corpus holdout-1=<labels.json>::<prepared-dir> \
+  --out ./replay-batch-out \
+  --fix generate --max-fix-cases 30 --seed 17 \
+  --base-url http://127.0.0.1:4097 --api-key-env SANDBOX_API_KEY \
+  --fix-api-key-env ZAI_GLM_API_KEY
+```
+
+A corpus is a gold label file plus a prepared directory (`normalized/<traj_id>/steps.json` + `extracted/<traj_id>/swe_raw/**`). Replayable = the raw trajectory carries `info.docker_config.base_image` AND the labels mark ≥1 gold incorrect step that is a real mid-trajectory action. The batch:
+
+- reports the true replayable count and a per-reason exclusion table (`--enumerate-only` prints both without touching a sandbox);
+- **skips submit-command golds when choosing k.** A gold step whose action is the agent's end-of-run submit (`COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`) marks a bad submit *decision*, not a failed command — there is no executable failure to reproduce, so it is never a counterfactual target. k advances to the first non-submit gold; a case whose golds are all submit steps is excluded as `gold-only-submit-step`. The report carries a per-corpus table of both counts (this matters most for split3, whose gold labels concentrate on the submit decision);
+- derives the working directory from the recorded run config (`info.config.environment.cwd`), falling back to `docker_config.cwd`, then to the first `pwd` observation;
+- builds and caches a uid-1000 derived image per (base image, cwd) pair — `ctb-replay:<hash>-uid1000` — because the sandbox platform pins commands to a non-root identity;
+- runs every case serially (image pulls contend on disk and registry bandwidth) and records pull/build failures as report rows, never silent skips;
+- uses the recorded per-step timeout (`info.config.environment.timeout`) unless `--step-timeout` overrides it.
+
+Fix generation (`--fix generate`) runs ONE chat completion per arm-A-reproduced case against an OpenAI-compatible endpoint (default: glm-5.2 on z.ai). The prompt carries the gold step's action and observation, ±3 surrounding steps, and the task statement; the reply must be a single corrected shell command. Cases beyond `--max-fix-cases` are excluded by a seeded random sample and marked `sampled-out` in the report. Arm B replays the prefix in its own fresh sandbox and executes the corrected command; `failureVanished` = exit 0 with the failure signature absent.
+
+Honest-reporting notes baked into the report:
+
+- A gold step whose recorded observation carries **no returncode** can never satisfy "arm A reproduces the recorded returncode": it counts against the replayability rate and the per-case table says why.
+- A gold step recorded with **returncode 0** (the labeled mistake succeeded — a wrong-direction action, not a crash) reproduces trivially; for such cases `failureVanished` is vacuous, so read the headline's separate fix-flip rate on the recorded-rc≠0 subset.
+
+Outputs: `batch-report.json`, `batch-report.md` (headline + exclusion + pull-failure + per-case tables), `cases.jsonl` (incremental, crash-safe), and one directory per case with the full `replay-verify` artifacts plus `armB-result.json`.
+
+## Analyst wire — finding in, executed proof out
+
+`replayVerifyFinding` (exported from the package root, `src/replay-wire.ts`) is the entry point the analyst product calls:
+
+```ts
+import { replayVerifyFinding } from '@tangle-network/traces'
+
+const { invocation, fixCommand, verdict } = await replayVerifyFinding(
+  { trajId: 'miniswe-…-zstd-1733-786102c1', subject: 'incorrect-steps-37-37-unescaped-consequence-39' },
+  { corpora, out: './proof', fixCaller, baseUrl, apiKey },
+)
+```
+
+The subject grammar is the analyst benchmark's `incorrect-steps-<first>-<last>-<escaped|unescaped>-consequence-<step>`; `--at` is the finding's **first** incorrect step (the finding's claim, which may differ from the gold label). The wire resolves the trajectory across the given corpora, generates the arm-B fix through the same one-call generator (or accepts a pre-supplied `fixCommand`), and returns the full `ReplayVerdict`. It throws with a precise reason when the finding cannot be replayed (malformed subject, unknown trajectory, non-SWE case, step out of range) — the product surfaces that reason instead of a proof.
+
 ## Orchestrator prerequisites
 
 replay-verify talks to a sandbox API (`--base-url`); in local development that is the sandbox SDK adapter in front of an orchestrator running the docker driver.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/traces",
-  "version": "0.11.2",
+  "version": "0.11.3",
   "description": "Point it at your coding-agent session traces (Claude Code, Codex, OpenCode, Gemini, Pi, …) and get failure-mode + efficiency findings. CLI + SDK over the @tangle-network/agent-eval analyst suite — observe live sessions, run your own analysts, redact, and upload to the Tangle Intelligence Platform.",
   "type": "module",
   "license": "MIT",
@@ -48,9 +48,9 @@
   },
   "dependencies": {
     "@tangle-network/agent-eval": "0.140.1",
-    "@tangle-network/agent-runtime": "0.120.0",
+    "@tangle-network/agent-runtime": "0.121.0",
     "@tangle-network/agent-trace-contract": "^1.0.2",
-    "@tangle-network/sandbox": "0.15.2"
+    "@tangle-network/sandbox": "0.16.0"
   },
   "devDependencies": {
     "@types/node": "^22.0.0",