@tangle-network/traces 0.11.3 → 0.11.5

package/dist/index.js

@@ -14,6 +14,7 @@ import {
   CodexTaskScopeError,
   CopilotAdapter,
   DEFAULT_HARNESS,
+  DEFAULT_SANDBOX_BASE_URL,
   EMPTY_SPAN_RUN_SNAPSHOT,
   EXPORT_RULE,
   EmptySessionError,
@@ -46,8 +47,10 @@ import {
   analyzeSpans,
   analyzeSteeringChain,
   applyRedactor,
+  assertSandboxReachable,
   buildFixPrompt,
   buildPolicyEvidenceRecord,
+  buildRetryFixPrompt,
   buildSessionIndex,
   buildSessionIndexFromRows,
   buildSpanRunTree,
@@ -56,9 +59,12 @@ import {
   chatTrajectoryToSpans,
   classifyLiveActions,
   classifyReaction,
+  classifyVerdict,
   claudeActor,
+  clipText,
   cmdReplayVerify,
   cmdReplayVerifyBatch,
+  cmdVerifyFindings,
   codexActor,
   collectPolicyEvidence,
   collectSessionIndex,
@@ -68,6 +74,7 @@ import {
   commandRedactor,
   conformanceOfSpans,
   connectors,
+  countScriptCommands,
   countSkillRunsJsonl,
   cwdMatchesSelection,
   defaultTraceLiveAnalysts,
@@ -84,6 +91,8 @@ import {
   extractFixCommand,
   fileRunContextSupervisorRunReader,
   findSpanSourceFiles,
+  findingReplayStep,
+  findingTrajectoryId,
   forEachTreeNode,
   generateFixCommand,
   goldIncorrectSteps,
@@ -114,6 +123,7 @@ import {
   otlpRowToSpan,
   outboundHash,
   parseCorpusFlag,
+  parseIncorrectStepsSubject,
   parseIsoToEpochMs,
   parseObservationOutput,
   parseRecordedReturncode,
@@ -121,10 +131,12 @@ import {
   parseReplayVerifyArgs,
   parseSession,
   parseSince,
+  parseVerifyFindingsArgs,
   planTraceAgenticRoute,
   planUpload,
   readCoordinationLog,
   readFileRunContext,
+  readFindingsFile,
   readJsonFile,
   readJsonl,
   readLabelEntries,
@@ -143,11 +155,15 @@ import {
   renderRunWatchSnapshot,
   renderSpanRunSnapshot,
   renderValidation,
+  renderVerifiedFindingsSection,
   replayBatchUsage,
   replayVerify,
+  replayVerifyFinding,
   replayVerifyUsage,
   resolveAdapter,
   resolveCaseResources,
+  resolveFindingInvocation,
+  resolveFindingReplayability,
   resolveOtlpInputFiles,
   resolveRepoAttrs,
   resolveRunContextDir,
@@ -155,6 +171,7 @@ import {
   resolveSessionRepoAttrs,
   runCommand,
   runExternalAnalyzers,
+  runFixLoop,
   runPipelines,
   runReplayBatch,
   runTotals,
@@ -201,6 +218,8 @@ import {
   userPromptSpan,
   validateOtlpSpans,
   validationExitCode,
+  verifyFindings,
+  verifyFindingsUsage,
   watchRunTarget,
   watchSessions,
   wrap,
@@ -214,7 +233,7 @@ import {
   writeTraceImprovementArtifacts,
   zaiChatCaller,
   zeroPricedCost
-} from "./chunk-CK5F5TWO.js";
+} from "./chunk-VTYOCFGF.js";
 
 // src/collect.ts
 async function collectSessions(opts = {}) {
@@ -230,79 +249,6 @@ async function collectSessions(opts = {}) {
   return out;
 }
 
-// src/replay-wire.ts
-var SUBJECT_PATTERN = /^incorrect-steps-(\d+)-(\d+)-(escaped|unescaped)-consequence-(\d+)$/;
-function parseIncorrectStepsSubject(subject) {
-  const match = SUBJECT_PATTERN.exec(subject);
-  if (!match) return null;
-  return {
-    firstStep: Number(match[1]),
-    lastStep: Number(match[2]),
-    escapeStatus: match[3],
-    consequenceStep: Number(match[4])
-  };
-}
-function resolveFindingInvocation(finding, corpora) {
-  const subject = parseIncorrectStepsSubject(finding.subject);
-  if (!subject) {
-    throw new Error(
-      `replay-wire: subject '${finding.subject}' is not incorrect-steps-<first>-<last>-<escaped|unescaped>-consequence-<step>`
-    );
-  }
-  const failures = [];
-  for (const corpus of corpora) {
-    const resolution = resolveCaseResources(corpus, finding.trajId);
-    if (resolution.resolved) {
-      const at = subject.firstStep;
-      const step = resolution.resources.steps.find((s) => s.step_id === at);
-      if (!step) {
-        throw new Error(
-          `replay-wire: finding step ${at} is outside ${finding.trajId} (${resolution.resources.steps.length} steps)`
-        );
-      }
-      return { resources: resolution.resources, subject, at };
-    }
-    failures.push(`${corpus.name}: ${resolution.reason}`);
-  }
-  throw new Error(
-    `replay-wire: trajectory ${finding.trajId} is not replayable in any corpus \u2014 ${failures.join("; ")}`
-  );
-}
-async function replayVerifyFinding(finding, options) {
-  if (options.fixCaller && options.fixCommand !== void 0) {
-    throw new Error("replay-wire: pass fixCaller or fixCommand, not both");
-  }
-  const invocation = resolveFindingInvocation(finding, options.corpora);
-  let fixCommand = options.fixCommand ?? null;
-  if (options.fixCaller) {
-    const generated = await generateFixCommand(options.fixCaller, {
-      taskStatement: invocation.resources.taskStatement,
-      steps: invocation.resources.steps,
-      k: invocation.at
-    });
-    if (!generated.succeeded) {
-      throw new Error(`replay-wire: fix generation failed \u2014 ${generated.error}`);
-    }
-    fixCommand = generated.value.command;
-  }
-  const verdict = await replayVerify({
-    stepsPath: invocation.resources.stepsPath,
-    image: invocation.resources.image,
-    at: invocation.at,
-    fixCommand: fixCommand ?? void 0,
-    cwd: invocation.resources.cwd,
-    out: options.out,
-    caseId: invocation.resources.trajId,
-    stepTimeoutMs: options.stepTimeoutMs ?? invocation.resources.recordedStepTimeoutMs ?? void 0,
-    prefixLimit: options.prefixLimit,
-    backend: options.backend,
-    apiKey: options.apiKey,
-    baseUrl: options.baseUrl,
-    onProgress: options.onProgress
-  });
-  return { invocation, fixCommand, verdict };
-}
-
 // src/index.ts
 import {
   AnalystRegistry,
@@ -329,6 +275,7 @@ export {
   CopilotAdapter,
   DEFAULT_HARNESS,
   DEFAULT_REDACTION_RULES,
+  DEFAULT_SANDBOX_BASE_URL,
   EMPTY_SPAN_RUN_SNAPSHOT,
   EXPORT_RULE,
   EmptySessionError,
@@ -361,9 +308,11 @@ export {
   analyzeSpans,
   analyzeSteeringChain,
   applyRedactor,
+  assertSandboxReachable,
   buildDefaultAnalystRegistry,
   buildFixPrompt,
   buildPolicyEvidenceRecord,
+  buildRetryFixPrompt,
   buildSessionIndex,
   buildSessionIndexFromRows,
   buildSpanRunTree,
@@ -372,9 +321,12 @@ export {
   chatTrajectoryToSpans,
   classifyLiveActions,
   classifyReaction,
+  classifyVerdict,
   claudeActor,
+  clipText,
   cmdReplayVerify,
   cmdReplayVerifyBatch,
+  cmdVerifyFindings,
   codexActor,
   collectPolicyEvidence,
   collectSessionIndex,
@@ -385,6 +337,7 @@ export {
   commandRedactor,
   conformanceOfSpans,
   connectors,
+  countScriptCommands,
   countSkillRunsJsonl,
   createHostedClient,
   cwdMatchesSelection,
@@ -402,6 +355,8 @@ export {
   extractFixCommand,
   fileRunContextSupervisorRunReader,
   findSpanSourceFiles,
+  findingReplayStep,
+  findingTrajectoryId,
   forEachTreeNode,
   generateFixCommand,
   goldIncorrectSteps,
@@ -442,10 +397,12 @@ export {
   parseReplayVerifyArgs,
   parseSession,
   parseSince,
+  parseVerifyFindingsArgs,
   planTraceAgenticRoute,
   planUpload,
   readCoordinationLog,
   readFileRunContext,
+  readFindingsFile,
   readJsonFile,
   readJsonl,
   readLabelEntries,
@@ -466,6 +423,7 @@ export {
   renderRunWatchSnapshot,
   renderSpanRunSnapshot,
   renderValidation,
+  renderVerifiedFindingsSection,
   replayBatchUsage,
   replayVerify,
   replayVerifyFinding,
@@ -473,6 +431,7 @@ export {
   resolveAdapter,
   resolveCaseResources,
   resolveFindingInvocation,
+  resolveFindingReplayability,
   resolveOtlpInputFiles,
   resolveRepoAttrs,
   resolveRunContextDir,
@@ -480,6 +439,7 @@ export {
   resolveSessionRepoAttrs,
   runCommand,
   runExternalAnalyzers,
+  runFixLoop,
   runPipelines,
   runReplayBatch,
   runTotals,
@@ -526,6 +486,8 @@ export {
   userPromptSpan,
   validateOtlpSpans,
   validationExitCode,
+  verifyFindings,
+  verifyFindingsUsage,
   watchRunTarget,
   watchSessions,
   wrap,

package/docs/replay-verify.md

@@ -77,6 +77,17 @@ A corpus is a gold label file plus a prepared directory (`normalized/<traj_id>/s
 
 Fix generation (`--fix generate`) runs ONE chat completion per arm-A-reproduced case against an OpenAI-compatible endpoint (default: glm-5.2 on z.ai). The prompt carries the gold step's action and observation, ±3 surrounding steps, and the task statement; the reply must be a single corrected shell command. Cases beyond `--max-fix-cases` are excluded by a seeded random sample and marked `sampled-out` in the report. Arm B replays the prefix in its own fresh sandbox and executes the corrected command; `failureVanished` = exit 0 with the failure signature absent.
 
+### Iterative fix loop (`--fix loop`)
+
+`--fix loop` (in `src/replay-fix-loop.ts`) replaces the single shot with an execution-feedback loop of up to `--fix-attempts` attempts (default 3) per case:
+
+- Attempt 1 is byte-identical to the one-shot prompt, so the report's `fixFlipAttempt1` stays directly comparable to `--fix generate`.
+- When an arm fails (nonzero exit or the failure signature persists) or the model call itself fails, the next prompt carries every prior command with its REAL executed stdout/stderr (clipped tails), never a paraphrase.
+- Retries may answer with a short script — at most 5 commands in one fenced block — executed as ONE `/bin/sh` unit; a longer script is rejected as a failed attempt and the rejection feeds the next prompt.
+- Every attempt runs in its own fresh sandbox with the same replayed prefix. A used sandbox is never mutated mid-arm, so a flip always proves the corrected step against the recorded prefix state, not against debris from an earlier attempt. Wall-time cost is one extra prefix replay per attempt.
+
+The report adds `fixFlipAttempt1` (flips at attempt 1 over cases whose attempt 1 executed), `flipsByAttempt`, the full per-attempt trail on each case row (`fix.attempts`), and one `armB-attempt<N>-result.json` per executed attempt (`armB-result.json` keeps the last executed attempt — the flipped one when the loop flips).
+
 Honest-reporting notes baked into the report:
 
 - A gold step whose recorded observation carries **no returncode** can never satisfy "arm A reproduces the recorded returncode": it counts against the replayability rate and the per-case table says why.
@@ -99,6 +110,12 @@ const { invocation, fixCommand, verdict } = await replayVerifyFinding(
 
 The subject grammar is the analyst benchmark's `incorrect-steps-<first>-<last>-<escaped|unescaped>-consequence-<step>`; `--at` is the finding's **first** incorrect step (the finding's claim, which may differ from the gold label). The wire resolves the trajectory across the given corpora, generates the arm-B fix through the same one-call generator (or accepts a pre-supplied `fixCommand`), and returns the full `ReplayVerdict`. It throws with a precise reason when the finding cannot be replayed (malformed subject, unknown trajectory, non-SWE case, step out of range) — the product surfaces that reason instead of a proof.
 
+## Product surface — verified findings
+
+`traces verify-findings` (and `traces analyze --verify-findings`) runs this proof per recorded analyst finding and annotates each with `reproduced | fix-flipped | divergent | not-replayable` plus a receipt directory.
+Unlike the wire above it accepts the shapes analysts actually emit (`incorrect-step-<n>` subjects, `metadata.block_first_step`, `trace://` evidence refs) and never throws on a finding-shaped dead end — the dead end becomes the finding's honest `not-replayable` receipt.
+See [Verified findings](./trace-analysts.md#verified-findings-executed-replay).
+
 ## Orchestrator prerequisites
 
 replay-verify talks to a sandbox API (`--base-url`); in local development that is the sandbox SDK adapter in front of an orchestrator running the docker driver.

package/docs/trace-analysts.md

@@ -210,6 +210,34 @@ console.log(renderAnalystBenchmarkMarkdown(result))
 Public labels test the measurement code, not the quality of every built-in analyst automatically.
 A real quality claim requires running the analyst over the corresponding trajectories, retaining all rows, and comparing it with named alternatives at equal model and request limits.
 
+## Verified findings (executed replay)
+
+An analyst finding is a cited claim until something executes it.
+`traces analyze --verify-findings` (and the standalone `traces verify-findings`) replays each finding's trajectory prefix in a real sandbox, re-runs the accused step, and annotates every finding with an executed verdict:
+
+| Verdict | Meaning |
+|---|---|
+| `reproduced` | the recorded failure signature (returncode + stable output substring) reproduced when the accused step re-ran |
+| `fix-flipped` | reproduced, and a supplied corrected command made the failure vanish in a fresh replay |
+| `divergent` | the step executed but the recorded failure did not reproduce — evidence against the finding, or against replay fidelity (the receipt carries prefix divergences so you can tell which) |
+| `not-replayable` | the finding could not be executed; the receipt names the precise reason (no step subject, unknown trajectory, no docker image, submit step, …) |
+
+```bash
+# Verify the findings an analyze run produced (marks each finding in the report):
+traces analyze --last 1 --llm --verify-findings \
+  --replay-corpus holdout=labels.json::prepared/ --verify-out ./receipts
+
+# Verify findings recorded earlier (e.g. extracted from an eval result.json):
+traces verify-findings --findings findings.json --out ./receipts \
+  --steps normalized/<traj>/steps.json --image <replay-ready-image> --cwd /app
+```
+
+Findings are matched by the shape analysts emit: subject `incorrect-step-<n>` (or the wire form `incorrect-steps-<f>-<l>-…`), `metadata.block_first_step`, and `trace://<trajectory>/…` evidence refs.
+Findings accusing the same step share one executed proof; each finding still gets its own receipt directory (`receipt.json` plus, when executed, `replay-verdict.json` and `report.md` with real stdout/stderr).
+Verification is execution, not generation: no LLM is involved unless you pass `--fix-command`.
+A missing sandbox is an error when any finding is replayable — verification never silently skips.
+Sandbox setup, execution semantics, and honest limits are in [Replay verification](./replay-verify.md).
+
 ## Turn findings into improvement
 
 Do not train or rewrite policy from an analyst's own prose alone.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/traces",
-  "version": "0.11.3",
+  "version": "0.11.5",
   "description": "Point it at your coding-agent session traces (Claude Code, Codex, OpenCode, Gemini, Pi, …) and get failure-mode + efficiency findings. CLI + SDK over the @tangle-network/agent-eval analyst suite — observe live sessions, run your own analysts, redact, and upload to the Tangle Intelligence Platform.",
   "type": "module",
   "license": "MIT",
@@ -47,10 +47,10 @@
     "node": ">=22"
   },
   "dependencies": {
-    "@tangle-network/agent-eval": "0.140.1",
-    "@tangle-network/agent-runtime": "0.121.0",
+    "@tangle-network/agent-eval": "0.143.0",
+    "@tangle-network/agent-runtime": "0.126.0",
     "@tangle-network/agent-trace-contract": "^1.0.2",
-    "@tangle-network/sandbox": "0.16.0"
+    "@tangle-network/sandbox": "0.17.2"
   },
   "devDependencies": {
     "@types/node": "^22.0.0",