@tangle-network/traces 0.11.0 → 0.11.2

package/dist/index.js

@@ -15,6 +15,7 @@ import {
   CopilotAdapter,
   DEFAULT_HARNESS,
   EMPTY_SPAN_RUN_SNAPSHOT,
+  EXPORT_RULE,
   EmptySessionError,
   FactoryAdapter,
   ForgeAdapter,
@@ -31,6 +32,7 @@ import {
   RULE,
   SPAWN_JOURNAL_FILE,
   STATUS_MARK,
+  SandboxCounterfactualRunner,
   SessionWorkflowError,
   TRACES_REDACTION_RULES,
   ZERO_ROLL,
@@ -38,8 +40,10 @@ import {
   alreadyUploaded,
   analyzeAdoption,
   analyzeLiveBatch,
+  analyzeLoopConvergence,
   analyzeReactions,
   analyzeSpans,
+  analyzeSteeringChain,
   applyRedactor,
   buildPolicyEvidenceRecord,
   buildSessionIndex,
@@ -51,6 +55,7 @@ import {
   classifyLiveActions,
   classifyReaction,
   claudeActor,
+  cmdReplayVerify,
   codexActor,
   collectPolicyEvidence,
   collectSessionIndex,
@@ -58,10 +63,12 @@ import {
   collectSessionWorkflow,
   commandAnalyzer,
   commandRedactor,
+  conformanceOfSpans,
   connectors,
   countSkillRunsJsonl,
   cwdMatchesSelection,
   defaultTraceLiveAnalysts,
+  deriveFailureSignature,
   describeSessionRelationship,
   equivalentGitCwds,
   executeUpload,
@@ -74,8 +81,11 @@ import {
   haloAnalyzer,
   hodoscopeAnalyzer,
   importCodeTraceBench,
+  incompleteInputsNote,
   indexSessionIdsByTrace,
   inferRole,
+  inferSpanKind,
+  ingestCodeTraceBenchSteps,
   inspectSessionIndex,
   int,
   isChatTrajectory,
@@ -91,8 +101,12 @@ import {
   mergeTracesConfig,
   ms,
   normalizeRemote,
+  otlpRowToSpan,
   outboundHash,
   parseIsoToEpochMs,
+  parseObservationOutput,
+  parseRecordedReturncode,
+  parseReplayVerifyArgs,
   parseSession,
   parseSince,
   planTraceAgenticRoute,
@@ -101,6 +115,7 @@ import {
   readFileRunContext,
   readJsonFile,
   readJsonl,
+  readOtlpInput,
   readRunContextJournal,
   readRunContextSnapshot,
   readRunWatchSnapshot,
@@ -108,11 +123,16 @@ import {
   readSpanRunSnapshot,
   recordSessionCorruption,
   redactSpans,
+  renderConformance,
   renderInspectionReport,
   renderRunContextSnapshot,
   renderRunWatchSnapshot,
   renderSpanRunSnapshot,
+  renderValidation,
+  replayVerify,
+  replayVerifyUsage,
   resolveAdapter,
+  resolveOtlpInputFiles,
   resolveRepoAttrs,
   resolveRunContextDir,
   resolveRunWatchTarget,
@@ -124,6 +144,7 @@ import {
   runTraceImprovement,
   runTraceInvestigation,
   runTraceStoreInvestigation,
+  sandboxReplayBackend,
   saveReport,
   saveState,
   scanSessions,
@@ -136,6 +157,7 @@ import {
   sessionHash,
   sessionIdFromAttributes,
   sessionIntegrityAttributes,
+  skippedAnalyses,
   span,
   spanRunTotals,
   stampRepoAttrs,
@@ -145,6 +167,7 @@ import {
   streamSessions,
   summarizeOtlpExecution,
   summarizeSpanExecution,
+  summarizeSpanStructure,
   takeJsonl,
   textIsSynthetic,
   toOpenInferenceSpan,
@@ -154,13 +177,16 @@ import {
   totalAgenticFailureMessage,
   traceAgenticKinds,
   traceStreamEventsFromSpans,
+  unavailableCapabilities,
   uploadKey,
   usd,
   userPromptSpan,
   validateOtlpSpans,
+  validationExitCode,
   watchRunTarget,
   watchSessions,
   wrap,
+  wrapActionForExec,
   writeHodoscopeInput,
   writeInspectionReportFile,
   writeOtlpFile,
@@ -169,7 +195,7 @@ import {
   writeTraceEvidenceExportFile,
   writeTraceImprovementArtifacts,
   zeroPricedCost
-} from "./chunk-JCI5YXNT.js";
+} from "./chunk-UY3M7XUF.js";
 
 // src/collect.ts
 async function collectSessions(opts = {}) {
@@ -212,6 +238,7 @@ export {
   DEFAULT_HARNESS,
   DEFAULT_REDACTION_RULES,
   EMPTY_SPAN_RUN_SNAPSHOT,
+  EXPORT_RULE,
   EmptySessionError,
   FactoryAdapter,
   ForgeAdapter,
@@ -228,6 +255,7 @@ export {
   RULE,
   SPAWN_JOURNAL_FILE,
   STATUS_MARK,
+  SandboxCounterfactualRunner,
   SessionWorkflowError,
   TRACES_REDACTION_RULES,
   ZERO_ROLL,
@@ -235,8 +263,10 @@ export {
   alreadyUploaded,
   analyzeAdoption,
   analyzeLiveBatch,
+  analyzeLoopConvergence,
   analyzeReactions,
   analyzeSpans,
+  analyzeSteeringChain,
   applyRedactor,
   buildDefaultAnalystRegistry,
   buildPolicyEvidenceRecord,
@@ -249,6 +279,7 @@ export {
   classifyLiveActions,
   classifyReaction,
   claudeActor,
+  cmdReplayVerify,
   codexActor,
   collectPolicyEvidence,
   collectSessionIndex,
@@ -257,11 +288,13 @@ export {
   collectSessions,
   commandAnalyzer,
   commandRedactor,
+  conformanceOfSpans,
   connectors,
   countSkillRunsJsonl,
   createHostedClient,
   cwdMatchesSelection,
   defaultTraceLiveAnalysts,
+  deriveFailureSignature,
   describeSessionRelationship,
   equivalentGitCwds,
   executeUpload,
@@ -275,8 +308,11 @@ export {
   hodoscopeAnalyzer,
   hostedClientFromEnv,
   importCodeTraceBench,
+  incompleteInputsNote,
   indexSessionIdsByTrace,
   inferRole,
+  inferSpanKind,
+  ingestCodeTraceBenchSteps,
   inspectSessionIndex,
   int,
   isChatTrajectory,
@@ -293,8 +329,12 @@ export {
   mergeTracesConfig,
   ms,
   normalizeRemote,
+  otlpRowToSpan,
   outboundHash,
   parseIsoToEpochMs,
+  parseObservationOutput,
+  parseRecordedReturncode,
+  parseReplayVerifyArgs,
   parseSession,
   parseSince,
   planTraceAgenticRoute,
@@ -303,6 +343,7 @@ export {
   readFileRunContext,
   readJsonFile,
   readJsonl,
+  readOtlpInput,
   readRunContextJournal,
   readRunContextSnapshot,
   readRunWatchSnapshot,
@@ -312,11 +353,16 @@ export {
   redactSpans,
   redactString,
   redactValue,
+  renderConformance,
   renderInspectionReport,
   renderRunContextSnapshot,
   renderRunWatchSnapshot,
   renderSpanRunSnapshot,
+  renderValidation,
+  replayVerify,
+  replayVerifyUsage,
   resolveAdapter,
+  resolveOtlpInputFiles,
   resolveRepoAttrs,
   resolveRunContextDir,
   resolveRunWatchTarget,
@@ -328,6 +374,7 @@ export {
   runTraceImprovement,
   runTraceInvestigation,
   runTraceStoreInvestigation,
+  sandboxReplayBackend,
   saveReport,
   saveState,
   scanSessions,
@@ -340,6 +387,7 @@ export {
   sessionHash,
   sessionIdFromAttributes,
   sessionIntegrityAttributes,
+  skippedAnalyses,
   span,
   spanRunTotals,
   stampRepoAttrs,
@@ -349,6 +397,7 @@ export {
   streamSessions,
   summarizeOtlpExecution,
   summarizeSpanExecution,
+  summarizeSpanStructure,
   takeJsonl,
   textIsSynthetic,
   toOpenInferenceSpan,
@@ -358,13 +407,16 @@ export {
   totalAgenticFailureMessage,
   traceAgenticKinds,
   traceStreamEventsFromSpans,
+  unavailableCapabilities,
   uploadKey,
   usd,
   userPromptSpan,
   validateOtlpSpans,
+  validationExitCode,
   watchRunTarget,
   watchSessions,
   wrap,
+  wrapActionForExec,
   writeHodoscopeInput,
   writeInspectionReportFile,
   writeOtlpFile,

package/docs/replay-verify.md

@@ -0,0 +1,55 @@
+# replay-verify — executed proofs for trajectory findings
+
+An analyst finding today is a cited claim: "step 37 is where the run went wrong."
+`traces replay-verify` upgrades that claim to an executed proof: it replays the trajectory prefix inside a real sandbox on the trajectory's own docker image, then runs two counterfactual arms at the error-critical step k:
+
+- **Arm A** re-executes the recorded step k and checks the recorded failure reproduces — same returncode plus a stable output substring (the failure signature).
+- **Arm B** executes a corrected step k and checks the failure vanishes (exit 0 and the signature absent).
+
+Each arm gets a fresh sandbox with its own prefix replay, so arm B is never contaminated by arm A's side effects.
+Under the hood each arm is an agent-eval `runCounterfactual` meta-run (`layer='meta'`, `parentRunId` = the ingested trajectory run), with the sandbox-backed `CounterfactualRunner` from `src/replay-verify.ts` supplying the `executeFrom` callback that scaffold leaves to consumers.
+
+## Usage
+
+```sh
+traces replay-verify \
+  --steps normalized/<traj_id>/steps.json \
+  --image <docker_config.base_image> \
+  --at 37 \
+  --cwd /home \
+  --fix-command "sed -i '145d' /home/zstd/programs/zstdcli.c && make -C /home/zstd zstd -j2 V=1" \
+  --signature 'undeclared (first use in this function)' \
+  --out ./replay-out \
+  --base-url http://127.0.0.1:4097 --api-key-env SANDBOX_API_KEY
+```
+
+Outputs `replay-verdict.json` (machine-readable: prefix divergences, both arms, timings, run ids) and `report.md` (human-readable with real stdout/stderr excerpts).
+`traces replay-verify --help` lists every flag.
+
+## Execution semantics
+
+Every step runs the way mini-SWE ran it: a fresh `/bin/sh` subshell from a fixed workdir.
+The action is base64-piped into `sh` after `cd`-ing to `--cwd`, so arbitrary quoting in recorded actions survives, and the reported exit code is the action's own.
+
+Prefix divergences (recorded returncode ≠ replayed exit) are recorded per step, never abort the replay, and are surfaced in the verdict.
+A high divergence rate is a finding about replayability, not a failure of the tool.
+
+## Honest limits
+
+- **Only SWE-style trajectories are replayable.** The docker image comes from the raw trajectory's `info.docker_config.base_image`; terminal-bench-style tasks that need external compose peers (databases, services) cannot be replayed with a single image.
+- **Commands never run as root.** The sandbox platform pins every customer command to the configured non-root subprocess identity (uid 1000 by default) — a deliberate security invariant with no override. Trajectories recorded as root (mini-SWE's default) will hit permission errors on root-owned trees. Fix by deriving a replay image that chowns the working tree to the sandbox uid:
+
+  ```dockerfile
+  FROM mswebench/facebook_m_zstd:pr-1733
+  RUN chown -R 1000:1000 /home
+  ```
+
+  This preserves the recorded failure semantics for build/edit workflows (they depend on writability, not on uid 0). The verdict records the image actually used.
+- **Copy-on-write forks are firecracker-only.** `box.branch()` could replay the prefix once and fork at k; on the docker driver each arm replays the prefix in its own fresh sandbox instead. Wall-time cost is one extra prefix replay per arm.
+- **Environment drift shows up as divergence.** The sandbox mounts a workspace at `/home/agent` and injects platform env; recorded observations that depend on exact `ls` output or env contents can differ while returncodes still match. Only returncodes drive the divergence count.
+- **The failure signature is derived, not assumed.** Default: the first recorded-output line containing "error". Compiler quote glyphs vary with locale, so prefer an explicit `--signature` with an ASCII-stable substring.
+
+## Orchestrator prerequisites
+
+replay-verify talks to a sandbox API (`--base-url`); in local development that is the sandbox SDK adapter in front of an orchestrator running the docker driver.
+Arbitrary docker images need the platform control runtime (node injected from a mounted Nix profile) because sidecar-less images rarely ship node: the orchestrator needs `NIX_PROFILE_PATH` set, and this CLI passes the `SIDECAR_PLATFORM_CONTROL_RUNTIME` container-env marker on create (the production sandbox API derives the same marker from `agent: false`).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/traces",
-  "version": "0.11.0",
+  "version": "0.11.2",
   "description": "Point it at your coding-agent session traces (Claude Code, Codex, OpenCode, Gemini, Pi, …) and get failure-mode + efficiency findings. CLI + SDK over the @tangle-network/agent-eval analyst suite — observe live sessions, run your own analysts, redact, and upload to the Tangle Intelligence Platform.",
   "type": "module",
   "license": "MIT",
@@ -47,8 +47,9 @@
     "node": ">=22"
   },
   "dependencies": {
-    "@tangle-network/agent-eval": "0.140.0",
-    "@tangle-network/agent-runtime": "0.116.0",
+    "@tangle-network/agent-eval": "0.140.1",
+    "@tangle-network/agent-runtime": "0.120.0",
+    "@tangle-network/agent-trace-contract": "^1.0.2",
     "@tangle-network/sandbox": "0.15.2"
   },
   "devDependencies": {
@@ -64,6 +65,7 @@
     "dev": "tsx src/cli.ts",
     "build": "tsup src/index.ts src/cli.ts --format esm --target node22 --clean --dts",
     "check:package": "node scripts/check-package-bin.mjs",
+    "check:source": "node scripts/check-source-text.mjs",
     "test": "vitest run",
     "test:watch": "vitest",
     "typecheck": "tsc --noEmit"

package/skills/inspect-agent-traces/SKILL.md

@@ -8,6 +8,34 @@ description: Inspect real agent workflows with the published Traces CLI and expo
 Use the deterministic CLI first.
 Keep inspection local and read-only.
 
+## Choose the way in
+
+If the trace is already OTLP (a system that emits `@tangle-network/agent-trace-contract`
+spans, or any conforming exporter), read it directly. No adapter is involved.
+
+```bash
+traces validate spans.otlp.jsonl          # what can this trace answer? exit 1 on error findings
+traces analyze --otlp spans.otlp.jsonl --out .traces/current.md
+traces analyze --otlp results/sessions --out .traces/all.md   # a directory of exports
+```
+
+Use `--harness` only for coding agents whose on-disk format we do not control.
+Those adapters are the legacy edge, not the way to integrate a system you own.
+
+A directory is a run directory: it holds the span export beside raw event, stream and
+SDK logs that are also `*.jsonl`. Only the OTLP files are read, and the rest are listed
+with what they actually hold. An `otlp/` subdirectory, when the producer made one, is
+read on its own.
+
+Any section headed `inputs incomplete`, or carrying an `Inputs incomplete` line above
+its table, is computed from a field the trace does not carry everywhere. Report those
+numbers as uncaptured, never as zero spend. The `trace conformance` section at the top
+names every such capability once; the markers repeat it where the number actually is.
+
+For a loop trace, read `round-over-round convergence` (did round N+1 improve on N) and
+`steering chain` (which verdict caused which retry) before drawing any conclusion about
+whether the agent was making progress.
+
 ## Select the workflow
 
 ```bash
@@ -32,7 +60,7 @@ Write normalized OpenInference spans for another tool:
 
 ```bash
 traces convert --harness codex --current --latest-turn --workflow \
-  --otlp .traces/current.otlp.jsonl
+  --otlp-out .traces/current.otlp.jsonl
 ```
 
 Write the complete deterministic review packet:
@@ -47,7 +75,8 @@ It does not edit an agent, repository, memory store, or knowledge base.
 
 ## Report
 
-- State the harness, selected task boundary, session and span counts, and integrity warnings.
+- State the source (`--otlp <path>` or the harness), selected task boundary, session and span counts, and integrity warnings.
+- State which capabilities the trace could not support, and name the analyses that reported nothing because of it.
 - Cite each finding with its exact `trace://` reference.
 - Mark missing outcome, cost, token, skill, or relationship data as unknown.
 - Do not infer task success from a completion message.