@tangle-network/traces 0.11.0 → 0.11.3

package/dist/cli.js

@@ -6,6 +6,8 @@ import {
   appendAll,
   buildPolicyEvidenceRecord,
   buildSessionIndexFromRows,
+  cmdReplayVerify,
+  cmdReplayVerifyBatch,
   collectSessionSelection,
   commandAnalyzer,
   commandRedactor,
@@ -26,8 +28,10 @@ import {
   parseSession,
   parseSince,
   planUpload,
+  readOtlpInput,
   readSessionIndexFile,
   renderInspectionReport,
+  renderValidation,
   resolveAdapter,
   resolveRunWatchTarget,
   runTraceImprovement,
@@ -42,6 +46,7 @@ import {
   streamSessions,
   totalAgenticFailureMessage,
   traceStreamEventsFromSpans,
+  validationExitCode,
   watchRunTarget,
   watchSessions,
   writeInspectionReportFile,
@@ -49,7 +54,7 @@ import {
   writePolicyEvidenceFile,
   writeSessionIndexFile,
   writeTraceEvidenceExportFile
-} from "./chunk-JCI5YXNT.js";
+} from "./chunk-CK5F5TWO.js";
 
 // src/cli.ts
 import { readFileSync } from "fs";
@@ -155,6 +160,9 @@ function parseArgs(argv) {
       case "--otlp":
         a.otlp = next();
         break;
+      case "--otlp-out":
+        a.otlpOut = next();
+        break;
       case "--llm":
         a.llm = true;
         break;
@@ -239,6 +247,30 @@ function parseArgs(argv) {
   }
   return a;
 }
+var OTLP_INPUT_COMMANDS = /* @__PURE__ */ new Set(["analyze", "investigate", "improve", "stream", "validate"]);
+function migrateWritingOtlpFlag(args) {
+  if (args.otlp === void 0 || OTLP_INPUT_COMMANDS.has(args.command)) return args;
+  if (args.otlpOut !== void 0) {
+    throw new Error(
+      `${args.command} was given both --otlp and --otlp-out. --otlp is the deprecated spelling of --otlp-out on writing commands; pass only --otlp-out.`
+    );
+  }
+  console.error(
+    `warning: --otlp is deprecated on \`${args.command}\` and will be removed in 0.12. --otlp now READS OTLP-JSONL; use --otlp-out <path> to write the artifact. Treating it as --otlp-out for this run.`
+  );
+  return { ...args, otlpOut: args.otlp, otlp: void 0 };
+}
+function validateOtlpSelection(raw) {
+  const args = migrateWritingOtlpFlag(raw);
+  if (args.otlp === void 0) return args;
+  if (args.input) throw new Error("--otlp cannot be combined with an input file");
+  if (args.session) throw new Error("--otlp reads a file and cannot be combined with --session");
+  if (args.supervisorRunDir) throw new Error("--otlp cannot be combined with --supervisor-run-dir");
+  if (args.workflow || args.latestTurn) {
+    throw new Error("--workflow and --latest-turn expand harness sessions and cannot be combined with --otlp");
+  }
+  return args;
+}
 var CURRENT_SESSION_COMMANDS = /* @__PURE__ */ new Set([
   "analyze",
   "investigate",
@@ -457,7 +489,45 @@ function selectedSessionSource(ref, spans, sessionIdOverride) {
     } : {}
   };
 }
+async function collectOtlpSpans(path) {
+  const input = await readOtlpInput(path);
+  if (input.spans.length === 0) {
+    throw new Error(
+      `no analyzable spans in ${path} (${input.files.length} file(s), ${input.rows.length} row(s), ${input.issues.length} unusable). Run \`traces validate ${path}\` for the per-finding reason.`
+    );
+  }
+  const fileByTrace = /* @__PURE__ */ new Map();
+  for (const file of input.files) {
+    for (const traceId of file.traceIds) if (!fileByTrace.has(traceId)) fileByTrace.set(traceId, file.path);
+  }
+  const byTrace = /* @__PURE__ */ new Map();
+  for (const span of input.spans) {
+    const grouped = byTrace.get(span.trace_id) ?? [];
+    grouped.push(span);
+    byTrace.set(span.trace_id, grouped);
+  }
+  return {
+    spans: [...input.spans],
+    harness: "otlp",
+    cwds: [],
+    sources: [...byTrace].map(([traceId, spans]) => selectedSessionSource({
+      harness: "otlp",
+      sessionId: traceId,
+      path: fileByTrace.get(traceId) ?? path,
+      cwd: null,
+      mtimeMs: 0
+    }, spans, traceId)),
+    conformance: input.validation,
+    conformanceSubject: path,
+    conformanceFiles: input.files,
+    conformanceSkipped: input.skipped,
+    conformanceIssues: input.issues,
+    conformanceWithheld: input.withheld,
+    conformanceUnreadable: input.unreadable
+  };
+}
 async function collectSpans(args) {
+  if (args.otlp) return collectOtlpSpans(args.otlp);
   if (args.input) {
     if (args.workflow) throw new Error("--workflow reads discovered sessions and cannot be combined with an input file");
     return collectImportedSpans(args);
@@ -481,11 +551,37 @@ async function collectSpans(args) {
     ...selection.workflow ? { workflow: selection.workflow } : {}
   };
 }
+async function cmdValidate(args) {
+  const path = args.otlp ?? args.input;
+  if (!path) {
+    throw new Error("validate needs an OTLP-JSONL file or directory: `traces validate <file|dir>`");
+  }
+  const input = await readOtlpInput(path);
+  const report = renderValidation(input.validation, {
+    subject: path,
+    files: input.files,
+    skipped: input.skipped,
+    issues: input.issues,
+    withheld: input.withheld,
+    unreadable: input.unreadable,
+    spans: input.spans
+  });
+  const validation = input.validation;
+  if (args.out) {
+    await saveReport(args.out, report);
+    console.log(
+      `conformance report \u2192 ${args.out}  (${validation.findings.length} finding(s), ${validation.capabilities.filter((capability) => capability.available).length}/${validation.capabilities.length} capabilities available)`
+    );
+  } else {
+    console.log(report);
+  }
+  process.exitCode = validationExitCode(validation);
+}
 async function cmdConvert(args) {
   const { spans, workflow } = await collectSpans(args);
   if (spans.length === 0) throw new Error("no spans found for the given selection");
   warnIncompleteWorkflow(workflow);
-  const path = await writeOtlpFile(spans, args.otlp);
+  const path = await writeOtlpFile(spans, args.otlpOut);
   console.log(`wrote ${spans.length} spans \u2192 ${path}`);
 }
 function workflowIssueText(issue) {
@@ -519,7 +615,7 @@ async function cmdEvidence(args) {
   const rows = selection.rows.filter((row) => row.spans.length > 0);
   if (rows.length === 0) throw new Error("no spans found for the given selection");
   warnIncompleteWorkflow(selection.workflow);
-  const otlpPath = args.otlp ? await writeOtlpFile(rows.flatMap((row) => row.spans), args.otlp) : void 0;
+  const otlpPath = args.otlpOut ? await writeOtlpFile(rows.flatMap((row) => row.spans), args.otlpOut) : void 0;
   const generatedAt = (/* @__PURE__ */ new Date()).toISOString();
   const records = await Promise.all(rows.map(
     (row) => buildPolicyEvidenceRecord(row.ref, row.spans, {
@@ -578,7 +674,7 @@ async function cmdExport(args) {
   if (!args.input) throw new Error("export needs an input file; run `traces export --help` for examples");
   const format = traceEvidenceFormat(args.format);
   const attributes = await loadExportAttributes(args);
-  const outPath = args.out ?? args.otlp;
+  const outPath = args.out ?? args.otlpOut;
   if (outPath) {
     const result2 = await writeTraceEvidenceExportFile(args.input, outPath, { format, attributes });
     console.log(
@@ -780,8 +876,20 @@ async function cmdInvestigate(args) {
   }
   assertAgenticAnalystsRan(args, result.agenticPerAnalyst);
 }
+function conformanceOptions(collected) {
+  return {
+    ...collected.conformance ? { conformance: collected.conformance } : {},
+    ...collected.conformanceSubject ? { conformanceSubject: collected.conformanceSubject } : {},
+    ...collected.conformanceFiles ? { conformanceFiles: collected.conformanceFiles } : {},
+    ...collected.conformanceSkipped ? { conformanceSkipped: collected.conformanceSkipped } : {},
+    ...collected.conformanceIssues ? { conformanceIssues: collected.conformanceIssues } : {},
+    ...collected.conformanceWithheld ? { conformanceWithheld: collected.conformanceWithheld } : {},
+    ...collected.conformanceUnreadable ? { conformanceUnreadable: collected.conformanceUnreadable } : {}
+  };
+}
 async function investigate(args, options = {}) {
-  const { spans, harness, cwds, sources, workflow } = await collectSpans(args);
+  const collected = await collectSpans(args);
+  const { spans, harness, cwds, sources, workflow } = collected;
   if (spans.length === 0) throw new Error("no spans found for the given selection");
   const config = args.config !== void 0 || options.loadDefaultConfig !== false ? await loadTracesConfig(args.config) : void 0;
   const analystModel = args.model ?? process.env.TRACES_ANALYST_MODEL ?? DEFAULT_ANALYST_MODEL;
@@ -796,14 +904,16 @@ async function investigate(args, options = {}) {
     engine,
     model: args.llm ? analystModel : args.model,
     budgetUsd: args.budget,
-    otlpOutPath: args.otlp,
+    otlpOutPath: args.otlpOut,
     externalAnalyzers: externalAnalyzersFromArgs(args),
     analyzerPrompt: args.analyzerPrompt,
-    log: analystLog
+    log: analystLog,
+    ...conformanceOptions(collected)
   }, config));
 }
 async function cmdImprove(args) {
-  const { spans, harness, cwds, sources, workflow } = await collectSpans(args);
+  const collected = await collectSpans(args);
+  const { spans, harness, cwds, sources, workflow } = collected;
   if (spans.length === 0) throw new Error("no spans found for the given selection");
   const config = await loadTracesConfig(args.config);
   const analystModel = args.model ?? process.env.TRACES_ANALYST_MODEL ?? DEFAULT_ANALYST_MODEL;
@@ -819,10 +929,11 @@ async function cmdImprove(args) {
       engine,
       model: args.llm ? analystModel : args.model,
       budgetUsd: args.budget,
-      otlpOutPath: args.otlp,
+      otlpOutPath: args.otlpOut,
       externalAnalyzers: externalAnalyzersFromArgs(args),
       analyzerPrompt: args.analyzerPrompt,
-      log: analystLog
+      log: analystLog,
+      ...conformanceOptions(collected)
     }, config),
     outDir: args.dir ?? args.out
   });
@@ -890,12 +1001,13 @@ async function streamExplicitSession(args, extraAnalysts) {
 async function cmdStream(args) {
   const config = await loadTracesConfig(args.config);
   const preset = streamPreset(args);
-  if (args.input) {
-    const { spans, harness } = await collectImportedSpans(args);
+  const source = args.otlp ?? args.input;
+  if (source) {
+    const { spans, harness } = args.otlp ? await collectOtlpSpans(args.otlp) : await collectImportedSpans(args);
     const ref = {
       harness,
-      sessionId: basename(args.input),
-      path: resolve(args.input),
+      sessionId: basename(source),
+      path: resolve(source),
       cwd: args.cwd ?? null,
       mtimeMs: Date.now()
     };
@@ -1034,7 +1146,7 @@ Window: since ${new Date(sinceMs).toISOString()}
     return;
   }
   if (args.dryRun) {
-    const res2 = await executeUpload(plan, { dryRun: true, otlpOut: args.otlp, stripContent: args.noContent, redactor });
+    const res2 = await executeUpload(plan, { dryRun: true, otlpOut: args.otlpOut, stripContent: args.noContent, redactor });
     console.log(`dry run: ${candidates.length - res2.skippedSessions} session(s), ${totalRedactions} redaction(s). Redacted OTLP -> ${res2.otlpPath}`);
     console.log("No upload performed. Set TANGLE_INGEST_URL / TANGLE_INGEST_API_KEY / TANGLE_TENANT_ID and drop --dry-run to send.");
     return;
@@ -1097,21 +1209,74 @@ The command validates every native step, rejects annotation leakage, writes
 <traj_id>.otlp.jsonl files, and records input and output SHA-256 hashes in
 codetracebench-import.json. The output directory must not already exist.`);
 }
+function usageValidate() {
+  console.log(`traces validate: report what a trace can and cannot answer
+
+Usage:
+  traces validate <file.jsonl|dir> [--out conformance.md]
+
+Reads OTLP-JSONL and prints the conformance report from
+@tangle-network/agent-trace-contract:
+
+  - findings by severity, each naming the CONSEQUENCE of the defect and the
+    capabilities it blocks
+  - the capability table: token-accounting, cost-attribution, tool-usage,
+    loop-convergence, tree-comparison, steering-chain, latency-analysis \u2014
+    available or not, with the trace's own reason
+  - rows that could not be read at all, with the line and the reason
+
+A directory reads the OTLP files under it, recursively \u2014 only the otlp/
+subdirectory when the producer made one. A run directory's raw event, stream and
+SDK logs are also *.jsonl; they are listed with what they actually hold, not read
+as several hundred broken spans. Identical re-declarations of a span across
+per-shot files are counted as repeats, not defects; only a repeat with DIFFERENT
+content is an error.
+
+Exit code:
+  0  no error findings, and at least one capability available (warnings mean
+     thinner analysis, not wrong analysis)
+  1  an error finding, or nothing analysable at all \u2014 an empty file, or one
+     whose every line was unreadable, answers no question and is not a pass
+
+Nothing is fixed up: a span with an unreadable timestamp is reported, never
+repaired, because a synthesized value puts invented work into a total.
+
+Examples:
+  traces validate spans.otlp.jsonl
+  traces validate results/sessions --out conformance.md
+  traces validate spans.otlp.jsonl && traces analyze --otlp spans.otlp.jsonl`);
+}
 function usage() {
-  console.log(`traces: analyze, observe & upload coding-agent sessions
+  console.log(`traces: analyze agent traces \u2014 yours, or any system's
+
+Two ways in:
+  --otlp <file|dir>   Spans a system already emitted in the
+                      @tangle-network/agent-trace-contract shape. No adapter, no
+                      translation. This is how you integrate a NEW system.
+  --harness <id>      Run an adapter over a coding agent's proprietary session
+                      store (Claude Code, Codex, OpenCode, \u2026) \u2014 the legacy edge,
+                      for formats we do not control.
 
 Commands:
+  validate  Report what a trace can and cannot answer (exit 1 on error findings)
   list      List discovered sessions
-  analyze   Run analyst suite + loop/waste pipelines over sessions or an input file
+  analyze   Run analyst suite + loop/waste pipelines over OTLP (--otlp), sessions,
+            or an input file; names every analysis the spans could not support
             (--supervisor-run-dir <dir> reports a supervision tree instead)
   investigate Run typed investigation flow, including BYO config + evidence-backed actions
   improve   Write findings, evidence, report, and canonical trace artifacts
-  convert   Emit OTLP-JSONL only (HALO: use analyze --analyzer halo)
+  convert   Emit OTLP-JSONL only, to --otlp-out (HALO: use analyze --analyzer halo)
   index     Emit a reusable session index JSON for later investigation
   inspect   Read a session index and print ranked improvement findings
   export    Convert evidence/events files to OpenInference JSONL for HALO
   import-codetracebench
             Convert CodeTracer-normalized CodeTraceBench trajectories in bulk
+  replay-verify
+            Replay a trajectory prefix in a sandbox, execute step k both recorded
+            and corrected, and emit an executed-proof verdict (--help for flags)
+  replay-verify-batch
+            Measure replayability + fix-flip rates across gold-labeled corpora:
+            arm A per replayable case, LLM-generated arm-B fixes (--help for flags)
   evidence  Emit compact session-evidence JSONL for downstream policy miners
   stream    Emit JSONL trace stream events for live visualizers or replay
   watch     Online observer: tail active sessions, notify on loops + semantic findings
@@ -1138,7 +1303,15 @@ Options:
   --since <t>      upload: window, 30m / 2h / 7d or an ISO date (default 24h); analyze: ISO cutoff
   --out <path>     Write report to a file
   --dir <path>     improve: write artifacts to this directory
-  --otlp <path>    OTLP artifact path (also evidence provenance / dry-run upload preview)
+  --otlp <file|dir>  READ OTLP-JSONL emitted by any system, skipping the adapters.
+                   A directory reads the OTLP files under it \u2014 only the otlp/
+                   subdirectory when the producer made one \u2014 and names the JSONL
+                   that is not OTLP instead of reading it as broken spans.
+                   Supported by: validate, analyze, investigate, improve, stream.
+                   On a WRITING command it is the deprecated spelling of
+                   --otlp-out; it still works, with a warning, until 0.12.
+  --otlp-out <path>  WRITE the OTLP-JSONL artifact here (also evidence
+                   provenance / dry-run upload preview)
   --format <kind>  analyze/export: auto | policy-evidence | sandbox-events | openinference | intelligence-spans | chat-trajectory
   --metadata <json> analyze/export file: attach JSON object fields as span attributes
   --attr <k=v>     analyze/export file: attach one span attribute (repeatable)
@@ -1166,7 +1339,8 @@ Options:
   --redactor <cmd> upload: external PII scrubber (JSON array stdin\u2192stdout) after the regex pass
   --yes, -y        upload: skip the confirmation prompt
   --version, -v    Print the installed traces version
-  --help, -h       Show help (use \`traces export --help\` or \`traces import-codetracebench --help\`)
+  --help, -h       Show help (use \`traces validate --help\`, \`traces export --help\`,
+                   or \`traces import-codetracebench --help\`)
 
 Watch a run tree (any depth; driver spend and child spend stay separate,
 and a cost the harness never priced reads "unknown", never $0):
@@ -1183,18 +1357,28 @@ async function main() {
     console.log(`traces ${packageVersion()}`);
     return;
   }
+  if (rawArgs[0] === "replay-verify") {
+    await cmdReplayVerify(rawArgs.slice(1));
+    return;
+  }
+  if (rawArgs[0] === "replay-verify-batch") {
+    await cmdReplayVerifyBatch(rawArgs.slice(1));
+    return;
+  }
   const parsedArgs = parseArgs(rawArgs);
   if (parsedArgs.help) {
     if (parsedArgs.command === "import-codetracebench" || parsedArgs.command === "help" && parsedArgs.input === "import-codetracebench") usageImportCodeTraceBench();
     else if (parsedArgs.command === "export" || parsedArgs.command === "help" && parsedArgs.input === "export") usageExport();
+    else if (parsedArgs.command === "validate" || parsedArgs.command === "help" && parsedArgs.input === "validate") usageValidate();
     else usage();
     return;
   }
-  const args = validateWorkflowSelection(applyCurrentSessionSelection(parsedArgs));
+  const args = validateOtlpSelection(validateWorkflowSelection(applyCurrentSessionSelection(parsedArgs)));
   switch (args.command) {
     case "help":
       if (args.input === "import-codetracebench") usageImportCodeTraceBench();
       else if (args.input === "export") usageExport();
+      else if (args.input === "validate") usageValidate();
       else usage();
       break;
     case "list":
@@ -1203,6 +1387,9 @@ async function main() {
     case "analyze":
       await cmdAnalyze(args);
       break;
+    case "validate":
+      await cmdValidate(args);
+      break;
     case "investigate":
       await cmdInvestigate(args);
       break;