@tangle-network/traces 0.9.16 → 0.9.18

package/README.md

@@ -23,6 +23,7 @@ It reads the transcripts your harness leaves on disk, reconstructs the run as sp
 - [Policy-mining evidence](#policy-mining-evidence)
 - [Upload to the Intelligence Platform](#upload-to-the-intelligence-platform)
 - [Trace analysts](#trace-analysts)
+- [Agent skills](#agent-skills)
 - [Library (SDK)](#library-sdk)
 - [Examples](#examples)
 - [Develop](#develop)
@@ -373,6 +374,32 @@ See [`examples/external-engines.ts`](./examples/external-engines.ts).
 
 > For the built-in agentic analysts (`--llm`), set `OPENAI_API_KEY`, or point at any OpenAI-compatible gateway with `OPENAI_BASE_URL` (e.g. an internal router) to use a non-OpenAI key.
 
+## Agent skills
+
+The npm package ships two AgentProfile-ready skills:
+
+| Skill | Use |
+|---|---|
+| [`inspect-agent-traces`](./skills/inspect-agent-traces/SKILL.md) | select workflows and export deterministic findings |
+| [`build-trace-analyst`](./skills/build-trace-analyst/SKILL.md) | write and calibrate a custom analyst |
+
+Load both from GitHub:
+
+```ts
+import { defineGitHubResource } from '@tangle-network/agent-interface'
+
+const ref = process.env.TRACES_SKILLS_REF
+if (!ref) throw new Error('TRACES_SKILLS_REF must be an immutable commit')
+
+const names = ['inspect-agent-traces', 'build-trace-analyst']
+const skills = names.map((name) => defineGitHubResource(`skills/${name}/SKILL.md`, {
+  repository: 'tangle-network/traces',
+  ref,
+  name,
+}))
+const profile = { name: 'trace-reviewer', resources: { skills } }
+```
+
 ## Release automation
 
 Merging to `main` publishes a patch release automatically:

package/dist/{chunk-I4W6DECY.js → chunk-F2OEMHVZ.js}

@@ -8975,7 +8975,58 @@ function traceCoverageFinding(pipelines, adoption) {
     }
   });
 }
-function deterministicFindings(pipelines, reactions, adoption) {
+function sortedToolErrorSpans(spans) {
+  return spans.filter((item) => item.attributes["openinference.span.kind"] === "TOOL" && item.status.code === "ERROR").sort((left, right) => left.start_time.localeCompare(right.start_time) || left.trace_id.localeCompare(right.trace_id) || left.span_id.localeCompare(right.span_id));
+}
+function toolErrorEvidence(item) {
+  return evidence(
+    "span",
+    spanEvidenceUri(item.trace_id, item.span_id),
+    `${String(item.attributes["tool.name"] ?? item.name)}: ${item.status.message ?? "tool span ended with ERROR"}`.slice(0, 240)
+  );
+}
+function toolErrorFinding(pipelines, spans) {
+  const terminalFailureTraceIds = new Set(
+    spans.filter((item) => item.parent_span_id === null && item.status.code === "ERROR").map((item) => item.trace_id)
+  );
+  const nonTerminalRuns = pipelines.toolUse.filter(
+    (run) => !terminalFailureTraceIds.has(run.runId)
+  );
+  const totalCalls = nonTerminalRuns.reduce((total, run) => total + run.totalCalls, 0);
+  const errorCount = nonTerminalRuns.reduce(
+    (total, run) => total + Object.values(run.byTool).reduce((runTotal, tool) => runTotal + tool.errors, 0),
+    0
+  );
+  if (errorCount === 0) return void 0;
+  const errorSpans = sortedToolErrorSpans(spans).filter((item) => !terminalFailureTraceIds.has(item.trace_id));
+  if (errorSpans.length === 0) return void 0;
+  const affectedRuns = new Set(errorSpans.map((item) => item.trace_id)).size;
+  const errorRate = errorCount / Math.max(1, totalCalls);
+  const citedSpans = errorSpans.slice(0, 3);
+  return makeFinding({
+    analyst_id: "traces-deterministic",
+    area: "tool-use",
+    subject: "non-terminal-tool-errors",
+    claim: `${errorCount}/${totalCalls} tool call(s) ended in errors across ${affectedRuns} run(s) without a terminal failure`,
+    severity: errorRate >= 0.25 ? "high" : "medium",
+    rationale: "Recovered tool errors still consume time and tokens, and repeated failures can hide brittle retry or recovery behavior.",
+    evidence_refs: citedSpans.map(toolErrorEvidence),
+    recommended_action: "Inspect the most frequent failing tool and its cited inputs, then change the retry or recovery policy so a failure produces new state before another call.",
+    validation_plan: "Rerun comparable sessions and require non-terminal tool errors to decrease without increasing terminal failures.",
+    confidence: 1,
+    metadata: {
+      source: "agent-eval.computeToolUseMetrics",
+      errorCount,
+      totalCalls,
+      errorRate,
+      affectedRuns,
+      citedErrorSpans: citedSpans.length,
+      omittedErrorSpans: Math.max(0, errorSpans.length - citedSpans.length)
+    },
+    id_basis: "non-terminal-tool-errors"
+  });
+}
+function deterministicFindings(pipelines, reactions, adoption, spans) {
   const findings = [];
   const coverage = traceCoverageFinding(pipelines, adoption);
   if (coverage) findings.push(coverage);
@@ -8998,10 +9049,16 @@ function deterministicFindings(pipelines, reactions, adoption) {
       metadata: { source: "traces.pipeline.stuckLoopView", loopCount: loops.length }
     }));
   }
+  const toolErrors = toolErrorFinding(pipelines, spans);
+  if (toolErrors) findings.push(toolErrors);
   const failedRuns = pipelines.failureClusters.totalFailures;
   if (failedRuns > 0) {
     const top = pipelines.failureClusters.clusters[0];
     const failureRate = failedRuns / Math.max(1, pipelines.failureClusters.totalRuns);
+    const exampleSpans = top?.exampleRunId ? spans.filter((item) => item.trace_id === top.exampleRunId) : [];
+    const exampleRoot = exampleSpans.find((item) => item.parent_span_id === null && item.status.code === "ERROR");
+    const exampleToolErrors = sortedToolErrorSpans(exampleSpans);
+    const citedToolErrors = exampleToolErrors.slice(0, 3);
     findings.push(makeFinding({
       analyst_id: "traces-deterministic",
       area: "reliability",
@@ -9009,12 +9066,17 @@ function deterministicFindings(pipelines, reactions, adoption) {
       severity: failureRate >= 0.1 ? "high" : "medium",
       rationale: "Execution errors consume time and tokens, and sparse error telemetry can hide whether the agent recovered or repeated the same failure.",
       evidence_refs: [
+        ...exampleRoot ? [evidence(
+          "span",
+          spanEvidenceUri(exampleRoot.trace_id, exampleRoot.span_id),
+          `${exampleRoot.name}: ${exampleRoot.status.message ?? "root span ended with ERROR"}`.slice(0, 240)
+        )] : [],
+        ...citedToolErrors.map(toolErrorEvidence),
         evidence(
           "metric",
           `pipelines.failure_cluster.${top?.failureClass ?? "unknown"}`,
           top ? `${top.runCount} run(s); example: ${top.exampleError ?? "error details not captured"}` : `${failedRuns} failed run(s)`
-        ),
-        ...top?.exampleRunId ? [evidence("span", `trace:${top.exampleRunId}`, top.exampleError)] : []
+        )
       ],
       recommended_action: top?.failureClass === "unknown" ? "Instrument the failing operation name and arguments, then start with the highest-frequency example error and verify the agent changes state before retrying." : "Fix the highest-frequency failure cluster first, then require the agent to inspect the error and change state before retrying.",
       validation_plan: "Rerun a comparable trace window and require failed runs and repeated failures in the top cluster to decrease.",
@@ -9024,7 +9086,9 @@ function deterministicFindings(pipelines, reactions, adoption) {
         failedRuns,
         totalRuns: pipelines.failureClusters.totalRuns,
         topFailureClass: top?.failureClass,
-        exampleRunId: top?.exampleRunId
+        exampleRunId: top?.exampleRunId,
+        citedToolErrorSpans: citedToolErrors.length,
+        omittedToolErrorSpans: Math.max(0, exampleToolErrors.length - citedToolErrors.length)
       }
     }));
   }
@@ -9212,7 +9276,7 @@ async function runTraceInvestigation(opts) {
     analyzeAdoption(opts.spans, { cwds: opts.cwds })
   ]);
   opts.signal?.throwIfAborted();
-  const deterministic = deterministicFindings(pipelines, reactions, adoption);
+  const deterministic = deterministicFindings(pipelines, reactions, adoption, opts.spans);
   const agenticRoute = opts.ai && !opts.agenticRegistry ? planTraceAgenticRoute(pipelines, reactions) : void 0;
   const analysis = await analyzeSpans(opts.spans, {
     ai: opts.ai,

package/dist/cli.js

@@ -43,7 +43,7 @@ import {
   writePolicyEvidenceFile,
   writeSessionIndexFile,
   writeTraceEvidenceExportFile
-} from "./chunk-I4W6DECY.js";
+} from "./chunk-F2OEMHVZ.js";
 
 // src/cli.ts
 import { readFileSync } from "fs";

package/dist/index.js

@@ -130,7 +130,7 @@ import {
   writeSessionIndexFile,
   writeTraceEvidenceExportFile,
   writeTraceImprovementArtifacts
-} from "./chunk-I4W6DECY.js";
+} from "./chunk-F2OEMHVZ.js";
 
 // src/collect.ts
 async function collectSessions(opts = {}) {

package/docs/trace-analysts.md

@@ -175,10 +175,11 @@ It must also avoid inventing issues on clean trajectories.
 | Dataset | Labels used |
 | --- | --- |
 | AgentRx | Failure category, failed step, and root-cause step |
-| CodeTraceBench | Incorrect and unhelpful action steps, including clean trajectories |
+| CodeTraceBench | Incorrect and unhelpful action steps; solved label-empty rows are clean controls |
 
 Use `runAnalystBenchmark` to report recall, precision, F1, root-step accuracy, citation coverage, citation validity, clean-case false positives, repeat agreement, latency, calls, tokens, and known cost.
 Use `compareAnalystRunners` for paired baseline and candidate comparisons.
+Failed label-empty CodeTraceBench rows are unlabeled, not clean controls.
 
 ```ts
 import {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/traces",
-  "version": "0.9.16",
+  "version": "0.9.18",
   "description": "Point it at your coding-agent session traces (Claude Code, Codex, OpenCode, Gemini, Pi, …) and get failure-mode + efficiency findings. CLI + SDK over the @tangle-network/agent-eval analyst suite — observe live sessions, run your own analysts, redact, and upload to the Tangle Intelligence Platform.",
   "type": "module",
   "license": "MIT",
@@ -29,7 +29,8 @@
       "types": "./dist/index.d.ts",
       "import": "./dist/index.js"
     },
-    "./package.json": "./package.json"
+    "./package.json": "./package.json",
+    "./skills/*": "./skills/*"
   },
   "bin": {
     "traces": "dist/cli.js"
@@ -38,6 +39,7 @@
     "dist",
     "docs",
     "examples",
+    "skills",
     "install.sh",
     "README.md"
   ],

package/skills/build-trace-analyst/SKILL.md

@@ -0,0 +1,60 @@
+---
+name: build-trace-analyst
+description: Write and calibrate a custom agent-eval trace analyst on real and labeled workflows.
+---
+
+# Build a trace analyst
+
+Implement one measurable finding that existing analysts do not provide.
+
+## Check coverage
+
+1. Run maintained analysis on one positive and one clean case.
+2. Call `buildDefaultAnalystRegistry().list()` for batch analysts.
+Check live analysts separately only for `traces stream`.
+3. Search source or public docs once for the target behavior, then decide.
+
+Extend an existing analyst when it already emits the target with usable evidence.
+Do not inspect bundled `dist` or `node_modules`.
+
+## Implement
+
+- Use `Analyst`, `AnalystRegistry`, `TraceAnalysisStore`, and `makeFinding` from `@tangle-network/traces`.
+- Start with `getOverview`; narrow with `queryTraces`, `viewSpans`, or `searchTrace`.
+- Pass `context.signal` through reads and model calls.
+- Return stable `analyst_id`, `area`, `subject`, and `id_basis` values.
+- Cite exact spans as `trace://<trace-id>/span/<span-id>`.
+- Return no finding when evidence is absent.
+- Use `cost.kind: 'deterministic'` only without model calls.
+- For `cost.kind: 'llm'`, use the injected chat client, budget, cancellation, and usage recording.
+- Extend `buildDefaultAnalystRegistry()` unless full replacement is intentional.
+
+Use the [custom analyst](../../examples/custom-analyst.ts) and [config](../../examples/improvement-config.mjs) examples rather than adding a wrapper.
+
+## Run
+
+```bash
+pnpm tsx examples/custom-analyst.ts --fixture
+pnpm tsx examples/custom-analyst.ts --good-fixture
+traces improve --harness codex --current --latest-turn --workflow \
+  --config traces.config.mjs --dir .traces/custom-analyst
+```
+
+Inspect every finding and cited span.
+
+## Calibrate
+
+Use `runAnalystBenchmark` with `registryBenchmarkRunner` from `@tangle-network/agent-eval/analyst`.
+Use AgentRx or CodeTraceBench labels when they match the finding.
+Compare the candidate with a no-findings baseline and relevant maintained or external analysts on identical cases and limits.
+
+Use at least 20 independent cases before comparative statistics.
+Retain every case row and report exact-step recall, precision, F1, root-step accuracy when labeled, citation coverage and validity, clean-case false positives, repeat agreement, failures, latency, calls, all token fields, and known or missing cost.
+
+Do not claim quality from fixtures alone.
+Require a fresh task comparison before using findings to change an agent or knowledge base.
+
+## Then consider
+
+- Use `inspect-agent-traces` to investigate a miss or false positive on its original workflow.
+- Use `calibrate-before-measure` before trusting a new label set or scoring rule.

package/skills/build-trace-analyst/agents/openai.yaml

@@ -0,0 +1,4 @@
+interface:
+  display_name: "Build Trace Analyst"
+  short_description: "Create and test a custom trace analyst"
+  default_prompt: "Use $build-trace-analyst to create, run, and evaluate an analyst for this trace question."

package/skills/inspect-agent-traces/SKILL.md

@@ -0,0 +1,60 @@
+---
+name: inspect-agent-traces
+description: Inspect real agent workflows with the published Traces CLI and export cited local findings.
+---
+
+# Inspect agent traces
+
+Use the deterministic CLI first.
+Keep inspection local and read-only.
+
+## Select the workflow
+
+```bash
+traces --version
+traces list --harness codex --cwd "$PWD"
+traces analyze --harness codex --current --latest-turn --workflow \
+  --out .traces/current.md
+```
+
+- Use `--current` for the active Codex session.
+- Use `--session <id-or-path>` to pin a listed session.
+- Use `--latest-turn` for the current task in a resumed Codex or Claude Code session.
+- Use `--workflow` to include workers linked by stable parent and child IDs.
+- Use `--max-workflow-sessions <n>` only when the default 100-file bound is insufficient.
+- For Claude Code, use `--harness claude-code --session <path> --latest-turn`; nested subagents are included.
+
+Never join agents by display name or timestamp when Traces reports missing or conflicting IDs.
+
+## Export
+
+Write normalized OpenInference spans for another tool:
+
+```bash
+traces convert --harness codex --current --latest-turn --workflow \
+  --otlp .traces/current.otlp.jsonl
+```
+
+Write the complete deterministic review packet:
+
+```bash
+traces improve --harness codex --current --latest-turn --workflow \
+  --dir .traces/improvement
+```
+
+`improve` writes findings, evidence, a report, and spans.
+It does not edit an agent, repository, memory store, or knowledge base.
+
+## Report
+
+- State the harness, selected task boundary, session and span counts, and integrity warnings.
+- Cite each finding with its exact `trace://` reference.
+- Mark missing outcome, cost, token, skill, or relationship data as unknown.
+- Do not infer task success from a completion message.
+- Do not infer skill use from reading a `SKILL.md`.
+- Do not upload traces unless explicitly requested.
+
+## Then consider
+
+- Use `build-trace-analyst` when deterministic findings cannot answer a repeated trace question.
+- Use `autopsy` when the result is empty, surprising, or suspiciously strong.

package/skills/inspect-agent-traces/agents/openai.yaml

@@ -0,0 +1,4 @@
+interface:
+  display_name: "Inspect Agent Traces"
+  short_description: "Inspect real workflows with Traces"
+  default_prompt: "Use $inspect-agent-traces to inspect the latest agent workflow and report cited findings."