@tangle-network/traces 0.23.0 → 1.0.0

package/examples/external-engines.ts

@@ -1,62 +0,0 @@
-/**
- * Run trace analyzers and PII scrubbers that traces does NOT bundle. You install
- * the tool, traces drives it. Same pattern for any future engine/model.
- *
- *   pnpm tsx examples/external-engines.ts
- */
-import {
-  applyRedactor,
-  collectSessions,
-  commandRedactor,
-  haloAnalyzer,
-  hodoscopeAnalyzer,
-  primeAnalyzer,
-  writeOtlpFile,
-} from '@tangle-network/traces'
-
-const [batch] = await collectSessions({ harnesses: ['claude-code'], last: 1 })
-if (!batch) {
-  console.log('no claude-code sessions found')
-  process.exit(0)
-}
-
-// 1) External ANALYZER: emit the OTLP artifact, then run HALO over it as a peer
-//    to the built-in analysts. (Install HALO: github.com/context-labs/halo)
-const otlp = await writeOtlpFile(batch.spans, '/tmp/session.otlp.jsonl')
-const halo = haloAnalyzer({
-  defaultPrompt: 'diagnose stuck loops and wasted tokens',
-  maxDepth: 0,
-  maxTurns: 3,
-  maxParallel: 1,
-  reasoningEffort: 'low',
-})
-const analysis = await halo.analyze(otlp)
-console.log(analysis.ok ? analysis.output : `halo unavailable: ${analysis.error}`)
-
-// 2) DISCOVERY ENGINE: sample distinct behavior for human review.
-//    Hodoscope requires Python 3.11, uv, and model credentials.
-const discovery = await hodoscopeAnalyzer().analyze(otlp)
-console.log(
-  discovery.ok
-    ? `${discovery.candidates?.length ?? 0} behavior candidates need review`
-    : `hodoscope unavailable: ${discovery.error}`,
-)
-
-// 3) PRIME ENGINE: one-shot RLM through an OpenAI-compatible bridge (run
-//    cli-bridge's prime backend locally, or set TRACES_PRIME_BRIDGE_URL).
-//    The full span projection travels inline; findings come back with
-//    validated trace:// span evidence.
-const prime = primeAnalyzer({ defaultPrompt: 'find unsupported completion claims' })
-const primeResult = await prime.analyze(otlp)
-console.log(
-  primeResult.ok
-    ? `${primeResult.findings?.length ?? 0} prime finding(s)\n${primeResult.output}`
-    : `prime bridge unavailable: ${primeResult.error}`,
-)
-
-// 4) External REDACTOR: scrub prose with your own PII model before upload. The
-//    command reads a JSON array of strings on stdin and writes the scrubbed array
-//    on stdout (a 3-line wrapper adapts openai/privacy-filter's `opf`).
-const redactor = commandRedactor({ command: 'my-pii-scrubber' })
-const { spans, changed } = await applyRedactor(batch.spans, redactor)
-console.log(`scrubbed ${changed} content field(s) across ${spans.length} spans`)

package/examples/improvement-config.mjs

@@ -1,63 +0,0 @@
-/**
- * BYO analyst config for:
- *
- *   traces improve --last 5 --config examples/improvement-config.mjs --dir .traces/improvement
- *   traces stream --mode agent --config examples/improvement-config.mjs
- */
-import { buildDefaultAnalystRegistry, makeFinding } from '@tangle-network/traces'
-
-const registry = buildDefaultAnalystRegistry()
-
-registry.register({
-  id: 'example-error-summary',
-  description: 'Reports the most frequent error signature in the selected traces.',
-  inputKind: 'trace-store',
-  cost: { kind: 'deterministic' },
-  version: '1.0.0',
-  async analyze(store) {
-    const overview = await store.getOverview({ has_errors: true })
-    const cluster = overview.error_clusters[0]
-    if (!cluster) return []
-    return [makeFinding({
-      analyst_id: 'example-error-summary',
-      area: 'tool-use',
-      subject: cluster.signature,
-      claim: `${cluster.span_count} failed span(s) share the error: ${cluster.status_message_sample}`,
-      severity: cluster.span_count >= 3 ? 'high' : 'medium',
-      evidence_refs: [{
-        kind: 'span',
-        uri: `trace://${encodeURIComponent(cluster.exemplar_trace_ids[0])}/span/${encodeURIComponent(cluster.exemplar_span_ids[0])}`,
-        excerpt: cluster.status_message_sample,
-      }],
-      recommended_action: `Fix or change the retry policy for ${cluster.tool_name ?? cluster.span_name ?? 'the failing operation'}.`,
-      validation_plan: 'Rerun the same task and confirm this error signature is absent.',
-      confidence: 1,
-      id_basis: cluster.signature,
-    })]
-  },
-})
-
-export default {
-  registry,
-  liveAnalysts: [{
-    id: 'example-live-claim-check',
-    analyze(context) {
-      if (!context.actions.some((action) => action.kind === 'claim')) return []
-      return [{
-        schemaVersion: 1,
-        kind: 'traces.live_finding',
-        id: `live.example-live-claim-check.${context.session.sessionId}`,
-        ruleId: 'example-live-claim-check',
-        fingerprint: `example-live-claim-check.${context.session.sessionId}`,
-        severity: 'info',
-        title: 'Custom live analyst saw a completion claim',
-        claim: 'The trace contains a claim-like assistant message.',
-        action: 'Keep this custom rule narrow; require evidence before alerting.',
-        check: 'Confirm the next stream event includes any required verification signal.',
-        evidence: [{ kind: 'metric', label: 'actions', value: String(context.actions.length) }],
-        session: context.session,
-        observedAt: context.generatedAt,
-      }]
-    },
-  }],
-}

package/examples/observe-and-alert.ts

@@ -1,19 +0,0 @@
-/**
- * Live observer → alert. Tails active sessions across all harnesses and fires
- * when an agent gets stuck in a loop. Read-only; Ctrl-C to stop.
- *
- *   tsx examples/observe-and-alert.ts
- */
-import { watchSessions } from '@tangle-network/traces'
-
-const controller = new AbortController()
-process.on('SIGINT', () => controller.abort())
-
-await watchSessions({
-  all: true,
-  signal: controller.signal,
-  onLoop: (loop) => {
-    // Swap console.log for Slack / PagerDuty / your alerter.
-    console.log(`[${loop.harness}] ${loop.sessionId.slice(0, 8)} stuck: ${loop.toolName} ×${loop.occurrences}`)
-  },
-})

package/examples/register-harness.ts

@@ -1,26 +0,0 @@
-/**
- * Add support for a new harness: implement HarnessTraceAdapter (locate + parse),
- * then use it via the `adapters` option. No registry edit and no fork.
- *
- *   tsx examples/register-harness.ts
- */
-import { collectSessions, type HarnessTraceAdapter, span } from '@tangle-network/traces'
-
-const myAdapter: HarnessTraceAdapter = {
-  harness: 'my-agent',
-  async locate() {
-    // Discover this harness's session files on disk and return refs.
-    return [{ harness: 'my-agent', sessionId: 'demo', path: '/tmp/demo', cwd: null, mtimeMs: Date.now() }]
-  },
-  async parse(ref) {
-    // Project the harness's native log onto normalized OTLP spans.
-    const now = new Date().toISOString()
-    return [
-      span({ traceId: ref.sessionId, spanId: 'root', name: 'session', kind: 'AGENT', startTime: now }),
-      span({ traceId: ref.sessionId, spanId: 's1', parentSpanId: 'root', name: 'tool.bash', kind: 'TOOL', startTime: now, tool: 'bash', content: 'ls' }),
-    ]
-  },
-}
-
-const [batch] = await collectSessions({ adapters: [myAdapter] })
-console.log(`${batch?.spans.length ?? 0} spans from ${batch?.ref.harness}`)

package/skills/build-trace-analyst/SKILL.md

@@ -1,76 +0,0 @@
----
-name: build-trace-analyst
-description: Write and calibrate a custom agent-eval trace analyst on real and labeled workflows.
----
-
-# Build a trace analyst
-
-Implement one measurable finding that existing analysts do not provide.
-
-## Check coverage
-
-1. Run maintained analysis on one positive and one clean case.
-2. Call `buildDefaultAnalystRegistry().list()` for batch analysts.
-Check live analysts separately only for `traces stream`.
-3. Search source or public docs once for the target behavior, then decide.
-
-Extend an existing analyst when it already emits the target with usable evidence.
-Do not inspect bundled `dist` or `node_modules`.
-
-Answer a one-off question with `traces ask --question "<text>"` instead.
-Build an analyst when the question repeats and its answer must be scored, not read.
-
-## Implement
-
-- Use `Analyst`, `AnalystRegistry`, `TraceAnalysisStore`, and `makeFinding` from `@tangle-network/traces`.
-- Start with `getOverview`; narrow with `queryTraces`, `viewSpans`, or `searchTrace`.
-- Pass `context.signal` through reads and model calls.
-- Return stable `analyst_id`, `area`, `subject`, and `id_basis` values.
-- Cite exact spans as `trace://<trace-id>/span/<span-id>`.
-- Return no finding when evidence is absent.
-- Use `cost.kind: 'deterministic'` only without model calls.
-- For `cost.kind: 'llm'`, use the injected chat client, budget, cancellation, and usage recording.
-- Extend `buildDefaultAnalystRegistry()` unless full replacement is intentional.
-
-Use the [custom analyst](../../examples/custom-analyst.ts) and [config](../../examples/improvement-config.mjs) examples rather than adding a wrapper.
-
-## Run
-
-```bash
-pnpm tsx examples/custom-analyst.ts --fixture
-pnpm tsx examples/custom-analyst.ts --good-fixture
-traces improve --harness codex --current --latest-turn --workflow \
-  --config traces.config.mjs --dir .traces/custom-analyst
-```
-
-Inspect every finding and cited span.
-
-## Calibrate
-
-Use `runAnalystBenchmark` with `registryBenchmarkRunner` from `@tangle-network/agent-eval/analyst`.
-Use AgentRx or CodeTraceBench labels when they match the finding.
-Compare the candidate with a no-findings baseline and relevant maintained or external analysts on identical cases and limits.
-
-Two shipped engines are available as comparison arms.
-`--analyzer prime` inlines the whole span projection into one prompt and needs no REPL.
-`--analyzer halo` and `--analyzer hodoscope` drill into the trace with paged tools.
-Compare an inline arm with a paged arm to separate the prompt from the navigation.
-
-```bash
-traces analyze --last 1 --analyzer prime
-```
-
-Assert the served model, not the requested id.
-A gateway can answer one id with another model, so a panel can report families it never ran.
-Use `assertServedModel` and `assertCrossFamilyServed` from `@tangle-network/agent-eval` before you trust a per-model or cross-family number.
-
-Use at least 20 independent cases before comparative statistics.
-Retain every case row and report exact-step recall, precision, F1, root-step accuracy when labeled, citation coverage and validity, clean-case false positives, repeat agreement, failures, latency, calls, all token fields, and known or missing cost.
-
-Do not claim quality from fixtures alone.
-Require a fresh task comparison before using findings to change an agent or knowledge base.
-
-## Then consider
-
-- Use `inspect-agent-traces` to investigate a miss or false positive on its original workflow.
-- Use `calibrate-before-measure` before trusting a new label set or scoring rule.

package/skills/build-trace-analyst/agents/openai.yaml

@@ -1,4 +0,0 @@
-interface:
-  display_name: "Build Trace Analyst"
-  short_description: "Create and test a custom trace analyst"
-  default_prompt: "Use $build-trace-analyst to create, run, and evaluate an analyst for this trace question."