@tangle-network/traces 0.9.11 → 0.9.13

package/dist/index.js

@@ -9,6 +9,7 @@ import {
   CodexAdapter,
   CodexExecAdapter,
   CodexExecStreamError,
+  CodexTaskScopeError,
   CopilotAdapter,
   DEFAULT_HARNESS,
   EmptySessionError,
@@ -23,6 +24,7 @@ import {
   PiAdapter,
   QwenAdapter,
   REACTIONS,
+  SessionWorkflowError,
   TRACES_REDACTION_RULES,
   alreadyUploaded,
   analyzeAdoption,
@@ -35,25 +37,32 @@ import {
   buildSessionIndexFromRows,
   buildTraceFindingPacket,
   capText,
+  chatTrajectoryToSpans,
   classifyLiveActions,
   classifyReaction,
   claudeActor,
   codexActor,
   collectPolicyEvidence,
   collectSessionIndex,
+  collectSessionSelection,
+  collectSessionWorkflow,
   commandAnalyzer,
   commandRedactor,
   countSkillRunsJsonl,
   cwdMatchesSelection,
   defaultTraceLiveAnalysts,
+  describeSessionRelationship,
   equivalentGitCwds,
   executeUpload,
   exportTraceEvidenceFile,
   exportTraceEvidenceRows,
   exportTraceEvidenceText,
   haloAnalyzer,
+  hodoscopeAnalyzer,
   indexSessionIdsByTrace,
   inspectSessionIndex,
+  isChatTrajectory,
+  isChatTrajectoryMessage,
   knownHarnesses,
   listAdapters,
   loadState,
@@ -112,14 +121,16 @@ import {
   traceStreamEventsFromSpans,
   uploadKey,
   userPromptSpan,
+  validateOtlpSpans,
   watchSessions,
+  writeHodoscopeInput,
   writeInspectionReportFile,
   writeOtlpFile,
   writePolicyEvidenceFile,
   writeSessionIndexFile,
   writeTraceEvidenceExportFile,
   writeTraceImprovementArtifacts
-} from "./chunk-ELWOJS65.js";
+} from "./chunk-H47X6QWV.js";
 
 // src/collect.ts
 async function collectSessions(opts = {}) {
@@ -155,6 +166,7 @@ export {
   CodexAdapter,
   CodexExecAdapter,
   CodexExecStreamError,
+  CodexTaskScopeError,
   CopilotAdapter,
   DEFAULT_HARNESS,
   DEFAULT_REDACTION_RULES,
@@ -170,6 +182,7 @@ export {
   PiAdapter,
   QwenAdapter,
   REACTIONS,
+  SessionWorkflowError,
   TRACES_REDACTION_RULES,
   alreadyUploaded,
   analyzeAdoption,
@@ -183,12 +196,15 @@ export {
   buildSessionIndexFromRows,
   buildTraceFindingPacket,
   capText,
+  chatTrajectoryToSpans,
   classifyLiveActions,
   classifyReaction,
   claudeActor,
   codexActor,
   collectPolicyEvidence,
   collectSessionIndex,
+  collectSessionSelection,
+  collectSessionWorkflow,
   collectSessions,
   commandAnalyzer,
   commandRedactor,
@@ -196,15 +212,19 @@ export {
   createHostedClient,
   cwdMatchesSelection,
   defaultTraceLiveAnalysts,
+  describeSessionRelationship,
   equivalentGitCwds,
   executeUpload,
   exportTraceEvidenceFile,
   exportTraceEvidenceRows,
   exportTraceEvidenceText,
   haloAnalyzer,
+  hodoscopeAnalyzer,
   hostedClientFromEnv,
   indexSessionIdsByTrace,
   inspectSessionIndex,
+  isChatTrajectory,
+  isChatTrajectoryMessage,
   knownHarnesses,
   listAdapters,
   loadState,
@@ -266,7 +286,9 @@ export {
   traceStreamEventsFromSpans,
   uploadKey,
   userPromptSpan,
+  validateOtlpSpans,
   watchSessions,
+  writeHodoscopeInput,
   writeInspectionReportFile,
   writeOtlpFile,
   writePolicyEvidenceFile,

package/docs/trace-analysts.md

@@ -0,0 +1,222 @@
+# Trace analysts
+
+Use `traces` to answer three different questions.
+Keeping them separate prevents an exploratory model output from being reported as a confirmed defect.
+
+| Question | Tool | Output |
+| --- | --- | --- |
+| What happened? | Built-in local checks | Findings from explicit trace facts |
+| Why might it have happened? | `--llm`, HALO, or a custom analyst | Findings or a diagnosis report with cited spans |
+| What behavior should we inspect? | Hodoscope | Samples marked `needs_review` |
+
+## Start here
+
+```bash
+# Free local checks over the active Codex coordinator and connected workers.
+traces analyze --harness codex --current --latest-turn --workflow
+
+# Limit a resumed Claude Code session to its latest task and subagents.
+traces analyze --harness claude-code --session <path> --latest-turn
+
+# Write reusable findings, evidence, report, and OpenInference spans.
+traces improve --harness codex --last 5 --dir .traces/improvement
+
+# Add the built-in model-assisted analysts with a spending limit.
+traces analyze --harness codex --last 5 --llm --budget 0.50
+```
+
+Without `--workflow`, each selected Codex file is analyzed independently.
+With it, `traces` follows stable parent and child session IDs, up to 100 files by default.
+`--latest-turn` keeps a long-lived resumed Codex or Claude Code session scoped to its most recent task and the workers spawned from that task.
+For Codex, when the selected session is a child, `traces` finds the last parent task that structurally spawned or targeted that stable child ID and reads only that task.
+The Codex parent log must carry both the child ID and a stable turn ID on the relevant spawn, send, follow-up, or lifecycle event.
+Without those fields, the result is explicitly incomplete; names, nicknames, and timestamps are never used as substitutes.
+It reports missing files, duplicate IDs, contradictory parents, and cycles instead of guessing.
+Use `--max-workflow-sessions <n>` to set a different bound.
+Claude Code already folds nested subagent files into its parent trace.
+Ordinary Claude subagents use their parent tool call ID.
+Claude Workflow subagents use the run ID and transcript directory returned by the parent `Workflow` call.
+If the same Workflow run is resumed, each child attaches to the latest matching call that started before it.
+The selected task includes only children attached to its calls.
+Returned directories from another resumed Claude session are parsed and included in source hashes.
+
+`traces improve` writes:
+
+```text
+.traces/improvement/
+  evidence.jsonl
+  findings.json
+  report.md
+  result.json
+  traces.otlp.jsonl
+```
+
+The OpenInference file is the shared input for external engines.
+The original trace and exact cited span remain available for review.
+
+## External engines
+
+External engines are optional tools that you install separately.
+One engine failing does not discard the other results.
+
+```bash
+# Recursive diagnosis over the same OpenInference trace.
+traces analyze --last 1 --analyzer halo --analyzer-prompt "find unsupported completion claims"
+
+# Sample distinct actions across a larger set of sessions.
+traces analyze --all --last 20 --analyzer hodoscope
+
+# Run any installed command that accepts an OpenInference file path.
+traces analyze --last 1 --analyzer my-trace-tool
+```
+
+`--model` is forwarded to the built-in model-assisted analysts, HALO, and Hodoscope.
+HALO and Hodoscope use their own provider clients and credentials.
+The Hodoscope adapter pins version `0.2.4` and uses Python 3.11 through `uvx`.
+
+External results have one of three explicit kinds:
+
+```ts
+type ExternalAnalysisResult =
+  | { kind: 'report'; output: string }
+  | { kind: 'findings'; findings: AnalystFinding[] }
+  | { kind: 'discovery'; candidates: ExternalDiscoveryCandidate[] }
+```
+
+Raw JSON is still a `report`.
+Only an adapter that validates the full finding shape may return `findings`.
+Hodoscope always returns `discovery`, and each candidate has `status: 'needs_review'` plus its source trace and span.
+
+## Write one analyst
+
+An analyst receives a paged trace store and returns typed findings.
+Start with deterministic logic when the trace contains enough facts.
+
+```ts
+import {
+  buildDefaultAnalystRegistry,
+  makeFinding,
+} from '@tangle-network/traces'
+import { defineTraceAnalyst } from '@tangle-network/agent-eval/analyst'
+
+const failedTools = defineTraceAnalyst({
+  id: 'failed-tools',
+  description: 'Reports repeated tool failure signatures.',
+  async analyze(store) {
+    const overview = await store.getOverview({ has_errors: true })
+    return overview.error_clusters.map((cluster) => makeFinding({
+      analyst_id: 'failed-tools',
+      area: 'tool-use',
+      subject: cluster.signature,
+      claim: `${cluster.span_count} spans share this failure`,
+      severity: cluster.span_count >= 3 ? 'high' : 'medium',
+      evidence_refs: [{
+        kind: 'span',
+        uri: `trace://${cluster.exemplar_trace_ids[0]}/span/${cluster.exemplar_span_ids[0]}`,
+      }],
+      recommended_action: 'Fix the operation or change its retry policy.',
+      validation_plan: 'Rerun the task and confirm the signature is absent.',
+      confidence: 1,
+      id_basis: cluster.signature,
+    }))
+  },
+})
+
+const registry = buildDefaultAnalystRegistry()
+registry.register(failedTools)
+
+export default { registry }
+```
+
+Run it without another wrapper:
+
+```bash
+traces improve --last 5 --config traces.config.mjs --dir .traces/improvement
+```
+
+The full runnable example is [`examples/custom-analyst.ts`](../examples/custom-analyst.ts).
+
+Run its fixed input to inspect the complete JSON:
+
+```bash
+pnpm tsx examples/custom-analyst.ts --fixture
+```
+
+The current fixture returns one finding.
+Its stable fields are:
+
+```json
+{
+  "analyst_id": "failed-tool-clusters",
+  "area": "tool-use",
+  "claim": "3 failed span(s) share the error: Command failed with exit code 127 on attempt 1",
+  "severity": "high",
+  "evidence_refs": [
+    {
+      "kind": "span",
+      "uri": "trace://fixture-three-failures/span/failed-1"
+    }
+  ],
+  "recommended_action": "Fix or change the retry policy for exec.",
+  "validation_plan": "Rerun the same task and confirm this error signature is absent.",
+  "confidence": 1
+}
+```
+
+`pnpm tsx examples/custom-analyst.ts --good-fixture` returns `[]`.
+
+## Test an analyst
+
+A useful analyst must beat a trivial no-findings baseline on labeled trajectories.
+It must also avoid inventing issues on clean trajectories.
+
+`agent-eval` provides adapters for two public datasets:
+
+| Dataset | Labels used |
+| --- | --- |
+| AgentRx | Failure category, failed step, and root-cause step |
+| CodeTraceBench | Incorrect and unhelpful action steps, including clean trajectories |
+
+Use `runAnalystBenchmark` to report recall, precision, F1, root-step accuracy, citation coverage, citation validity, clean-case false positives, repeat agreement, latency, calls, tokens, and known cost.
+Use `compareAnalystRunners` for paired baseline and candidate comparisons.
+
+```ts
+import {
+  codeTraceBenchCase,
+  registryBenchmarkRunner,
+  renderAnalystBenchmarkMarkdown,
+  runAnalystBenchmark,
+} from '@tangle-network/agent-eval/analyst'
+import { otlpTextToTraceAnalysisStore } from '@tangle-network/agent-eval/traces'
+import { chatTrajectoryToSpans, serializeSpans } from '@tangle-network/traces'
+
+const cases = rows.map(({ label, trajectory }) => {
+  const spans = chatTrajectoryToSpans(trajectory, { traceId: label.traj_id })
+  const traceStore = otlpTextToTraceAnalysisStore(serializeSpans(spans))
+  return codeTraceBenchCase(label, { traceStore })
+})
+
+const result = await runAnalystBenchmark({
+  cases,
+  runners: [registryBenchmarkRunner({ id: 'candidate', registry })],
+  repetitions: 3,
+  maxConcurrency: 4,
+})
+
+console.log(renderAnalystBenchmarkMarkdown(result))
+```
+
+Public labels test the measurement code, not the quality of every built-in analyst automatically.
+A real quality claim requires running the analyst over the corresponding trajectories, retaining all rows, and comparing it with named alternatives at equal model and request limits.
+
+## Turn findings into improvement
+
+Do not train or rewrite policy from an analyst's own prose alone.
+Attach independent feedback or a measured task outcome to the trace first.
+Then promote the reviewed failure into an eval case, make one targeted change, and compare before and after on fresh cases.
+
+The required review path is:
+
+```text
+production trace -> finding -> reviewed feedback -> eval case -> candidate change -> comparison
+```

package/examples/custom-analyst.ts

@@ -0,0 +1,179 @@
+/**
+ * Run a deterministic custom analyst over a real session, or use --fixture for
+ * a known three-error input whose output is stable enough to inspect.
+ *
+ *   pnpm tsx examples/custom-analyst.ts
+ *   pnpm tsx examples/custom-analyst.ts --fixture
+ */
+import {
+  AnalystRegistry,
+  type Analyst,
+  analyzeSpans,
+  collectSessions,
+  makeFinding,
+  type OtlpSpan,
+  span,
+  type TraceAnalysisStore,
+} from '@tangle-network/traces'
+
+export const failedToolAnalyst: Analyst<TraceAnalysisStore> = {
+  id: 'failed-tool-clusters',
+  description: 'Groups failed spans by stable error signature and cites one exact span per group.',
+  inputKind: 'trace-store',
+  cost: { kind: 'deterministic' },
+  version: '1.0.0',
+  async analyze(store) {
+    const overview = await store.getOverview({ has_errors: true })
+    return overview.error_clusters.map((cluster) => {
+      const traceId = cluster.exemplar_trace_ids[0]
+      const spanId = cluster.exemplar_span_ids[0]
+      return makeFinding({
+        analyst_id: 'failed-tool-clusters',
+        area: 'tool-use',
+        subject: cluster.signature,
+        claim: `${cluster.span_count} failed span(s) share the error: ${cluster.status_message_sample}`,
+        rationale: `The error affected ${cluster.trace_count}/${overview.errors.trace_count} trace(s) with errors.`,
+        severity: cluster.span_count >= 3 ? 'high' : 'medium',
+        evidence_refs: traceId && spanId
+          ? [{
+              kind: 'span',
+              uri: `trace://${encodeURIComponent(traceId)}/span/${encodeURIComponent(spanId)}`,
+              excerpt: cluster.status_message_sample,
+            }]
+          : [],
+        recommended_action: `Fix or change the retry policy for ${cluster.tool_name ?? cluster.span_name ?? 'the failing operation'}.`,
+        validation_plan: 'Rerun the same task and confirm this error signature is absent.',
+        confidence: 1,
+        metadata: {
+          deterministic: true,
+          signature: cluster.signature,
+          trace_count: cluster.trace_count,
+          span_count: cluster.span_count,
+        },
+        id_basis: cluster.signature,
+      })
+    })
+  },
+}
+
+export function knownBadSpans(): OtlpSpan[] {
+  const root = 'fixture-root'
+  return [
+    span({
+      traceId: 'fixture-three-failures',
+      spanId: root,
+      name: 'session',
+      kind: 'AGENT',
+      startTime: '2026-01-01T00:00:00.000Z',
+      service: 'example',
+    }),
+    ...[1, 2, 3].map((attempt) =>
+      span({
+        traceId: 'fixture-three-failures',
+        spanId: `failed-${attempt}`,
+        parentSpanId: root,
+        name: 'tool.exec',
+        kind: 'TOOL',
+        startTime: `2026-01-01T00:00:0${attempt}.000Z`,
+        endTime: `2026-01-01T00:00:0${attempt}.100Z`,
+        status: 'ERROR',
+        statusMessage: `Command failed with exit code 127 on attempt ${attempt}`,
+        service: 'example',
+        tool: 'exec',
+        step: attempt,
+        extra: { 'input.value': JSON.stringify({ command: 'missing-command' }) },
+      }),
+    ),
+    span({
+      traceId: 'fixture-three-failures',
+      spanId: 'unsupported-claim',
+      parentSpanId: root,
+      name: 'message.assistant',
+      kind: 'CHAIN',
+      startTime: '2026-01-01T00:00:04.000Z',
+      service: 'example',
+      step: 4,
+      content: 'Implemented successfully. Everything works.',
+    }),
+  ]
+}
+
+export function knownGoodSpans(): OtlpSpan[] {
+  const root = 'fixture-root'
+  return [
+    span({
+      traceId: 'fixture-verified-change',
+      spanId: root,
+      name: 'session',
+      kind: 'AGENT',
+      startTime: '2026-01-01T00:00:00.000Z',
+      service: 'example',
+    }),
+    span({
+      traceId: 'fixture-verified-change',
+      spanId: 'edit-1',
+      parentSpanId: root,
+      name: 'tool.apply_patch',
+      kind: 'TOOL',
+      startTime: '2026-01-01T00:00:01.000Z',
+      endTime: '2026-01-01T00:00:01.100Z',
+      service: 'example',
+      tool: 'apply_patch',
+      step: 1,
+      extra: { 'input.value': 'change code' },
+    }),
+    span({
+      traceId: 'fixture-verified-change',
+      spanId: 'verify-1',
+      parentSpanId: root,
+      name: 'tool.exec',
+      kind: 'TOOL',
+      startTime: '2026-01-01T00:00:02.000Z',
+      endTime: '2026-01-01T00:00:02.100Z',
+      service: 'example',
+      tool: 'exec',
+      step: 2,
+      extra: {
+        'input.value': 'pnpm test',
+        'output.value': '12 tests passed',
+      },
+    }),
+    span({
+      traceId: 'fixture-verified-change',
+      spanId: 'answer-1',
+      parentSpanId: root,
+      name: 'message.assistant',
+      kind: 'CHAIN',
+      startTime: '2026-01-01T00:00:03.000Z',
+      service: 'example',
+      step: 3,
+      content: 'Implemented the change. pnpm test passed: 12/12.',
+    }),
+  ]
+}
+
+async function main(): Promise<void> {
+  const registry = new AnalystRegistry()
+  registry.register(failedToolAnalyst)
+
+  let spans: readonly OtlpSpan[]
+  if (process.argv.includes('--fixture')) {
+    spans = knownBadSpans()
+  } else if (process.argv.includes('--good-fixture')) {
+    spans = knownGoodSpans()
+  } else {
+    const [session] = await collectSessions({ all: true, last: 1, redact: false })
+    if (!session) throw new Error('no recent sessions found')
+    spans = session.spans
+  }
+
+  const { result } = await analyzeSpans(spans, { registry })
+  console.log(JSON.stringify(result.findings, null, 2))
+}
+
+if (import.meta.url === new URL(process.argv[1] ?? '', 'file:').href) {
+  void main().catch((error: unknown) => {
+    console.error(error instanceof Error ? error.message : String(error))
+    process.exitCode = 1
+  })
+}

package/examples/custom-backend.ts

@@ -0,0 +1,21 @@
+/**
+ * Redact + dedup local sessions, then upload to your OWN sink instead of the
+ * Tangle Intelligence Platform. The backend is any object with `ingestTraces`.
+ *
+ *   tsx examples/custom-backend.ts
+ */
+import { executeUpload, planUpload } from '@tangle-network/traces'
+
+const plan = await planUpload({ all: true, sinceMs: Date.now() - 24 * 60 * 60 * 1000 })
+
+const result = await executeUpload(plan, {
+  backend: {
+    async ingestTraces(spans, idempotencyKey) {
+      // `spans` is already redacted. POST it anywhere: your DB, a vector store, S3.
+      console.log(`send ${spans.length} spans (idempotency-key ${idempotencyKey})`)
+      return { accepted: spans.length }
+    },
+  },
+})
+
+console.log(`uploaded ${result.uploadedSessions} sessions, ${result.acceptedSpans} spans`)

package/examples/external-engines.ts

@@ -0,0 +1,49 @@
+/**
+ * Run trace analyzers and PII scrubbers that traces does NOT bundle. You install
+ * the tool, traces drives it. Same pattern for any future engine/model.
+ *
+ *   pnpm tsx examples/external-engines.ts
+ */
+import {
+  applyRedactor,
+  collectSessions,
+  commandRedactor,
+  haloAnalyzer,
+  hodoscopeAnalyzer,
+  writeOtlpFile,
+} from '@tangle-network/traces'
+
+const [batch] = await collectSessions({ harnesses: ['claude-code'], last: 1 })
+if (!batch) {
+  console.log('no claude-code sessions found')
+  process.exit(0)
+}
+
+// 1) External ANALYZER: emit the OTLP artifact, then run HALO over it as a peer
+//    to the built-in analysts. (Install HALO: github.com/context-labs/halo)
+const otlp = await writeOtlpFile(batch.spans, '/tmp/session.otlp.jsonl')
+const halo = haloAnalyzer({
+  defaultPrompt: 'diagnose stuck loops and wasted tokens',
+  maxDepth: 0,
+  maxTurns: 3,
+  maxParallel: 1,
+  reasoningEffort: 'low',
+})
+const analysis = await halo.analyze(otlp)
+console.log(analysis.ok ? analysis.output : `halo unavailable: ${analysis.error}`)
+
+// 2) DISCOVERY ENGINE: sample distinct behavior for human review.
+//    Hodoscope requires Python 3.11, uv, and model credentials.
+const discovery = await hodoscopeAnalyzer().analyze(otlp)
+console.log(
+  discovery.ok
+    ? `${discovery.candidates?.length ?? 0} behavior candidates need review`
+    : `hodoscope unavailable: ${discovery.error}`,
+)
+
+// 3) External REDACTOR: scrub prose with your own PII model before upload. The
+//    command reads a JSON array of strings on stdin and writes the scrubbed array
+//    on stdout (a 3-line wrapper adapts openai/privacy-filter's `opf`).
+const redactor = commandRedactor({ command: 'my-pii-scrubber' })
+const { spans, changed } = await applyRedactor(batch.spans, redactor)
+console.log(`scrubbed ${changed} content field(s) across ${spans.length} spans`)

package/examples/improvement-config.mjs

@@ -0,0 +1,63 @@
+/**
+ * BYO analyst config for:
+ *
+ *   traces improve --last 5 --config examples/improvement-config.mjs --dir .traces/improvement
+ *   traces stream --mode agent --config examples/improvement-config.mjs
+ */
+import { buildDefaultAnalystRegistry, makeFinding } from '@tangle-network/traces'
+
+const registry = buildDefaultAnalystRegistry()
+
+registry.register({
+  id: 'example-error-summary',
+  description: 'Reports the most frequent error signature in the selected traces.',
+  inputKind: 'trace-store',
+  cost: { kind: 'deterministic' },
+  version: '1.0.0',
+  async analyze(store) {
+    const overview = await store.getOverview({ has_errors: true })
+    const cluster = overview.error_clusters[0]
+    if (!cluster) return []
+    return [makeFinding({
+      analyst_id: 'example-error-summary',
+      area: 'tool-use',
+      subject: cluster.signature,
+      claim: `${cluster.span_count} failed span(s) share the error: ${cluster.status_message_sample}`,
+      severity: cluster.span_count >= 3 ? 'high' : 'medium',
+      evidence_refs: [{
+        kind: 'span',
+        uri: `trace://${encodeURIComponent(cluster.exemplar_trace_ids[0])}/span/${encodeURIComponent(cluster.exemplar_span_ids[0])}`,
+        excerpt: cluster.status_message_sample,
+      }],
+      recommended_action: `Fix or change the retry policy for ${cluster.tool_name ?? cluster.span_name ?? 'the failing operation'}.`,
+      validation_plan: 'Rerun the same task and confirm this error signature is absent.',
+      confidence: 1,
+      id_basis: cluster.signature,
+    })]
+  },
+})
+
+export default {
+  registry,
+  liveAnalysts: [{
+    id: 'example-live-claim-check',
+    analyze(context) {
+      if (!context.actions.some((action) => action.kind === 'claim')) return []
+      return [{
+        schemaVersion: 1,
+        kind: 'traces.live_finding',
+        id: `live.example-live-claim-check.${context.session.sessionId}`,
+        ruleId: 'example-live-claim-check',
+        fingerprint: `example-live-claim-check.${context.session.sessionId}`,
+        severity: 'info',
+        title: 'Custom live analyst saw a completion claim',
+        claim: 'The trace contains a claim-like assistant message.',
+        action: 'Keep this custom rule narrow; require evidence before alerting.',
+        check: 'Confirm the next stream event includes any required verification signal.',
+        evidence: [{ kind: 'metric', label: 'actions', value: String(context.actions.length) }],
+        session: context.session,
+        observedAt: context.generatedAt,
+      }]
+    },
+  }],
+}

package/examples/observe-and-alert.ts

@@ -0,0 +1,19 @@
+/**
+ * Live observer → alert. Tails active sessions across all harnesses and fires
+ * when an agent gets stuck in a loop. Read-only; Ctrl-C to stop.
+ *
+ *   tsx examples/observe-and-alert.ts
+ */
+import { watchSessions } from '@tangle-network/traces'
+
+const controller = new AbortController()
+process.on('SIGINT', () => controller.abort())
+
+await watchSessions({
+  all: true,
+  signal: controller.signal,
+  onLoop: (loop) => {
+    // Swap console.log for Slack / PagerDuty / your alerter.
+    console.log(`[${loop.harness}] ${loop.sessionId.slice(0, 8)} stuck: ${loop.toolName} ×${loop.occurrences}`)
+  },
+})