@tangle-network/traces 0.11.5 → 0.11.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +54 -2
- package/dist/{chunk-VTYOCFGF.js → chunk-MEAEXKUV.js} +1496 -271
- package/dist/cli.js +163 -61
- package/dist/index.d.ts +451 -1
- package/dist/index.js +33 -1
- package/docs/trace-analysts.md +39 -1
- package/examples/external-engines.ts +14 -1
- package/package.json +6 -5
- package/skills/build-trace-analyst/SKILL.md +13 -0
- package/skills/inspect-agent-traces/SKILL.md +30 -4
package/dist/index.js
CHANGED
|
@@ -25,6 +25,7 @@ import {
|
|
|
25
25
|
JsonSourceError,
|
|
26
26
|
JsonlParseError,
|
|
27
27
|
OpencodeAdapter,
|
|
28
|
+
POLICY_EVIDENCE_NOTE,
|
|
28
29
|
PartialUploadError,
|
|
29
30
|
PiAdapter,
|
|
30
31
|
QwenAdapter,
|
|
@@ -47,6 +48,7 @@ import {
|
|
|
47
48
|
analyzeSpans,
|
|
48
49
|
analyzeSteeringChain,
|
|
49
50
|
applyRedactor,
|
|
51
|
+
assembleSessionBundle,
|
|
50
52
|
assertSandboxReachable,
|
|
51
53
|
buildFixPrompt,
|
|
52
54
|
buildPolicyEvidenceRecord,
|
|
@@ -57,6 +59,7 @@ import {
|
|
|
57
59
|
buildTraceFindingPacket,
|
|
58
60
|
capText,
|
|
59
61
|
chatTrajectoryToSpans,
|
|
62
|
+
classifyEvidenceOnlyPath,
|
|
60
63
|
classifyLiveActions,
|
|
61
64
|
classifyReaction,
|
|
62
65
|
classifyVerdict,
|
|
@@ -83,6 +86,8 @@ import {
|
|
|
83
86
|
describeSessionRelationship,
|
|
84
87
|
dockerImagePreparer,
|
|
85
88
|
enumerateReplayableCases,
|
|
89
|
+
environmentAttributes,
|
|
90
|
+
environmentFromSpanAttributes,
|
|
86
91
|
equivalentGitCwds,
|
|
87
92
|
executeUpload,
|
|
88
93
|
exportTraceEvidenceFile,
|
|
@@ -90,6 +95,8 @@ import {
|
|
|
90
95
|
exportTraceEvidenceText,
|
|
91
96
|
extractFixCommand,
|
|
92
97
|
fileRunContextSupervisorRunReader,
|
|
98
|
+
findContextRoot,
|
|
99
|
+
findSessionTextLeaks,
|
|
93
100
|
findSpanSourceFiles,
|
|
94
101
|
findingReplayStep,
|
|
95
102
|
findingTrajectoryId,
|
|
@@ -97,7 +104,9 @@ import {
|
|
|
97
104
|
generateFixCommand,
|
|
98
105
|
goldIncorrectSteps,
|
|
99
106
|
haloAnalyzer,
|
|
107
|
+
hashSessionBundleFiles,
|
|
100
108
|
hodoscopeAnalyzer,
|
|
109
|
+
httpJsonTransport,
|
|
101
110
|
importCodeTraceBench,
|
|
102
111
|
incompleteInputsNote,
|
|
103
112
|
indexSessionIdsByTrace,
|
|
@@ -112,6 +121,7 @@ import {
|
|
|
112
121
|
isSubmitAction,
|
|
113
122
|
knownHarnesses,
|
|
114
123
|
listAdapters,
|
|
124
|
+
listSessionBundleFiles,
|
|
115
125
|
loadState,
|
|
116
126
|
loadTracesConfig,
|
|
117
127
|
locateSessions,
|
|
@@ -134,6 +144,8 @@ import {
|
|
|
134
144
|
parseVerifyFindingsArgs,
|
|
135
145
|
planTraceAgenticRoute,
|
|
136
146
|
planUpload,
|
|
147
|
+
primeAnalyzer,
|
|
148
|
+
projectSessionBundle,
|
|
137
149
|
readCoordinationLog,
|
|
138
150
|
readFileRunContext,
|
|
139
151
|
readFindingsFile,
|
|
@@ -144,6 +156,7 @@ import {
|
|
|
144
156
|
readRunContextJournal,
|
|
145
157
|
readRunContextSnapshot,
|
|
146
158
|
readRunWatchSnapshot,
|
|
159
|
+
readSessionBundleManifest,
|
|
147
160
|
readSessionIndexFile,
|
|
148
161
|
readSpanRunSnapshot,
|
|
149
162
|
recordSessionCorruption,
|
|
@@ -192,9 +205,11 @@ import {
|
|
|
192
205
|
sessionHash,
|
|
193
206
|
sessionIdFromAttributes,
|
|
194
207
|
sessionIntegrityAttributes,
|
|
208
|
+
sessionTextSignatures,
|
|
195
209
|
skippedAnalyses,
|
|
196
210
|
span,
|
|
197
211
|
spanRunTotals,
|
|
212
|
+
stampEnvironmentAttrs,
|
|
198
213
|
stampRepoAttrs,
|
|
199
214
|
stampSessionIdentity,
|
|
200
215
|
stampSessionIntegrity,
|
|
@@ -228,12 +243,13 @@ import {
|
|
|
228
243
|
writeInspectionReportFile,
|
|
229
244
|
writeOtlpFile,
|
|
230
245
|
writePolicyEvidenceFile,
|
|
246
|
+
writeSessionBundleManifest,
|
|
231
247
|
writeSessionIndexFile,
|
|
232
248
|
writeTraceEvidenceExportFile,
|
|
233
249
|
writeTraceImprovementArtifacts,
|
|
234
250
|
zaiChatCaller,
|
|
235
251
|
zeroPricedCost
|
|
236
|
-
} from "./chunk-
|
|
252
|
+
} from "./chunk-MEAEXKUV.js";
|
|
237
253
|
|
|
238
254
|
// src/collect.ts
|
|
239
255
|
async function collectSessions(opts = {}) {
|
|
@@ -286,6 +302,7 @@ export {
|
|
|
286
302
|
JsonSourceError,
|
|
287
303
|
JsonlParseError,
|
|
288
304
|
OpencodeAdapter,
|
|
305
|
+
POLICY_EVIDENCE_NOTE,
|
|
289
306
|
PartialUploadError,
|
|
290
307
|
PiAdapter,
|
|
291
308
|
QwenAdapter,
|
|
@@ -308,6 +325,7 @@ export {
|
|
|
308
325
|
analyzeSpans,
|
|
309
326
|
analyzeSteeringChain,
|
|
310
327
|
applyRedactor,
|
|
328
|
+
assembleSessionBundle,
|
|
311
329
|
assertSandboxReachable,
|
|
312
330
|
buildDefaultAnalystRegistry,
|
|
313
331
|
buildFixPrompt,
|
|
@@ -319,6 +337,7 @@ export {
|
|
|
319
337
|
buildTraceFindingPacket,
|
|
320
338
|
capText,
|
|
321
339
|
chatTrajectoryToSpans,
|
|
340
|
+
classifyEvidenceOnlyPath,
|
|
322
341
|
classifyLiveActions,
|
|
323
342
|
classifyReaction,
|
|
324
343
|
classifyVerdict,
|
|
@@ -347,6 +366,8 @@ export {
|
|
|
347
366
|
describeSessionRelationship,
|
|
348
367
|
dockerImagePreparer,
|
|
349
368
|
enumerateReplayableCases,
|
|
369
|
+
environmentAttributes,
|
|
370
|
+
environmentFromSpanAttributes,
|
|
350
371
|
equivalentGitCwds,
|
|
351
372
|
executeUpload,
|
|
352
373
|
exportTraceEvidenceFile,
|
|
@@ -354,6 +375,8 @@ export {
|
|
|
354
375
|
exportTraceEvidenceText,
|
|
355
376
|
extractFixCommand,
|
|
356
377
|
fileRunContextSupervisorRunReader,
|
|
378
|
+
findContextRoot,
|
|
379
|
+
findSessionTextLeaks,
|
|
357
380
|
findSpanSourceFiles,
|
|
358
381
|
findingReplayStep,
|
|
359
382
|
findingTrajectoryId,
|
|
@@ -361,8 +384,10 @@ export {
|
|
|
361
384
|
generateFixCommand,
|
|
362
385
|
goldIncorrectSteps,
|
|
363
386
|
haloAnalyzer,
|
|
387
|
+
hashSessionBundleFiles,
|
|
364
388
|
hodoscopeAnalyzer,
|
|
365
389
|
hostedClientFromEnv,
|
|
390
|
+
httpJsonTransport,
|
|
366
391
|
importCodeTraceBench,
|
|
367
392
|
incompleteInputsNote,
|
|
368
393
|
indexSessionIdsByTrace,
|
|
@@ -377,6 +402,7 @@ export {
|
|
|
377
402
|
isSubmitAction,
|
|
378
403
|
knownHarnesses,
|
|
379
404
|
listAdapters,
|
|
405
|
+
listSessionBundleFiles,
|
|
380
406
|
loadState,
|
|
381
407
|
loadTracesConfig,
|
|
382
408
|
locateSessions,
|
|
@@ -400,6 +426,8 @@ export {
|
|
|
400
426
|
parseVerifyFindingsArgs,
|
|
401
427
|
planTraceAgenticRoute,
|
|
402
428
|
planUpload,
|
|
429
|
+
primeAnalyzer,
|
|
430
|
+
projectSessionBundle,
|
|
403
431
|
readCoordinationLog,
|
|
404
432
|
readFileRunContext,
|
|
405
433
|
readFindingsFile,
|
|
@@ -410,6 +438,7 @@ export {
|
|
|
410
438
|
readRunContextJournal,
|
|
411
439
|
readRunContextSnapshot,
|
|
412
440
|
readRunWatchSnapshot,
|
|
441
|
+
readSessionBundleManifest,
|
|
413
442
|
readSessionIndexFile,
|
|
414
443
|
readSpanRunSnapshot,
|
|
415
444
|
recordSessionCorruption,
|
|
@@ -460,9 +489,11 @@ export {
|
|
|
460
489
|
sessionHash,
|
|
461
490
|
sessionIdFromAttributes,
|
|
462
491
|
sessionIntegrityAttributes,
|
|
492
|
+
sessionTextSignatures,
|
|
463
493
|
skippedAnalyses,
|
|
464
494
|
span,
|
|
465
495
|
spanRunTotals,
|
|
496
|
+
stampEnvironmentAttrs,
|
|
466
497
|
stampRepoAttrs,
|
|
467
498
|
stampSessionIdentity,
|
|
468
499
|
stampSessionIntegrity,
|
|
@@ -496,6 +527,7 @@ export {
|
|
|
496
527
|
writeInspectionReportFile,
|
|
497
528
|
writeOtlpFile,
|
|
498
529
|
writePolicyEvidenceFile,
|
|
530
|
+
writeSessionBundleManifest,
|
|
499
531
|
writeSessionIndexFile,
|
|
500
532
|
writeTraceEvidenceExportFile,
|
|
501
533
|
writeTraceImprovementArtifacts,
|
package/docs/trace-analysts.md
CHANGED
|
@@ -68,9 +68,12 @@ traces analyze --all --last 20 --analyzer hodoscope
|
|
|
68
68
|
|
|
69
69
|
# Run any installed command that accepts an OpenInference file path.
|
|
70
70
|
traces analyze --last 1 --analyzer my-trace-tool
|
|
71
|
+
|
|
72
|
+
# One-shot prime-RLM analysis through a local OpenAI-compatible bridge.
|
|
73
|
+
traces analyze --last 1 --analyzer prime --analyzer-prompt "find unsupported completion claims"
|
|
71
74
|
```
|
|
72
75
|
|
|
73
|
-
`--model` is forwarded to the built-in model-assisted analysts, HALO, and
|
|
76
|
+
`--model` is forwarded to the built-in model-assisted analysts, HALO, Hodoscope, and prime.
|
|
74
77
|
HALO and Hodoscope use their own provider clients and credentials.
|
|
75
78
|
The Hodoscope adapter pins version `0.2.4` and uses Python 3.11 through `uvx`.
|
|
76
79
|
|
|
@@ -87,6 +90,41 @@ Raw JSON is still a `report`.
|
|
|
87
90
|
Only an adapter that validates the full finding shape may return `findings`.
|
|
88
91
|
Hodoscope always returns `discovery`, and each candidate has `status: 'needs_review'` plus its source trace and span.
|
|
89
92
|
|
|
93
|
+
### Prime engine
|
|
94
|
+
|
|
95
|
+
`--analyzer prime` runs a one-shot analyst over the emitted OTLP artifact through an OpenAI-compatible bridge, such as cli-bridge's prime backend.
|
|
96
|
+
Unlike the `--llm` analysts, which drill into the trace with paged tools, prime has no REPL and no trace tools: the full span projection is inlined into a single prompt as JSON.
|
|
97
|
+
Oversized projections are re-rendered with a per-attribute character cap; if the projection still exceeds the inline budget the engine fails loud instead of silently dropping spans.
|
|
98
|
+
|
|
99
|
+
Prerequisites:
|
|
100
|
+
|
|
101
|
+
- A running bridge that accepts `POST /v1/chat/completions` and routes the configured model to the prime backend.
|
|
102
|
+
- No key handling here: the bridge owns provider credentials.
|
|
103
|
+
|
|
104
|
+
Configuration (flags first, then environment, then defaults):
|
|
105
|
+
|
|
106
|
+
| Setting | Source | Default |
|
|
107
|
+
| --- | --- | --- |
|
|
108
|
+
| Bridge root URL | `TRACES_PRIME_BRIDGE_URL` | `http://localhost:4181` |
|
|
109
|
+
| Model | `--model`, then `TRACES_PRIME_MODEL` | `prime/zai/glm-5.2` |
|
|
110
|
+
| Per-call deadline | `TRACES_PRIME_TIMEOUT_MS` | `1200000` (20 min) |
|
|
111
|
+
| Question | `--analyzer-prompt` | a general diagnosis question |
|
|
112
|
+
|
|
113
|
+
```bash
|
|
114
|
+
traces analyze --last 1 --analyzer prime
|
|
115
|
+
TRACES_PRIME_BRIDGE_URL=http://localhost:4181 traces analyze --last 1 --analyzer prime --analyzer-prompt "find unsupported completion claims"
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
Output expectations:
|
|
119
|
+
|
|
120
|
+
- The reply contract is one fenced JSON block of short strings citing span ids verbatim; a structurally malformed reply gets exactly one bounded repair turn that carries the malformed reply and the contract, never the trajectory.
|
|
121
|
+
- A still-malformed reply after repair is a failed result (`ok: false`) with the raw reply preserved for inspection; one failed engine never discards the other results.
|
|
122
|
+
- Valid rows become full `findings` with `trace://` span evidence, validated against the artifact like every other findings-kind engine; rows citing unknown or ambiguous span ids are rejected with a recorded reason.
|
|
123
|
+
- Zero findings from a well-formed reply is an honest null, not a failure.
|
|
124
|
+
- Bridge-reported token usage and call counts are recorded in the result output; cost stays uncaptured because this adapter has no pricing table.
|
|
125
|
+
|
|
126
|
+
The scored prime-vs-dspy comparison — same trajectories, same scoring — lives in `@tangle-network/agent-eval`'s analyst benchmark (`runAnalystBenchmark`); this engine is the capture-side entry point, not the scoreboard.
|
|
127
|
+
|
|
90
128
|
## Write one analyst
|
|
91
129
|
|
|
92
130
|
An analyst receives a paged trace store and returns typed findings.
|
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
commandRedactor,
|
|
11
11
|
haloAnalyzer,
|
|
12
12
|
hodoscopeAnalyzer,
|
|
13
|
+
primeAnalyzer,
|
|
13
14
|
writeOtlpFile,
|
|
14
15
|
} from '@tangle-network/traces'
|
|
15
16
|
|
|
@@ -41,7 +42,19 @@ console.log(
|
|
|
41
42
|
: `hodoscope unavailable: ${discovery.error}`,
|
|
42
43
|
)
|
|
43
44
|
|
|
44
|
-
// 3)
|
|
45
|
+
// 3) PRIME ENGINE: one-shot RLM through an OpenAI-compatible bridge (run
|
|
46
|
+
// cli-bridge's prime backend locally, or set TRACES_PRIME_BRIDGE_URL).
|
|
47
|
+
// The full span projection travels inline; findings come back with
|
|
48
|
+
// validated trace:// span evidence.
|
|
49
|
+
const prime = primeAnalyzer({ defaultPrompt: 'find unsupported completion claims' })
|
|
50
|
+
const primeResult = await prime.analyze(otlp)
|
|
51
|
+
console.log(
|
|
52
|
+
primeResult.ok
|
|
53
|
+
? `${primeResult.findings?.length ?? 0} prime finding(s)\n${primeResult.output}`
|
|
54
|
+
: `prime bridge unavailable: ${primeResult.error}`,
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
// 4) External REDACTOR: scrub prose with your own PII model before upload. The
|
|
45
58
|
// command reads a JSON array of strings on stdin and writes the scrubbed array
|
|
46
59
|
// on stdout (a 3-line wrapper adapts openai/privacy-filter's `opf`).
|
|
47
60
|
const redactor = commandRedactor({ command: 'my-pii-scrubber' })
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tangle-network/traces",
|
|
3
|
-
"version": "0.11.
|
|
3
|
+
"version": "0.11.6",
|
|
4
4
|
"description": "Point it at your coding-agent session traces (Claude Code, Codex, OpenCode, Gemini, Pi, …) and get failure-mode + efficiency findings. CLI + SDK over the @tangle-network/agent-eval analyst suite — observe live sessions, run your own analysts, redact, and upload to the Tangle Intelligence Platform.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -44,13 +44,14 @@
|
|
|
44
44
|
"README.md"
|
|
45
45
|
],
|
|
46
46
|
"engines": {
|
|
47
|
-
"node": ">=22"
|
|
47
|
+
"node": ">=22.13.0"
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
|
-
"@tangle-network/agent-eval": "0.
|
|
51
|
-
"@tangle-network/agent-
|
|
50
|
+
"@tangle-network/agent-eval": "0.145.22",
|
|
51
|
+
"@tangle-network/agent-interface": "1.0.0",
|
|
52
|
+
"@tangle-network/agent-runtime": "0.137.0",
|
|
52
53
|
"@tangle-network/agent-trace-contract": "^1.0.2",
|
|
53
|
-
"@tangle-network/sandbox": "0.
|
|
54
|
+
"@tangle-network/sandbox": "0.27.1"
|
|
54
55
|
},
|
|
55
56
|
"devDependencies": {
|
|
56
57
|
"@types/node": "^22.0.0",
|
|
@@ -48,6 +48,19 @@ Use `runAnalystBenchmark` with `registryBenchmarkRunner` from `@tangle-network/a
|
|
|
48
48
|
Use AgentRx or CodeTraceBench labels when they match the finding.
|
|
49
49
|
Compare the candidate with a no-findings baseline and relevant maintained or external analysts on identical cases and limits.
|
|
50
50
|
|
|
51
|
+
Two shipped engines are available as comparison arms.
|
|
52
|
+
`--analyzer prime` inlines the whole span projection into one prompt and needs no REPL.
|
|
53
|
+
`--analyzer halo` and `--analyzer hodoscope` drill into the trace with paged tools.
|
|
54
|
+
Compare an inline arm with a paged arm to separate the prompt from the navigation.
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
traces analyze --last 1 --analyzer prime
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
Assert the served model, not the requested id.
|
|
61
|
+
A gateway can answer one id with another model, so a panel can report families it never ran.
|
|
62
|
+
Use `assertServedModel` and `assertCrossFamilyServed` from `@tangle-network/agent-eval` before you trust a per-model or cross-family number.
|
|
63
|
+
|
|
51
64
|
Use at least 20 independent cases before comparative statistics.
|
|
52
65
|
Retain every case row and report exact-step recall, precision, F1, root-step accuracy when labeled, citation coverage and validity, clean-case false positives, repeat agreement, failures, latency, calls, all token fields, and known or missing cost.
|
|
53
66
|
|
|
@@ -22,10 +22,9 @@ traces analyze --otlp results/sessions --out .traces/all.md # a directory of e
|
|
|
22
22
|
Use `--harness` only for coding agents whose on-disk format we do not control.
|
|
23
23
|
Those adapters are the legacy edge, not the way to integrate a system you own.
|
|
24
24
|
|
|
25
|
-
A
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
read on its own.
|
|
25
|
+
A run directory holds the span export beside raw event, stream and SDK logs that are
|
|
26
|
+
also `*.jsonl`. Only the OTLP files are read; the rest are listed with what they hold.
|
|
27
|
+
An `otlp/` subdirectory, when present, is read on its own.
|
|
29
28
|
|
|
30
29
|
Any section headed `inputs incomplete`, or carrying an `Inputs incomplete` line above
|
|
31
30
|
its table, is computed from a field the trace does not carry everywhere. Report those
|
|
@@ -73,6 +72,33 @@ traces improve --harness codex --current --latest-turn --workflow \
|
|
|
73
72
|
`improve` writes findings, evidence, a report, and spans.
|
|
74
73
|
It does not edit an agent, repository, memory store, or knowledge base.
|
|
75
74
|
|
|
75
|
+
Write one session's durable evidence directory for a later reader:
|
|
76
|
+
|
|
77
|
+
```bash
|
|
78
|
+
traces bundle --harness claude-code --session <id-or-path> --out .traces/bundle
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
`bundle` copies the transcript, the derived report and spans, the `.evolve` ledger rows inside the session window, and a `manifest.json` with a SHA-256 per file.
|
|
82
|
+
It spends no model call.
|
|
83
|
+
A missing transcript stops the assembly.
|
|
84
|
+
An absent optional input is recorded in `manifest.absent` with the probed path.
|
|
85
|
+
|
|
86
|
+
## Pick the view for the reader
|
|
87
|
+
|
|
88
|
+
That bundle is the FULL view (`manifest.view: "full"`), and it holds the whole session transcript.
|
|
89
|
+
Never give it to a writer that must not see an earlier conclusion: the transcript holds the text of every file the session wrote, so a check for the earlier report FILE passes while its CONTENT is still readable.
|
|
90
|
+
|
|
91
|
+
Project the writer's copy instead:
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
traces bundle-view .traces/bundle --view evidence-only --out .traces/writer
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
That view carries `derived/session-index.json`, `derived/evidence.jsonl`, and the structured `ledger/` records.
|
|
98
|
+
It excludes the transcripts, the report, the spans, and every prose ledger file by name, with rules and hashes in `manifest.excluded`.
|
|
99
|
+
Before writing, it compares each carried file against each excluded file for shared 8-word runs of prose, and drops any file that repeats one.
|
|
100
|
+
`manifest.view` names which copy you hold.
|
|
101
|
+
|
|
76
102
|
## Report
|
|
77
103
|
|
|
78
104
|
- State the source (`--otlp <path>` or the harness), selected task boundary, session and span counts, and integrity warnings.
|