@tangle-network/traces 0.11.4 → 0.11.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -14,6 +14,7 @@ import {
14
14
  CodexTaskScopeError,
15
15
  CopilotAdapter,
16
16
  DEFAULT_HARNESS,
17
+ DEFAULT_SANDBOX_BASE_URL,
17
18
  EMPTY_SPAN_RUN_SNAPSHOT,
18
19
  EXPORT_RULE,
19
20
  EmptySessionError,
@@ -24,6 +25,7 @@ import {
24
25
  JsonSourceError,
25
26
  JsonlParseError,
26
27
  OpencodeAdapter,
28
+ POLICY_EVIDENCE_NOTE,
27
29
  PartialUploadError,
28
30
  PiAdapter,
29
31
  QwenAdapter,
@@ -46,19 +48,26 @@ import {
46
48
  analyzeSpans,
47
49
  analyzeSteeringChain,
48
50
  applyRedactor,
51
+ assembleSessionBundle,
52
+ assertSandboxReachable,
49
53
  buildFixPrompt,
50
54
  buildPolicyEvidenceRecord,
55
+ buildRetryFixPrompt,
51
56
  buildSessionIndex,
52
57
  buildSessionIndexFromRows,
53
58
  buildSpanRunTree,
54
59
  buildTraceFindingPacket,
55
60
  capText,
56
61
  chatTrajectoryToSpans,
62
+ classifyEvidenceOnlyPath,
57
63
  classifyLiveActions,
58
64
  classifyReaction,
65
+ classifyVerdict,
59
66
  claudeActor,
67
+ clipText,
60
68
  cmdReplayVerify,
61
69
  cmdReplayVerifyBatch,
70
+ cmdVerifyFindings,
62
71
  codexActor,
63
72
  collectPolicyEvidence,
64
73
  collectSessionIndex,
@@ -68,6 +77,7 @@ import {
68
77
  commandRedactor,
69
78
  conformanceOfSpans,
70
79
  connectors,
80
+ countScriptCommands,
71
81
  countSkillRunsJsonl,
72
82
  cwdMatchesSelection,
73
83
  defaultTraceLiveAnalysts,
@@ -76,6 +86,8 @@ import {
76
86
  describeSessionRelationship,
77
87
  dockerImagePreparer,
78
88
  enumerateReplayableCases,
89
+ environmentAttributes,
90
+ environmentFromSpanAttributes,
79
91
  equivalentGitCwds,
80
92
  executeUpload,
81
93
  exportTraceEvidenceFile,
@@ -83,12 +95,18 @@ import {
83
95
  exportTraceEvidenceText,
84
96
  extractFixCommand,
85
97
  fileRunContextSupervisorRunReader,
98
+ findContextRoot,
99
+ findSessionTextLeaks,
86
100
  findSpanSourceFiles,
101
+ findingReplayStep,
102
+ findingTrajectoryId,
87
103
  forEachTreeNode,
88
104
  generateFixCommand,
89
105
  goldIncorrectSteps,
90
106
  haloAnalyzer,
107
+ hashSessionBundleFiles,
91
108
  hodoscopeAnalyzer,
109
+ httpJsonTransport,
92
110
  importCodeTraceBench,
93
111
  incompleteInputsNote,
94
112
  indexSessionIdsByTrace,
@@ -103,6 +121,7 @@ import {
103
121
  isSubmitAction,
104
122
  knownHarnesses,
105
123
  listAdapters,
124
+ listSessionBundleFiles,
106
125
  loadState,
107
126
  loadTracesConfig,
108
127
  locateSessions,
@@ -114,6 +133,7 @@ import {
114
133
  otlpRowToSpan,
115
134
  outboundHash,
116
135
  parseCorpusFlag,
136
+ parseIncorrectStepsSubject,
117
137
  parseIsoToEpochMs,
118
138
  parseObservationOutput,
119
139
  parseRecordedReturncode,
@@ -121,10 +141,14 @@ import {
121
141
  parseReplayVerifyArgs,
122
142
  parseSession,
123
143
  parseSince,
144
+ parseVerifyFindingsArgs,
124
145
  planTraceAgenticRoute,
125
146
  planUpload,
147
+ primeAnalyzer,
148
+ projectSessionBundle,
126
149
  readCoordinationLog,
127
150
  readFileRunContext,
151
+ readFindingsFile,
128
152
  readJsonFile,
129
153
  readJsonl,
130
154
  readLabelEntries,
@@ -132,6 +156,7 @@ import {
132
156
  readRunContextJournal,
133
157
  readRunContextSnapshot,
134
158
  readRunWatchSnapshot,
159
+ readSessionBundleManifest,
135
160
  readSessionIndexFile,
136
161
  readSpanRunSnapshot,
137
162
  recordSessionCorruption,
@@ -143,11 +168,15 @@ import {
143
168
  renderRunWatchSnapshot,
144
169
  renderSpanRunSnapshot,
145
170
  renderValidation,
171
+ renderVerifiedFindingsSection,
146
172
  replayBatchUsage,
147
173
  replayVerify,
174
+ replayVerifyFinding,
148
175
  replayVerifyUsage,
149
176
  resolveAdapter,
150
177
  resolveCaseResources,
178
+ resolveFindingInvocation,
179
+ resolveFindingReplayability,
151
180
  resolveOtlpInputFiles,
152
181
  resolveRepoAttrs,
153
182
  resolveRunContextDir,
@@ -155,6 +184,7 @@ import {
155
184
  resolveSessionRepoAttrs,
156
185
  runCommand,
157
186
  runExternalAnalyzers,
187
+ runFixLoop,
158
188
  runPipelines,
159
189
  runReplayBatch,
160
190
  runTotals,
@@ -175,9 +205,11 @@ import {
175
205
  sessionHash,
176
206
  sessionIdFromAttributes,
177
207
  sessionIntegrityAttributes,
208
+ sessionTextSignatures,
178
209
  skippedAnalyses,
179
210
  span,
180
211
  spanRunTotals,
212
+ stampEnvironmentAttrs,
181
213
  stampRepoAttrs,
182
214
  stampSessionIdentity,
183
215
  stampSessionIntegrity,
@@ -201,6 +233,8 @@ import {
201
233
  userPromptSpan,
202
234
  validateOtlpSpans,
203
235
  validationExitCode,
236
+ verifyFindings,
237
+ verifyFindingsUsage,
204
238
  watchRunTarget,
205
239
  watchSessions,
206
240
  wrap,
@@ -209,12 +243,13 @@ import {
209
243
  writeInspectionReportFile,
210
244
  writeOtlpFile,
211
245
  writePolicyEvidenceFile,
246
+ writeSessionBundleManifest,
212
247
  writeSessionIndexFile,
213
248
  writeTraceEvidenceExportFile,
214
249
  writeTraceImprovementArtifacts,
215
250
  zaiChatCaller,
216
251
  zeroPricedCost
217
- } from "./chunk-AGPOIWPQ.js";
252
+ } from "./chunk-MEAEXKUV.js";
218
253
 
219
254
  // src/collect.ts
220
255
  async function collectSessions(opts = {}) {
@@ -230,79 +265,6 @@ async function collectSessions(opts = {}) {
230
265
  return out;
231
266
  }
232
267
 
233
- // src/replay-wire.ts
234
- var SUBJECT_PATTERN = /^incorrect-steps-(\d+)-(\d+)-(escaped|unescaped)-consequence-(\d+)$/;
235
- function parseIncorrectStepsSubject(subject) {
236
- const match = SUBJECT_PATTERN.exec(subject);
237
- if (!match) return null;
238
- return {
239
- firstStep: Number(match[1]),
240
- lastStep: Number(match[2]),
241
- escapeStatus: match[3],
242
- consequenceStep: Number(match[4])
243
- };
244
- }
245
- function resolveFindingInvocation(finding, corpora) {
246
- const subject = parseIncorrectStepsSubject(finding.subject);
247
- if (!subject) {
248
- throw new Error(
249
- `replay-wire: subject '${finding.subject}' is not incorrect-steps-<first>-<last>-<escaped|unescaped>-consequence-<step>`
250
- );
251
- }
252
- const failures = [];
253
- for (const corpus of corpora) {
254
- const resolution = resolveCaseResources(corpus, finding.trajId);
255
- if (resolution.resolved) {
256
- const at = subject.firstStep;
257
- const step = resolution.resources.steps.find((s) => s.step_id === at);
258
- if (!step) {
259
- throw new Error(
260
- `replay-wire: finding step ${at} is outside ${finding.trajId} (${resolution.resources.steps.length} steps)`
261
- );
262
- }
263
- return { resources: resolution.resources, subject, at };
264
- }
265
- failures.push(`${corpus.name}: ${resolution.reason}`);
266
- }
267
- throw new Error(
268
- `replay-wire: trajectory ${finding.trajId} is not replayable in any corpus \u2014 ${failures.join("; ")}`
269
- );
270
- }
271
- async function replayVerifyFinding(finding, options) {
272
- if (options.fixCaller && options.fixCommand !== void 0) {
273
- throw new Error("replay-wire: pass fixCaller or fixCommand, not both");
274
- }
275
- const invocation = resolveFindingInvocation(finding, options.corpora);
276
- let fixCommand = options.fixCommand ?? null;
277
- if (options.fixCaller) {
278
- const generated = await generateFixCommand(options.fixCaller, {
279
- taskStatement: invocation.resources.taskStatement,
280
- steps: invocation.resources.steps,
281
- k: invocation.at
282
- });
283
- if (!generated.succeeded) {
284
- throw new Error(`replay-wire: fix generation failed \u2014 ${generated.error}`);
285
- }
286
- fixCommand = generated.value.command;
287
- }
288
- const verdict = await replayVerify({
289
- stepsPath: invocation.resources.stepsPath,
290
- image: invocation.resources.image,
291
- at: invocation.at,
292
- fixCommand: fixCommand ?? void 0,
293
- cwd: invocation.resources.cwd,
294
- out: options.out,
295
- caseId: invocation.resources.trajId,
296
- stepTimeoutMs: options.stepTimeoutMs ?? invocation.resources.recordedStepTimeoutMs ?? void 0,
297
- prefixLimit: options.prefixLimit,
298
- backend: options.backend,
299
- apiKey: options.apiKey,
300
- baseUrl: options.baseUrl,
301
- onProgress: options.onProgress
302
- });
303
- return { invocation, fixCommand, verdict };
304
- }
305
-
306
268
  // src/index.ts
307
269
  import {
308
270
  AnalystRegistry,
@@ -329,6 +291,7 @@ export {
329
291
  CopilotAdapter,
330
292
  DEFAULT_HARNESS,
331
293
  DEFAULT_REDACTION_RULES,
294
+ DEFAULT_SANDBOX_BASE_URL,
332
295
  EMPTY_SPAN_RUN_SNAPSHOT,
333
296
  EXPORT_RULE,
334
297
  EmptySessionError,
@@ -339,6 +302,7 @@ export {
339
302
  JsonSourceError,
340
303
  JsonlParseError,
341
304
  OpencodeAdapter,
305
+ POLICY_EVIDENCE_NOTE,
342
306
  PartialUploadError,
343
307
  PiAdapter,
344
308
  QwenAdapter,
@@ -361,20 +325,27 @@ export {
361
325
  analyzeSpans,
362
326
  analyzeSteeringChain,
363
327
  applyRedactor,
328
+ assembleSessionBundle,
329
+ assertSandboxReachable,
364
330
  buildDefaultAnalystRegistry,
365
331
  buildFixPrompt,
366
332
  buildPolicyEvidenceRecord,
333
+ buildRetryFixPrompt,
367
334
  buildSessionIndex,
368
335
  buildSessionIndexFromRows,
369
336
  buildSpanRunTree,
370
337
  buildTraceFindingPacket,
371
338
  capText,
372
339
  chatTrajectoryToSpans,
340
+ classifyEvidenceOnlyPath,
373
341
  classifyLiveActions,
374
342
  classifyReaction,
343
+ classifyVerdict,
375
344
  claudeActor,
345
+ clipText,
376
346
  cmdReplayVerify,
377
347
  cmdReplayVerifyBatch,
348
+ cmdVerifyFindings,
378
349
  codexActor,
379
350
  collectPolicyEvidence,
380
351
  collectSessionIndex,
@@ -385,6 +356,7 @@ export {
385
356
  commandRedactor,
386
357
  conformanceOfSpans,
387
358
  connectors,
359
+ countScriptCommands,
388
360
  countSkillRunsJsonl,
389
361
  createHostedClient,
390
362
  cwdMatchesSelection,
@@ -394,6 +366,8 @@ export {
394
366
  describeSessionRelationship,
395
367
  dockerImagePreparer,
396
368
  enumerateReplayableCases,
369
+ environmentAttributes,
370
+ environmentFromSpanAttributes,
397
371
  equivalentGitCwds,
398
372
  executeUpload,
399
373
  exportTraceEvidenceFile,
@@ -401,13 +375,19 @@ export {
401
375
  exportTraceEvidenceText,
402
376
  extractFixCommand,
403
377
  fileRunContextSupervisorRunReader,
378
+ findContextRoot,
379
+ findSessionTextLeaks,
404
380
  findSpanSourceFiles,
381
+ findingReplayStep,
382
+ findingTrajectoryId,
405
383
  forEachTreeNode,
406
384
  generateFixCommand,
407
385
  goldIncorrectSteps,
408
386
  haloAnalyzer,
387
+ hashSessionBundleFiles,
409
388
  hodoscopeAnalyzer,
410
389
  hostedClientFromEnv,
390
+ httpJsonTransport,
411
391
  importCodeTraceBench,
412
392
  incompleteInputsNote,
413
393
  indexSessionIdsByTrace,
@@ -422,6 +402,7 @@ export {
422
402
  isSubmitAction,
423
403
  knownHarnesses,
424
404
  listAdapters,
405
+ listSessionBundleFiles,
425
406
  loadState,
426
407
  loadTracesConfig,
427
408
  locateSessions,
@@ -442,10 +423,14 @@ export {
442
423
  parseReplayVerifyArgs,
443
424
  parseSession,
444
425
  parseSince,
426
+ parseVerifyFindingsArgs,
445
427
  planTraceAgenticRoute,
446
428
  planUpload,
429
+ primeAnalyzer,
430
+ projectSessionBundle,
447
431
  readCoordinationLog,
448
432
  readFileRunContext,
433
+ readFindingsFile,
449
434
  readJsonFile,
450
435
  readJsonl,
451
436
  readLabelEntries,
@@ -453,6 +438,7 @@ export {
453
438
  readRunContextJournal,
454
439
  readRunContextSnapshot,
455
440
  readRunWatchSnapshot,
441
+ readSessionBundleManifest,
456
442
  readSessionIndexFile,
457
443
  readSpanRunSnapshot,
458
444
  recordSessionCorruption,
@@ -466,6 +452,7 @@ export {
466
452
  renderRunWatchSnapshot,
467
453
  renderSpanRunSnapshot,
468
454
  renderValidation,
455
+ renderVerifiedFindingsSection,
469
456
  replayBatchUsage,
470
457
  replayVerify,
471
458
  replayVerifyFinding,
@@ -473,6 +460,7 @@ export {
473
460
  resolveAdapter,
474
461
  resolveCaseResources,
475
462
  resolveFindingInvocation,
463
+ resolveFindingReplayability,
476
464
  resolveOtlpInputFiles,
477
465
  resolveRepoAttrs,
478
466
  resolveRunContextDir,
@@ -480,6 +468,7 @@ export {
480
468
  resolveSessionRepoAttrs,
481
469
  runCommand,
482
470
  runExternalAnalyzers,
471
+ runFixLoop,
483
472
  runPipelines,
484
473
  runReplayBatch,
485
474
  runTotals,
@@ -500,9 +489,11 @@ export {
500
489
  sessionHash,
501
490
  sessionIdFromAttributes,
502
491
  sessionIntegrityAttributes,
492
+ sessionTextSignatures,
503
493
  skippedAnalyses,
504
494
  span,
505
495
  spanRunTotals,
496
+ stampEnvironmentAttrs,
506
497
  stampRepoAttrs,
507
498
  stampSessionIdentity,
508
499
  stampSessionIntegrity,
@@ -526,6 +517,8 @@ export {
526
517
  userPromptSpan,
527
518
  validateOtlpSpans,
528
519
  validationExitCode,
520
+ verifyFindings,
521
+ verifyFindingsUsage,
529
522
  watchRunTarget,
530
523
  watchSessions,
531
524
  wrap,
@@ -534,6 +527,7 @@ export {
534
527
  writeInspectionReportFile,
535
528
  writeOtlpFile,
536
529
  writePolicyEvidenceFile,
530
+ writeSessionBundleManifest,
537
531
  writeSessionIndexFile,
538
532
  writeTraceEvidenceExportFile,
539
533
  writeTraceImprovementArtifacts,
@@ -77,6 +77,17 @@ A corpus is a gold label file plus a prepared directory (`normalized/<traj_id>/s
77
77
 
78
78
  Fix generation (`--fix generate`) runs ONE chat completion per arm-A-reproduced case against an OpenAI-compatible endpoint (default: glm-5.2 on z.ai). The prompt carries the gold step's action and observation, ±3 surrounding steps, and the task statement; the reply must be a single corrected shell command. Cases beyond `--max-fix-cases` are excluded by a seeded random sample and marked `sampled-out` in the report. Arm B replays the prefix in its own fresh sandbox and executes the corrected command; `failureVanished` = exit 0 with the failure signature absent.
79
79
 
80
+ ### Iterative fix loop (`--fix loop`)
81
+
82
+ `--fix loop` (in `src/replay-fix-loop.ts`) replaces the single shot with an execution-feedback loop of up to `--fix-attempts` attempts (default 3) per case:
83
+
84
+ - Attempt 1 is byte-identical to the one-shot prompt, so the report's `fixFlipAttempt1` stays directly comparable to `--fix generate`.
85
+ - When an arm fails (nonzero exit or the failure signature persists) or the model call itself fails, the next prompt carries every prior command with its REAL executed stdout/stderr (clipped tails), never a paraphrase.
86
+ - Retries may answer with a short script — at most 5 commands in one fenced block — executed as ONE `/bin/sh` unit; a longer script is rejected as a failed attempt and the rejection feeds the next prompt.
87
+ - Every attempt runs in its own fresh sandbox with the same replayed prefix. A used sandbox is never mutated mid-arm, so a flip always proves the corrected step against the recorded prefix state, not against debris from an earlier attempt. Wall-time cost is one extra prefix replay per attempt.
88
+
89
+ The report adds `fixFlipAttempt1` (flips at attempt 1 over cases whose attempt 1 executed), `flipsByAttempt`, the full per-attempt trail on each case row (`fix.attempts`), and one `armB-attempt<N>-result.json` per executed attempt (`armB-result.json` keeps the last executed attempt — the flipped one when the loop flips).
90
+
80
91
  Honest-reporting notes baked into the report:
81
92
 
82
93
  - A gold step whose recorded observation carries **no returncode** can never satisfy "arm A reproduces the recorded returncode": it counts against the replayability rate and the per-case table says why.
@@ -99,6 +110,12 @@ const { invocation, fixCommand, verdict } = await replayVerifyFinding(
99
110
 
100
111
  The subject grammar is the analyst benchmark's `incorrect-steps-<first>-<last>-<escaped|unescaped>-consequence-<step>`; `--at` is the finding's **first** incorrect step (the finding's claim, which may differ from the gold label). The wire resolves the trajectory across the given corpora, generates the arm-B fix through the same one-call generator (or accepts a pre-supplied `fixCommand`), and returns the full `ReplayVerdict`. It throws with a precise reason when the finding cannot be replayed (malformed subject, unknown trajectory, non-SWE case, step out of range) — the product surfaces that reason instead of a proof.
101
112
 
113
+ ## Product surface — verified findings
114
+
115
+ `traces verify-findings` (and `traces analyze --verify-findings`) runs this proof per recorded analyst finding and annotates each with `reproduced | fix-flipped | divergent | not-replayable` plus a receipt directory.
116
+ Unlike the wire above it accepts the shapes analysts actually emit (`incorrect-step-<n>` subjects, `metadata.block_first_step`, `trace://` evidence refs) and never throws on a finding-shaped dead end — the dead end becomes the finding's honest `not-replayable` receipt.
117
+ See [Verified findings](./trace-analysts.md#verified-findings-executed-replay).
118
+
102
119
  ## Orchestrator prerequisites
103
120
 
104
121
  replay-verify talks to a sandbox API (`--base-url`); in local development that is the sandbox SDK adapter in front of an orchestrator running the docker driver.
@@ -68,9 +68,12 @@ traces analyze --all --last 20 --analyzer hodoscope
68
68
 
69
69
  # Run any installed command that accepts an OpenInference file path.
70
70
  traces analyze --last 1 --analyzer my-trace-tool
71
+
72
+ # One-shot prime-RLM analysis through a local OpenAI-compatible bridge.
73
+ traces analyze --last 1 --analyzer prime --analyzer-prompt "find unsupported completion claims"
71
74
  ```
72
75
 
73
- `--model` is forwarded to the built-in model-assisted analysts, HALO, and Hodoscope.
76
+ `--model` is forwarded to the built-in model-assisted analysts, HALO, Hodoscope, and prime.
74
77
  HALO and Hodoscope use their own provider clients and credentials.
75
78
  The Hodoscope adapter pins version `0.2.4` and uses Python 3.11 through `uvx`.
76
79
 
@@ -87,6 +90,41 @@ Raw JSON is still a `report`.
87
90
  Only an adapter that validates the full finding shape may return `findings`.
88
91
  Hodoscope always returns `discovery`, and each candidate has `status: 'needs_review'` plus its source trace and span.
89
92
 
93
+ ### Prime engine
94
+
95
+ `--analyzer prime` runs a one-shot analyst over the emitted OTLP artifact through an OpenAI-compatible bridge, such as cli-bridge's prime backend.
96
+ Unlike the `--llm` analysts, which drill into the trace with paged tools, prime has no REPL and no trace tools: the full span projection is inlined into a single prompt as JSON.
97
+ Oversized projections are re-rendered with a per-attribute character cap; if the projection still exceeds the inline budget the engine fails loud instead of silently dropping spans.
98
+
99
+ Prerequisites:
100
+
101
+ - A running bridge that accepts `POST /v1/chat/completions` and routes the configured model to the prime backend.
102
+ - No key handling here: the bridge owns provider credentials.
103
+
104
+ Configuration (flags first, then environment, then defaults):
105
+
106
+ | Setting | Source | Default |
107
+ | --- | --- | --- |
108
+ | Bridge root URL | `TRACES_PRIME_BRIDGE_URL` | `http://localhost:4181` |
109
+ | Model | `--model`, then `TRACES_PRIME_MODEL` | `prime/zai/glm-5.2` |
110
+ | Per-call deadline | `TRACES_PRIME_TIMEOUT_MS` | `1200000` (20 min) |
111
+ | Question | `--analyzer-prompt` | a general diagnosis question |
112
+
113
+ ```bash
114
+ traces analyze --last 1 --analyzer prime
115
+ TRACES_PRIME_BRIDGE_URL=http://localhost:4181 traces analyze --last 1 --analyzer prime --analyzer-prompt "find unsupported completion claims"
116
+ ```
117
+
118
+ Output expectations:
119
+
120
+ - The reply contract is one fenced JSON block of short strings citing span ids verbatim; a structurally malformed reply gets exactly one bounded repair turn that carries the malformed reply and the contract, never the trajectory.
121
+ - A still-malformed reply after repair is a failed result (`ok: false`) with the raw reply preserved for inspection; one failed engine never discards the other results.
122
+ - Valid rows become full `findings` with `trace://` span evidence, validated against the artifact like every other findings-kind engine; rows citing unknown or ambiguous span ids are rejected with a recorded reason.
123
+ - Zero findings from a well-formed reply is an honest null, not a failure.
124
+ - Bridge-reported token usage and call counts are recorded in the result output; cost stays uncaptured because this adapter has no pricing table.
125
+
126
+ The scored prime-vs-dspy comparison — same trajectories, same scoring — lives in `@tangle-network/agent-eval`'s analyst benchmark (`runAnalystBenchmark`); this engine is the capture-side entry point, not the scoreboard.
127
+
90
128
  ## Write one analyst
91
129
 
92
130
  An analyst receives a paged trace store and returns typed findings.
@@ -210,6 +248,34 @@ console.log(renderAnalystBenchmarkMarkdown(result))
210
248
  Public labels test the measurement code, not the quality of every built-in analyst automatically.
211
249
  A real quality claim requires running the analyst over the corresponding trajectories, retaining all rows, and comparing it with named alternatives at equal model and request limits.
212
250
 
251
+ ## Verified findings (executed replay)
252
+
253
+ An analyst finding is a cited claim until something executes it.
254
+ `traces analyze --verify-findings` (and the standalone `traces verify-findings`) replays each finding's trajectory prefix in a real sandbox, re-runs the accused step, and annotates every finding with an executed verdict:
255
+
256
+ | Verdict | Meaning |
257
+ |---|---|
258
+ | `reproduced` | the recorded failure signature (returncode + stable output substring) reproduced when the accused step re-ran |
259
+ | `fix-flipped` | reproduced, and a supplied corrected command made the failure vanish in a fresh replay |
260
+ | `divergent` | the step executed but the recorded failure did not reproduce — evidence against the finding, or against replay fidelity (the receipt carries prefix divergences so you can tell which) |
261
+ | `not-replayable` | the finding could not be executed; the receipt names the precise reason (no step subject, unknown trajectory, no docker image, submit step, …) |
262
+
263
+ ```bash
264
+ # Verify the findings an analyze run produced (marks each finding in the report):
265
+ traces analyze --last 1 --llm --verify-findings \
266
+ --replay-corpus holdout=labels.json::prepared/ --verify-out ./receipts
267
+
268
+ # Verify findings recorded earlier (e.g. extracted from an eval result.json):
269
+ traces verify-findings --findings findings.json --out ./receipts \
270
+ --steps normalized/<traj>/steps.json --image <replay-ready-image> --cwd /app
271
+ ```
272
+
273
+ Findings are matched by the shape analysts emit: subject `incorrect-step-<n>` (or the wire form `incorrect-steps-<f>-<l>-…`), `metadata.block_first_step`, and `trace://<trajectory>/…` evidence refs.
274
+ Findings accusing the same step share one executed proof; each finding still gets its own receipt directory (`receipt.json` plus, when executed, `replay-verdict.json` and `report.md` with real stdout/stderr).
275
+ Verification is execution, not generation: no LLM is involved unless you pass `--fix-command`.
276
+ A missing sandbox is an error when any finding is replayable — verification never silently skips.
277
+ Sandbox setup, execution semantics, and honest limits are in [Replay verification](./replay-verify.md).
278
+
213
279
  ## Turn findings into improvement
214
280
 
215
281
  Do not train or rewrite policy from an analyst's own prose alone.
@@ -10,6 +10,7 @@ import {
10
10
  commandRedactor,
11
11
  haloAnalyzer,
12
12
  hodoscopeAnalyzer,
13
+ primeAnalyzer,
13
14
  writeOtlpFile,
14
15
  } from '@tangle-network/traces'
15
16
 
@@ -41,7 +42,19 @@ console.log(
41
42
  : `hodoscope unavailable: ${discovery.error}`,
42
43
  )
43
44
 
44
- // 3) External REDACTOR: scrub prose with your own PII model before upload. The
45
+ // 3) PRIME ENGINE: one-shot RLM through an OpenAI-compatible bridge (run
46
+ // cli-bridge's prime backend locally, or set TRACES_PRIME_BRIDGE_URL).
47
+ // The full span projection travels inline; findings come back with
48
+ // validated trace:// span evidence.
49
+ const prime = primeAnalyzer({ defaultPrompt: 'find unsupported completion claims' })
50
+ const primeResult = await prime.analyze(otlp)
51
+ console.log(
52
+ primeResult.ok
53
+ ? `${primeResult.findings?.length ?? 0} prime finding(s)\n${primeResult.output}`
54
+ : `prime bridge unavailable: ${primeResult.error}`,
55
+ )
56
+
57
+ // 4) External REDACTOR: scrub prose with your own PII model before upload. The
45
58
  // command reads a JSON array of strings on stdin and writes the scrubbed array
46
59
  // on stdout (a 3-line wrapper adapts openai/privacy-filter's `opf`).
47
60
  const redactor = commandRedactor({ command: 'my-pii-scrubber' })
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@tangle-network/traces",
3
- "version": "0.11.4",
3
+ "version": "0.11.6",
4
4
  "description": "Point it at your coding-agent session traces (Claude Code, Codex, OpenCode, Gemini, Pi, …) and get failure-mode + efficiency findings. CLI + SDK over the @tangle-network/agent-eval analyst suite — observe live sessions, run your own analysts, redact, and upload to the Tangle Intelligence Platform.",
5
5
  "type": "module",
6
6
  "license": "MIT",
@@ -44,13 +44,14 @@
44
44
  "README.md"
45
45
  ],
46
46
  "engines": {
47
- "node": ">=22"
47
+ "node": ">=22.13.0"
48
48
  },
49
49
  "dependencies": {
50
- "@tangle-network/agent-eval": "0.142.2",
51
- "@tangle-network/agent-runtime": "0.123.0",
50
+ "@tangle-network/agent-eval": "0.145.22",
51
+ "@tangle-network/agent-interface": "1.0.0",
52
+ "@tangle-network/agent-runtime": "0.137.0",
52
53
  "@tangle-network/agent-trace-contract": "^1.0.2",
53
- "@tangle-network/sandbox": "0.17.2"
54
+ "@tangle-network/sandbox": "0.27.1"
54
55
  },
55
56
  "devDependencies": {
56
57
  "@types/node": "^22.0.0",
@@ -48,6 +48,19 @@ Use `runAnalystBenchmark` with `registryBenchmarkRunner` from `@tangle-network/a
48
48
  Use AgentRx or CodeTraceBench labels when they match the finding.
49
49
  Compare the candidate with a no-findings baseline and relevant maintained or external analysts on identical cases and limits.
50
50
 
51
+ Two shipped engines are available as comparison arms.
52
+ `--analyzer prime` inlines the whole span projection into one prompt and needs no REPL.
53
+ `--analyzer halo` and `--analyzer hodoscope` drill into the trace with paged tools.
54
+ Compare an inline arm with a paged arm to separate the prompt from the navigation.
55
+
56
+ ```bash
57
+ traces analyze --last 1 --analyzer prime
58
+ ```
59
+
60
+ Assert the served model, not the requested id.
61
+ A gateway can answer one id with another model, so a panel can report families it never ran.
62
+ Use `assertServedModel` and `assertCrossFamilyServed` from `@tangle-network/agent-eval` before you trust a per-model or cross-family number.
63
+
51
64
  Use at least 20 independent cases before comparative statistics.
52
65
  Retain every case row and report exact-step recall, precision, F1, root-step accuracy when labeled, citation coverage and validity, clean-case false positives, repeat agreement, failures, latency, calls, all token fields, and known or missing cost.
53
66
 
@@ -22,10 +22,9 @@ traces analyze --otlp results/sessions --out .traces/all.md # a directory of e
22
22
  Use `--harness` only for coding agents whose on-disk format we do not control.
23
23
  Those adapters are the legacy edge, not the way to integrate a system you own.
24
24
 
25
- A directory is a run directory: it holds the span export beside raw event, stream and
26
- SDK logs that are also `*.jsonl`. Only the OTLP files are read, and the rest are listed
27
- with what they actually hold. An `otlp/` subdirectory, when the producer made one, is
28
- read on its own.
25
+ A run directory holds the span export beside raw event, stream and SDK logs that are
26
+ also `*.jsonl`. Only the OTLP files are read; the rest are listed with what they hold.
27
+ An `otlp/` subdirectory, when present, is read on its own.
29
28
 
30
29
  Any section headed `inputs incomplete`, or carrying an `Inputs incomplete` line above
31
30
  its table, is computed from a field the trace does not carry everywhere. Report those
@@ -73,6 +72,33 @@ traces improve --harness codex --current --latest-turn --workflow \
73
72
  `improve` writes findings, evidence, a report, and spans.
74
73
  It does not edit an agent, repository, memory store, or knowledge base.
75
74
 
75
+ Write one session's durable evidence directory for a later reader:
76
+
77
+ ```bash
78
+ traces bundle --harness claude-code --session <id-or-path> --out .traces/bundle
79
+ ```
80
+
81
+ `bundle` copies the transcript, the derived report and spans, the `.evolve` ledger rows inside the session window, and a `manifest.json` with a SHA-256 per file.
82
+ It spends no model call.
83
+ A missing transcript stops the assembly.
84
+ An absent optional input is recorded in `manifest.absent` with the probed path.
85
+
86
+ ## Pick the view for the reader
87
+
88
+ That bundle is the FULL view (`manifest.view: "full"`), and it holds the whole session transcript.
89
+ Never give it to a writer that must not see an earlier conclusion: the transcript holds the text of every file the session wrote, so a check for the earlier report FILE passes while its CONTENT is still readable.
90
+
91
+ Project the writer's copy instead:
92
+
93
+ ```bash
94
+ traces bundle-view .traces/bundle --view evidence-only --out .traces/writer
95
+ ```
96
+
97
+ That view carries `derived/session-index.json`, `derived/evidence.jsonl`, and the structured `ledger/` records.
98
+ It excludes the transcripts, the report, the spans, and every prose ledger file by name, with rules and hashes in `manifest.excluded`.
99
+ Before writing, it compares each carried file against each excluded file for shared 8-word runs of prose, and drops any file that repeats one.
100
+ `manifest.view` names which copy you hold.
101
+
76
102
  ## Report
77
103
 
78
104
  - State the source (`--otlp <path>` or the harness), selected task boundary, session and span counts, and integrity warnings.