@tangle-network/traces 0.11.4 → 0.11.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +61 -2
- package/dist/{chunk-AGPOIWPQ.js → chunk-MEAEXKUV.js} +2475 -316
- package/dist/cli.js +226 -65
- package/dist/index.d.ts +738 -4
- package/dist/index.js +68 -74
- package/docs/replay-verify.md +17 -0
- package/docs/trace-analysts.md +67 -1
- package/examples/external-engines.ts +14 -1
- package/package.json +6 -5
- package/skills/build-trace-analyst/SKILL.md +13 -0
- package/skills/inspect-agent-traces/SKILL.md +30 -4
package/dist/index.js
CHANGED
|
@@ -14,6 +14,7 @@ import {
|
|
|
14
14
|
CodexTaskScopeError,
|
|
15
15
|
CopilotAdapter,
|
|
16
16
|
DEFAULT_HARNESS,
|
|
17
|
+
DEFAULT_SANDBOX_BASE_URL,
|
|
17
18
|
EMPTY_SPAN_RUN_SNAPSHOT,
|
|
18
19
|
EXPORT_RULE,
|
|
19
20
|
EmptySessionError,
|
|
@@ -24,6 +25,7 @@ import {
|
|
|
24
25
|
JsonSourceError,
|
|
25
26
|
JsonlParseError,
|
|
26
27
|
OpencodeAdapter,
|
|
28
|
+
POLICY_EVIDENCE_NOTE,
|
|
27
29
|
PartialUploadError,
|
|
28
30
|
PiAdapter,
|
|
29
31
|
QwenAdapter,
|
|
@@ -46,19 +48,26 @@ import {
|
|
|
46
48
|
analyzeSpans,
|
|
47
49
|
analyzeSteeringChain,
|
|
48
50
|
applyRedactor,
|
|
51
|
+
assembleSessionBundle,
|
|
52
|
+
assertSandboxReachable,
|
|
49
53
|
buildFixPrompt,
|
|
50
54
|
buildPolicyEvidenceRecord,
|
|
55
|
+
buildRetryFixPrompt,
|
|
51
56
|
buildSessionIndex,
|
|
52
57
|
buildSessionIndexFromRows,
|
|
53
58
|
buildSpanRunTree,
|
|
54
59
|
buildTraceFindingPacket,
|
|
55
60
|
capText,
|
|
56
61
|
chatTrajectoryToSpans,
|
|
62
|
+
classifyEvidenceOnlyPath,
|
|
57
63
|
classifyLiveActions,
|
|
58
64
|
classifyReaction,
|
|
65
|
+
classifyVerdict,
|
|
59
66
|
claudeActor,
|
|
67
|
+
clipText,
|
|
60
68
|
cmdReplayVerify,
|
|
61
69
|
cmdReplayVerifyBatch,
|
|
70
|
+
cmdVerifyFindings,
|
|
62
71
|
codexActor,
|
|
63
72
|
collectPolicyEvidence,
|
|
64
73
|
collectSessionIndex,
|
|
@@ -68,6 +77,7 @@ import {
|
|
|
68
77
|
commandRedactor,
|
|
69
78
|
conformanceOfSpans,
|
|
70
79
|
connectors,
|
|
80
|
+
countScriptCommands,
|
|
71
81
|
countSkillRunsJsonl,
|
|
72
82
|
cwdMatchesSelection,
|
|
73
83
|
defaultTraceLiveAnalysts,
|
|
@@ -76,6 +86,8 @@ import {
|
|
|
76
86
|
describeSessionRelationship,
|
|
77
87
|
dockerImagePreparer,
|
|
78
88
|
enumerateReplayableCases,
|
|
89
|
+
environmentAttributes,
|
|
90
|
+
environmentFromSpanAttributes,
|
|
79
91
|
equivalentGitCwds,
|
|
80
92
|
executeUpload,
|
|
81
93
|
exportTraceEvidenceFile,
|
|
@@ -83,12 +95,18 @@ import {
|
|
|
83
95
|
exportTraceEvidenceText,
|
|
84
96
|
extractFixCommand,
|
|
85
97
|
fileRunContextSupervisorRunReader,
|
|
98
|
+
findContextRoot,
|
|
99
|
+
findSessionTextLeaks,
|
|
86
100
|
findSpanSourceFiles,
|
|
101
|
+
findingReplayStep,
|
|
102
|
+
findingTrajectoryId,
|
|
87
103
|
forEachTreeNode,
|
|
88
104
|
generateFixCommand,
|
|
89
105
|
goldIncorrectSteps,
|
|
90
106
|
haloAnalyzer,
|
|
107
|
+
hashSessionBundleFiles,
|
|
91
108
|
hodoscopeAnalyzer,
|
|
109
|
+
httpJsonTransport,
|
|
92
110
|
importCodeTraceBench,
|
|
93
111
|
incompleteInputsNote,
|
|
94
112
|
indexSessionIdsByTrace,
|
|
@@ -103,6 +121,7 @@ import {
|
|
|
103
121
|
isSubmitAction,
|
|
104
122
|
knownHarnesses,
|
|
105
123
|
listAdapters,
|
|
124
|
+
listSessionBundleFiles,
|
|
106
125
|
loadState,
|
|
107
126
|
loadTracesConfig,
|
|
108
127
|
locateSessions,
|
|
@@ -114,6 +133,7 @@ import {
|
|
|
114
133
|
otlpRowToSpan,
|
|
115
134
|
outboundHash,
|
|
116
135
|
parseCorpusFlag,
|
|
136
|
+
parseIncorrectStepsSubject,
|
|
117
137
|
parseIsoToEpochMs,
|
|
118
138
|
parseObservationOutput,
|
|
119
139
|
parseRecordedReturncode,
|
|
@@ -121,10 +141,14 @@ import {
|
|
|
121
141
|
parseReplayVerifyArgs,
|
|
122
142
|
parseSession,
|
|
123
143
|
parseSince,
|
|
144
|
+
parseVerifyFindingsArgs,
|
|
124
145
|
planTraceAgenticRoute,
|
|
125
146
|
planUpload,
|
|
147
|
+
primeAnalyzer,
|
|
148
|
+
projectSessionBundle,
|
|
126
149
|
readCoordinationLog,
|
|
127
150
|
readFileRunContext,
|
|
151
|
+
readFindingsFile,
|
|
128
152
|
readJsonFile,
|
|
129
153
|
readJsonl,
|
|
130
154
|
readLabelEntries,
|
|
@@ -132,6 +156,7 @@ import {
|
|
|
132
156
|
readRunContextJournal,
|
|
133
157
|
readRunContextSnapshot,
|
|
134
158
|
readRunWatchSnapshot,
|
|
159
|
+
readSessionBundleManifest,
|
|
135
160
|
readSessionIndexFile,
|
|
136
161
|
readSpanRunSnapshot,
|
|
137
162
|
recordSessionCorruption,
|
|
@@ -143,11 +168,15 @@ import {
|
|
|
143
168
|
renderRunWatchSnapshot,
|
|
144
169
|
renderSpanRunSnapshot,
|
|
145
170
|
renderValidation,
|
|
171
|
+
renderVerifiedFindingsSection,
|
|
146
172
|
replayBatchUsage,
|
|
147
173
|
replayVerify,
|
|
174
|
+
replayVerifyFinding,
|
|
148
175
|
replayVerifyUsage,
|
|
149
176
|
resolveAdapter,
|
|
150
177
|
resolveCaseResources,
|
|
178
|
+
resolveFindingInvocation,
|
|
179
|
+
resolveFindingReplayability,
|
|
151
180
|
resolveOtlpInputFiles,
|
|
152
181
|
resolveRepoAttrs,
|
|
153
182
|
resolveRunContextDir,
|
|
@@ -155,6 +184,7 @@ import {
|
|
|
155
184
|
resolveSessionRepoAttrs,
|
|
156
185
|
runCommand,
|
|
157
186
|
runExternalAnalyzers,
|
|
187
|
+
runFixLoop,
|
|
158
188
|
runPipelines,
|
|
159
189
|
runReplayBatch,
|
|
160
190
|
runTotals,
|
|
@@ -175,9 +205,11 @@ import {
|
|
|
175
205
|
sessionHash,
|
|
176
206
|
sessionIdFromAttributes,
|
|
177
207
|
sessionIntegrityAttributes,
|
|
208
|
+
sessionTextSignatures,
|
|
178
209
|
skippedAnalyses,
|
|
179
210
|
span,
|
|
180
211
|
spanRunTotals,
|
|
212
|
+
stampEnvironmentAttrs,
|
|
181
213
|
stampRepoAttrs,
|
|
182
214
|
stampSessionIdentity,
|
|
183
215
|
stampSessionIntegrity,
|
|
@@ -201,6 +233,8 @@ import {
|
|
|
201
233
|
userPromptSpan,
|
|
202
234
|
validateOtlpSpans,
|
|
203
235
|
validationExitCode,
|
|
236
|
+
verifyFindings,
|
|
237
|
+
verifyFindingsUsage,
|
|
204
238
|
watchRunTarget,
|
|
205
239
|
watchSessions,
|
|
206
240
|
wrap,
|
|
@@ -209,12 +243,13 @@ import {
|
|
|
209
243
|
writeInspectionReportFile,
|
|
210
244
|
writeOtlpFile,
|
|
211
245
|
writePolicyEvidenceFile,
|
|
246
|
+
writeSessionBundleManifest,
|
|
212
247
|
writeSessionIndexFile,
|
|
213
248
|
writeTraceEvidenceExportFile,
|
|
214
249
|
writeTraceImprovementArtifacts,
|
|
215
250
|
zaiChatCaller,
|
|
216
251
|
zeroPricedCost
|
|
217
|
-
} from "./chunk-
|
|
252
|
+
} from "./chunk-MEAEXKUV.js";
|
|
218
253
|
|
|
219
254
|
// src/collect.ts
|
|
220
255
|
async function collectSessions(opts = {}) {
|
|
@@ -230,79 +265,6 @@ async function collectSessions(opts = {}) {
|
|
|
230
265
|
return out;
|
|
231
266
|
}
|
|
232
267
|
|
|
233
|
-
// src/replay-wire.ts
|
|
234
|
-
var SUBJECT_PATTERN = /^incorrect-steps-(\d+)-(\d+)-(escaped|unescaped)-consequence-(\d+)$/;
|
|
235
|
-
function parseIncorrectStepsSubject(subject) {
|
|
236
|
-
const match = SUBJECT_PATTERN.exec(subject);
|
|
237
|
-
if (!match) return null;
|
|
238
|
-
return {
|
|
239
|
-
firstStep: Number(match[1]),
|
|
240
|
-
lastStep: Number(match[2]),
|
|
241
|
-
escapeStatus: match[3],
|
|
242
|
-
consequenceStep: Number(match[4])
|
|
243
|
-
};
|
|
244
|
-
}
|
|
245
|
-
function resolveFindingInvocation(finding, corpora) {
|
|
246
|
-
const subject = parseIncorrectStepsSubject(finding.subject);
|
|
247
|
-
if (!subject) {
|
|
248
|
-
throw new Error(
|
|
249
|
-
`replay-wire: subject '${finding.subject}' is not incorrect-steps-<first>-<last>-<escaped|unescaped>-consequence-<step>`
|
|
250
|
-
);
|
|
251
|
-
}
|
|
252
|
-
const failures = [];
|
|
253
|
-
for (const corpus of corpora) {
|
|
254
|
-
const resolution = resolveCaseResources(corpus, finding.trajId);
|
|
255
|
-
if (resolution.resolved) {
|
|
256
|
-
const at = subject.firstStep;
|
|
257
|
-
const step = resolution.resources.steps.find((s) => s.step_id === at);
|
|
258
|
-
if (!step) {
|
|
259
|
-
throw new Error(
|
|
260
|
-
`replay-wire: finding step ${at} is outside ${finding.trajId} (${resolution.resources.steps.length} steps)`
|
|
261
|
-
);
|
|
262
|
-
}
|
|
263
|
-
return { resources: resolution.resources, subject, at };
|
|
264
|
-
}
|
|
265
|
-
failures.push(`${corpus.name}: ${resolution.reason}`);
|
|
266
|
-
}
|
|
267
|
-
throw new Error(
|
|
268
|
-
`replay-wire: trajectory ${finding.trajId} is not replayable in any corpus \u2014 ${failures.join("; ")}`
|
|
269
|
-
);
|
|
270
|
-
}
|
|
271
|
-
async function replayVerifyFinding(finding, options) {
|
|
272
|
-
if (options.fixCaller && options.fixCommand !== void 0) {
|
|
273
|
-
throw new Error("replay-wire: pass fixCaller or fixCommand, not both");
|
|
274
|
-
}
|
|
275
|
-
const invocation = resolveFindingInvocation(finding, options.corpora);
|
|
276
|
-
let fixCommand = options.fixCommand ?? null;
|
|
277
|
-
if (options.fixCaller) {
|
|
278
|
-
const generated = await generateFixCommand(options.fixCaller, {
|
|
279
|
-
taskStatement: invocation.resources.taskStatement,
|
|
280
|
-
steps: invocation.resources.steps,
|
|
281
|
-
k: invocation.at
|
|
282
|
-
});
|
|
283
|
-
if (!generated.succeeded) {
|
|
284
|
-
throw new Error(`replay-wire: fix generation failed \u2014 ${generated.error}`);
|
|
285
|
-
}
|
|
286
|
-
fixCommand = generated.value.command;
|
|
287
|
-
}
|
|
288
|
-
const verdict = await replayVerify({
|
|
289
|
-
stepsPath: invocation.resources.stepsPath,
|
|
290
|
-
image: invocation.resources.image,
|
|
291
|
-
at: invocation.at,
|
|
292
|
-
fixCommand: fixCommand ?? void 0,
|
|
293
|
-
cwd: invocation.resources.cwd,
|
|
294
|
-
out: options.out,
|
|
295
|
-
caseId: invocation.resources.trajId,
|
|
296
|
-
stepTimeoutMs: options.stepTimeoutMs ?? invocation.resources.recordedStepTimeoutMs ?? void 0,
|
|
297
|
-
prefixLimit: options.prefixLimit,
|
|
298
|
-
backend: options.backend,
|
|
299
|
-
apiKey: options.apiKey,
|
|
300
|
-
baseUrl: options.baseUrl,
|
|
301
|
-
onProgress: options.onProgress
|
|
302
|
-
});
|
|
303
|
-
return { invocation, fixCommand, verdict };
|
|
304
|
-
}
|
|
305
|
-
|
|
306
268
|
// src/index.ts
|
|
307
269
|
import {
|
|
308
270
|
AnalystRegistry,
|
|
@@ -329,6 +291,7 @@ export {
|
|
|
329
291
|
CopilotAdapter,
|
|
330
292
|
DEFAULT_HARNESS,
|
|
331
293
|
DEFAULT_REDACTION_RULES,
|
|
294
|
+
DEFAULT_SANDBOX_BASE_URL,
|
|
332
295
|
EMPTY_SPAN_RUN_SNAPSHOT,
|
|
333
296
|
EXPORT_RULE,
|
|
334
297
|
EmptySessionError,
|
|
@@ -339,6 +302,7 @@ export {
|
|
|
339
302
|
JsonSourceError,
|
|
340
303
|
JsonlParseError,
|
|
341
304
|
OpencodeAdapter,
|
|
305
|
+
POLICY_EVIDENCE_NOTE,
|
|
342
306
|
PartialUploadError,
|
|
343
307
|
PiAdapter,
|
|
344
308
|
QwenAdapter,
|
|
@@ -361,20 +325,27 @@ export {
|
|
|
361
325
|
analyzeSpans,
|
|
362
326
|
analyzeSteeringChain,
|
|
363
327
|
applyRedactor,
|
|
328
|
+
assembleSessionBundle,
|
|
329
|
+
assertSandboxReachable,
|
|
364
330
|
buildDefaultAnalystRegistry,
|
|
365
331
|
buildFixPrompt,
|
|
366
332
|
buildPolicyEvidenceRecord,
|
|
333
|
+
buildRetryFixPrompt,
|
|
367
334
|
buildSessionIndex,
|
|
368
335
|
buildSessionIndexFromRows,
|
|
369
336
|
buildSpanRunTree,
|
|
370
337
|
buildTraceFindingPacket,
|
|
371
338
|
capText,
|
|
372
339
|
chatTrajectoryToSpans,
|
|
340
|
+
classifyEvidenceOnlyPath,
|
|
373
341
|
classifyLiveActions,
|
|
374
342
|
classifyReaction,
|
|
343
|
+
classifyVerdict,
|
|
375
344
|
claudeActor,
|
|
345
|
+
clipText,
|
|
376
346
|
cmdReplayVerify,
|
|
377
347
|
cmdReplayVerifyBatch,
|
|
348
|
+
cmdVerifyFindings,
|
|
378
349
|
codexActor,
|
|
379
350
|
collectPolicyEvidence,
|
|
380
351
|
collectSessionIndex,
|
|
@@ -385,6 +356,7 @@ export {
|
|
|
385
356
|
commandRedactor,
|
|
386
357
|
conformanceOfSpans,
|
|
387
358
|
connectors,
|
|
359
|
+
countScriptCommands,
|
|
388
360
|
countSkillRunsJsonl,
|
|
389
361
|
createHostedClient,
|
|
390
362
|
cwdMatchesSelection,
|
|
@@ -394,6 +366,8 @@ export {
|
|
|
394
366
|
describeSessionRelationship,
|
|
395
367
|
dockerImagePreparer,
|
|
396
368
|
enumerateReplayableCases,
|
|
369
|
+
environmentAttributes,
|
|
370
|
+
environmentFromSpanAttributes,
|
|
397
371
|
equivalentGitCwds,
|
|
398
372
|
executeUpload,
|
|
399
373
|
exportTraceEvidenceFile,
|
|
@@ -401,13 +375,19 @@ export {
|
|
|
401
375
|
exportTraceEvidenceText,
|
|
402
376
|
extractFixCommand,
|
|
403
377
|
fileRunContextSupervisorRunReader,
|
|
378
|
+
findContextRoot,
|
|
379
|
+
findSessionTextLeaks,
|
|
404
380
|
findSpanSourceFiles,
|
|
381
|
+
findingReplayStep,
|
|
382
|
+
findingTrajectoryId,
|
|
405
383
|
forEachTreeNode,
|
|
406
384
|
generateFixCommand,
|
|
407
385
|
goldIncorrectSteps,
|
|
408
386
|
haloAnalyzer,
|
|
387
|
+
hashSessionBundleFiles,
|
|
409
388
|
hodoscopeAnalyzer,
|
|
410
389
|
hostedClientFromEnv,
|
|
390
|
+
httpJsonTransport,
|
|
411
391
|
importCodeTraceBench,
|
|
412
392
|
incompleteInputsNote,
|
|
413
393
|
indexSessionIdsByTrace,
|
|
@@ -422,6 +402,7 @@ export {
|
|
|
422
402
|
isSubmitAction,
|
|
423
403
|
knownHarnesses,
|
|
424
404
|
listAdapters,
|
|
405
|
+
listSessionBundleFiles,
|
|
425
406
|
loadState,
|
|
426
407
|
loadTracesConfig,
|
|
427
408
|
locateSessions,
|
|
@@ -442,10 +423,14 @@ export {
|
|
|
442
423
|
parseReplayVerifyArgs,
|
|
443
424
|
parseSession,
|
|
444
425
|
parseSince,
|
|
426
|
+
parseVerifyFindingsArgs,
|
|
445
427
|
planTraceAgenticRoute,
|
|
446
428
|
planUpload,
|
|
429
|
+
primeAnalyzer,
|
|
430
|
+
projectSessionBundle,
|
|
447
431
|
readCoordinationLog,
|
|
448
432
|
readFileRunContext,
|
|
433
|
+
readFindingsFile,
|
|
449
434
|
readJsonFile,
|
|
450
435
|
readJsonl,
|
|
451
436
|
readLabelEntries,
|
|
@@ -453,6 +438,7 @@ export {
|
|
|
453
438
|
readRunContextJournal,
|
|
454
439
|
readRunContextSnapshot,
|
|
455
440
|
readRunWatchSnapshot,
|
|
441
|
+
readSessionBundleManifest,
|
|
456
442
|
readSessionIndexFile,
|
|
457
443
|
readSpanRunSnapshot,
|
|
458
444
|
recordSessionCorruption,
|
|
@@ -466,6 +452,7 @@ export {
|
|
|
466
452
|
renderRunWatchSnapshot,
|
|
467
453
|
renderSpanRunSnapshot,
|
|
468
454
|
renderValidation,
|
|
455
|
+
renderVerifiedFindingsSection,
|
|
469
456
|
replayBatchUsage,
|
|
470
457
|
replayVerify,
|
|
471
458
|
replayVerifyFinding,
|
|
@@ -473,6 +460,7 @@ export {
|
|
|
473
460
|
resolveAdapter,
|
|
474
461
|
resolveCaseResources,
|
|
475
462
|
resolveFindingInvocation,
|
|
463
|
+
resolveFindingReplayability,
|
|
476
464
|
resolveOtlpInputFiles,
|
|
477
465
|
resolveRepoAttrs,
|
|
478
466
|
resolveRunContextDir,
|
|
@@ -480,6 +468,7 @@ export {
|
|
|
480
468
|
resolveSessionRepoAttrs,
|
|
481
469
|
runCommand,
|
|
482
470
|
runExternalAnalyzers,
|
|
471
|
+
runFixLoop,
|
|
483
472
|
runPipelines,
|
|
484
473
|
runReplayBatch,
|
|
485
474
|
runTotals,
|
|
@@ -500,9 +489,11 @@ export {
|
|
|
500
489
|
sessionHash,
|
|
501
490
|
sessionIdFromAttributes,
|
|
502
491
|
sessionIntegrityAttributes,
|
|
492
|
+
sessionTextSignatures,
|
|
503
493
|
skippedAnalyses,
|
|
504
494
|
span,
|
|
505
495
|
spanRunTotals,
|
|
496
|
+
stampEnvironmentAttrs,
|
|
506
497
|
stampRepoAttrs,
|
|
507
498
|
stampSessionIdentity,
|
|
508
499
|
stampSessionIntegrity,
|
|
@@ -526,6 +517,8 @@ export {
|
|
|
526
517
|
userPromptSpan,
|
|
527
518
|
validateOtlpSpans,
|
|
528
519
|
validationExitCode,
|
|
520
|
+
verifyFindings,
|
|
521
|
+
verifyFindingsUsage,
|
|
529
522
|
watchRunTarget,
|
|
530
523
|
watchSessions,
|
|
531
524
|
wrap,
|
|
@@ -534,6 +527,7 @@ export {
|
|
|
534
527
|
writeInspectionReportFile,
|
|
535
528
|
writeOtlpFile,
|
|
536
529
|
writePolicyEvidenceFile,
|
|
530
|
+
writeSessionBundleManifest,
|
|
537
531
|
writeSessionIndexFile,
|
|
538
532
|
writeTraceEvidenceExportFile,
|
|
539
533
|
writeTraceImprovementArtifacts,
|
package/docs/replay-verify.md
CHANGED
|
@@ -77,6 +77,17 @@ A corpus is a gold label file plus a prepared directory (`normalized/<traj_id>/s
|
|
|
77
77
|
|
|
78
78
|
Fix generation (`--fix generate`) runs ONE chat completion per arm-A-reproduced case against an OpenAI-compatible endpoint (default: glm-5.2 on z.ai). The prompt carries the gold step's action and observation, ±3 surrounding steps, and the task statement; the reply must be a single corrected shell command. Cases beyond `--max-fix-cases` are excluded by a seeded random sample and marked `sampled-out` in the report. Arm B replays the prefix in its own fresh sandbox and executes the corrected command; `failureVanished` = exit 0 with the failure signature absent.
|
|
79
79
|
|
|
80
|
+
### Iterative fix loop (`--fix loop`)
|
|
81
|
+
|
|
82
|
+
`--fix loop` (in `src/replay-fix-loop.ts`) replaces the single shot with an execution-feedback loop of up to `--fix-attempts` attempts (default 3) per case:
|
|
83
|
+
|
|
84
|
+
- Attempt 1 is byte-identical to the one-shot prompt, so the report's `fixFlipAttempt1` stays directly comparable to `--fix generate`.
|
|
85
|
+
- When an arm fails (nonzero exit or the failure signature persists) or the model call itself fails, the next prompt carries every prior command with its REAL executed stdout/stderr (clipped tails), never a paraphrase.
|
|
86
|
+
- Retries may answer with a short script — at most 5 commands in one fenced block — executed as ONE `/bin/sh` unit; a longer script is rejected as a failed attempt and the rejection feeds the next prompt.
|
|
87
|
+
- Every attempt runs in its own fresh sandbox with the same replayed prefix. A used sandbox is never mutated mid-arm, so a flip always proves the corrected step against the recorded prefix state, not against debris from an earlier attempt. Wall-time cost is one extra prefix replay per attempt.
|
|
88
|
+
|
|
89
|
+
The report adds `fixFlipAttempt1` (flips at attempt 1 over cases whose attempt 1 executed), `flipsByAttempt`, the full per-attempt trail on each case row (`fix.attempts`), and one `armB-attempt<N>-result.json` per executed attempt (`armB-result.json` keeps the last executed attempt — the flipped one when the loop flips).
|
|
90
|
+
|
|
80
91
|
Honest-reporting notes baked into the report:
|
|
81
92
|
|
|
82
93
|
- A gold step whose recorded observation carries **no returncode** can never satisfy "arm A reproduces the recorded returncode": it counts against the replayability rate and the per-case table says why.
|
|
@@ -99,6 +110,12 @@ const { invocation, fixCommand, verdict } = await replayVerifyFinding(
|
|
|
99
110
|
|
|
100
111
|
The subject grammar is the analyst benchmark's `incorrect-steps-<first>-<last>-<escaped|unescaped>-consequence-<step>`; `--at` is the finding's **first** incorrect step (the finding's claim, which may differ from the gold label). The wire resolves the trajectory across the given corpora, generates the arm-B fix through the same one-call generator (or accepts a pre-supplied `fixCommand`), and returns the full `ReplayVerdict`. It throws with a precise reason when the finding cannot be replayed (malformed subject, unknown trajectory, non-SWE case, step out of range) — the product surfaces that reason instead of a proof.
|
|
101
112
|
|
|
113
|
+
## Product surface — verified findings
|
|
114
|
+
|
|
115
|
+
`traces verify-findings` (and `traces analyze --verify-findings`) runs this proof per recorded analyst finding and annotates each with `reproduced | fix-flipped | divergent | not-replayable` plus a receipt directory.
|
|
116
|
+
Unlike the wire above it accepts the shapes analysts actually emit (`incorrect-step-<n>` subjects, `metadata.block_first_step`, `trace://` evidence refs) and never throws on a finding-shaped dead end — the dead end becomes the finding's honest `not-replayable` receipt.
|
|
117
|
+
See [Verified findings](./trace-analysts.md#verified-findings-executed-replay).
|
|
118
|
+
|
|
102
119
|
## Orchestrator prerequisites
|
|
103
120
|
|
|
104
121
|
replay-verify talks to a sandbox API (`--base-url`); in local development that is the sandbox SDK adapter in front of an orchestrator running the docker driver.
|
package/docs/trace-analysts.md
CHANGED
|
@@ -68,9 +68,12 @@ traces analyze --all --last 20 --analyzer hodoscope
|
|
|
68
68
|
|
|
69
69
|
# Run any installed command that accepts an OpenInference file path.
|
|
70
70
|
traces analyze --last 1 --analyzer my-trace-tool
|
|
71
|
+
|
|
72
|
+
# One-shot prime-RLM analysis through a local OpenAI-compatible bridge.
|
|
73
|
+
traces analyze --last 1 --analyzer prime --analyzer-prompt "find unsupported completion claims"
|
|
71
74
|
```
|
|
72
75
|
|
|
73
|
-
`--model` is forwarded to the built-in model-assisted analysts, HALO, and
|
|
76
|
+
`--model` is forwarded to the built-in model-assisted analysts, HALO, Hodoscope, and prime.
|
|
74
77
|
HALO and Hodoscope use their own provider clients and credentials.
|
|
75
78
|
The Hodoscope adapter pins version `0.2.4` and uses Python 3.11 through `uvx`.
|
|
76
79
|
|
|
@@ -87,6 +90,41 @@ Raw JSON is still a `report`.
|
|
|
87
90
|
Only an adapter that validates the full finding shape may return `findings`.
|
|
88
91
|
Hodoscope always returns `discovery`, and each candidate has `status: 'needs_review'` plus its source trace and span.
|
|
89
92
|
|
|
93
|
+
### Prime engine
|
|
94
|
+
|
|
95
|
+
`--analyzer prime` runs a one-shot analyst over the emitted OTLP artifact through an OpenAI-compatible bridge, such as cli-bridge's prime backend.
|
|
96
|
+
Unlike the `--llm` analysts, which drill into the trace with paged tools, prime has no REPL and no trace tools: the full span projection is inlined into a single prompt as JSON.
|
|
97
|
+
Oversized projections are re-rendered with a per-attribute character cap; if the projection still exceeds the inline budget the engine fails loud instead of silently dropping spans.
|
|
98
|
+
|
|
99
|
+
Prerequisites:
|
|
100
|
+
|
|
101
|
+
- A running bridge that accepts `POST /v1/chat/completions` and routes the configured model to the prime backend.
|
|
102
|
+
- No key handling here: the bridge owns provider credentials.
|
|
103
|
+
|
|
104
|
+
Configuration (flags first, then environment, then defaults):
|
|
105
|
+
|
|
106
|
+
| Setting | Source | Default |
|
|
107
|
+
| --- | --- | --- |
|
|
108
|
+
| Bridge root URL | `TRACES_PRIME_BRIDGE_URL` | `http://localhost:4181` |
|
|
109
|
+
| Model | `--model`, then `TRACES_PRIME_MODEL` | `prime/zai/glm-5.2` |
|
|
110
|
+
| Per-call deadline | `TRACES_PRIME_TIMEOUT_MS` | `1200000` (20 min) |
|
|
111
|
+
| Question | `--analyzer-prompt` | a general diagnosis question |
|
|
112
|
+
|
|
113
|
+
```bash
|
|
114
|
+
traces analyze --last 1 --analyzer prime
|
|
115
|
+
TRACES_PRIME_BRIDGE_URL=http://localhost:4181 traces analyze --last 1 --analyzer prime --analyzer-prompt "find unsupported completion claims"
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
Output expectations:
|
|
119
|
+
|
|
120
|
+
- The reply contract is one fenced JSON block of short strings citing span ids verbatim; a structurally malformed reply gets exactly one bounded repair turn that carries the malformed reply and the contract, never the trajectory.
|
|
121
|
+
- A still-malformed reply after repair is a failed result (`ok: false`) with the raw reply preserved for inspection; one failed engine never discards the other results.
|
|
122
|
+
- Valid rows become full `findings` with `trace://` span evidence, validated against the artifact like every other findings-kind engine; rows citing unknown or ambiguous span ids are rejected with a recorded reason.
|
|
123
|
+
- Zero findings from a well-formed reply is an honest null, not a failure.
|
|
124
|
+
- Bridge-reported token usage and call counts are recorded in the result output; cost stays uncaptured because this adapter has no pricing table.
|
|
125
|
+
|
|
126
|
+
The scored prime-vs-dspy comparison — same trajectories, same scoring — lives in `@tangle-network/agent-eval`'s analyst benchmark (`runAnalystBenchmark`); this engine is the capture-side entry point, not the scoreboard.
|
|
127
|
+
|
|
90
128
|
## Write one analyst
|
|
91
129
|
|
|
92
130
|
An analyst receives a paged trace store and returns typed findings.
|
|
@@ -210,6 +248,34 @@ console.log(renderAnalystBenchmarkMarkdown(result))
|
|
|
210
248
|
Public labels test the measurement code, not the quality of every built-in analyst automatically.
|
|
211
249
|
A real quality claim requires running the analyst over the corresponding trajectories, retaining all rows, and comparing it with named alternatives at equal model and request limits.
|
|
212
250
|
|
|
251
|
+
## Verified findings (executed replay)
|
|
252
|
+
|
|
253
|
+
An analyst finding is a cited claim until something executes it.
|
|
254
|
+
`traces analyze --verify-findings` (and the standalone `traces verify-findings`) replays each finding's trajectory prefix in a real sandbox, re-runs the accused step, and annotates every finding with an executed verdict:
|
|
255
|
+
|
|
256
|
+
| Verdict | Meaning |
|
|
257
|
+
|---|---|
|
|
258
|
+
| `reproduced` | the recorded failure signature (returncode + stable output substring) reproduced when the accused step re-ran |
|
|
259
|
+
| `fix-flipped` | reproduced, and a supplied corrected command made the failure vanish in a fresh replay |
|
|
260
|
+
| `divergent` | the step executed but the recorded failure did not reproduce — evidence against the finding, or against replay fidelity (the receipt carries prefix divergences so you can tell which) |
|
|
261
|
+
| `not-replayable` | the finding could not be executed; the receipt names the precise reason (no step subject, unknown trajectory, no docker image, submit step, …) |
|
|
262
|
+
|
|
263
|
+
```bash
|
|
264
|
+
# Verify the findings an analyze run produced (marks each finding in the report):
|
|
265
|
+
traces analyze --last 1 --llm --verify-findings \
|
|
266
|
+
--replay-corpus holdout=labels.json::prepared/ --verify-out ./receipts
|
|
267
|
+
|
|
268
|
+
# Verify findings recorded earlier (e.g. extracted from an eval result.json):
|
|
269
|
+
traces verify-findings --findings findings.json --out ./receipts \
|
|
270
|
+
--steps normalized/<traj>/steps.json --image <replay-ready-image> --cwd /app
|
|
271
|
+
```
|
|
272
|
+
|
|
273
|
+
Findings are matched by the shape analysts emit: subject `incorrect-step-<n>` (or the wire form `incorrect-steps-<f>-<l>-…`), `metadata.block_first_step`, and `trace://<trajectory>/…` evidence refs.
|
|
274
|
+
Findings accusing the same step share one executed proof; each finding still gets its own receipt directory (`receipt.json` plus, when executed, `replay-verdict.json` and `report.md` with real stdout/stderr).
|
|
275
|
+
Verification is execution, not generation: no LLM is involved unless you pass `--fix-command`.
|
|
276
|
+
A missing sandbox is an error when any finding is replayable — verification never silently skips.
|
|
277
|
+
Sandbox setup, execution semantics, and honest limits are in [Replay verification](./replay-verify.md).
|
|
278
|
+
|
|
213
279
|
## Turn findings into improvement
|
|
214
280
|
|
|
215
281
|
Do not train or rewrite policy from an analyst's own prose alone.
|
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
commandRedactor,
|
|
11
11
|
haloAnalyzer,
|
|
12
12
|
hodoscopeAnalyzer,
|
|
13
|
+
primeAnalyzer,
|
|
13
14
|
writeOtlpFile,
|
|
14
15
|
} from '@tangle-network/traces'
|
|
15
16
|
|
|
@@ -41,7 +42,19 @@ console.log(
|
|
|
41
42
|
: `hodoscope unavailable: ${discovery.error}`,
|
|
42
43
|
)
|
|
43
44
|
|
|
44
|
-
// 3)
|
|
45
|
+
// 3) PRIME ENGINE: one-shot RLM through an OpenAI-compatible bridge (run
|
|
46
|
+
// cli-bridge's prime backend locally, or set TRACES_PRIME_BRIDGE_URL).
|
|
47
|
+
// The full span projection travels inline; findings come back with
|
|
48
|
+
// validated trace:// span evidence.
|
|
49
|
+
const prime = primeAnalyzer({ defaultPrompt: 'find unsupported completion claims' })
|
|
50
|
+
const primeResult = await prime.analyze(otlp)
|
|
51
|
+
console.log(
|
|
52
|
+
primeResult.ok
|
|
53
|
+
? `${primeResult.findings?.length ?? 0} prime finding(s)\n${primeResult.output}`
|
|
54
|
+
: `prime bridge unavailable: ${primeResult.error}`,
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
// 4) External REDACTOR: scrub prose with your own PII model before upload. The
|
|
45
58
|
// command reads a JSON array of strings on stdin and writes the scrubbed array
|
|
46
59
|
// on stdout (a 3-line wrapper adapts openai/privacy-filter's `opf`).
|
|
47
60
|
const redactor = commandRedactor({ command: 'my-pii-scrubber' })
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tangle-network/traces",
|
|
3
|
-
"version": "0.11.
|
|
3
|
+
"version": "0.11.6",
|
|
4
4
|
"description": "Point it at your coding-agent session traces (Claude Code, Codex, OpenCode, Gemini, Pi, …) and get failure-mode + efficiency findings. CLI + SDK over the @tangle-network/agent-eval analyst suite — observe live sessions, run your own analysts, redact, and upload to the Tangle Intelligence Platform.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -44,13 +44,14 @@
|
|
|
44
44
|
"README.md"
|
|
45
45
|
],
|
|
46
46
|
"engines": {
|
|
47
|
-
"node": ">=22"
|
|
47
|
+
"node": ">=22.13.0"
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
|
-
"@tangle-network/agent-eval": "0.
|
|
51
|
-
"@tangle-network/agent-
|
|
50
|
+
"@tangle-network/agent-eval": "0.145.22",
|
|
51
|
+
"@tangle-network/agent-interface": "1.0.0",
|
|
52
|
+
"@tangle-network/agent-runtime": "0.137.0",
|
|
52
53
|
"@tangle-network/agent-trace-contract": "^1.0.2",
|
|
53
|
-
"@tangle-network/sandbox": "0.
|
|
54
|
+
"@tangle-network/sandbox": "0.27.1"
|
|
54
55
|
},
|
|
55
56
|
"devDependencies": {
|
|
56
57
|
"@types/node": "^22.0.0",
|
|
@@ -48,6 +48,19 @@ Use `runAnalystBenchmark` with `registryBenchmarkRunner` from `@tangle-network/a
|
|
|
48
48
|
Use AgentRx or CodeTraceBench labels when they match the finding.
|
|
49
49
|
Compare the candidate with a no-findings baseline and relevant maintained or external analysts on identical cases and limits.
|
|
50
50
|
|
|
51
|
+
Two shipped engines are available as comparison arms.
|
|
52
|
+
`--analyzer prime` inlines the whole span projection into one prompt and needs no REPL.
|
|
53
|
+
`--analyzer halo` and `--analyzer hodoscope` drill into the trace with paged tools.
|
|
54
|
+
Compare an inline arm with a paged arm to separate the prompt from the navigation.
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
traces analyze --last 1 --analyzer prime
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
Assert the served model, not the requested id.
|
|
61
|
+
A gateway can answer one id with another model, so a panel can report families it never ran.
|
|
62
|
+
Use `assertServedModel` and `assertCrossFamilyServed` from `@tangle-network/agent-eval` before you trust a per-model or cross-family number.
|
|
63
|
+
|
|
51
64
|
Use at least 20 independent cases before comparative statistics.
|
|
52
65
|
Retain every case row and report exact-step recall, precision, F1, root-step accuracy when labeled, citation coverage and validity, clean-case false positives, repeat agreement, failures, latency, calls, all token fields, and known or missing cost.
|
|
53
66
|
|
|
@@ -22,10 +22,9 @@ traces analyze --otlp results/sessions --out .traces/all.md # a directory of e
|
|
|
22
22
|
Use `--harness` only for coding agents whose on-disk format we do not control.
|
|
23
23
|
Those adapters are the legacy edge, not the way to integrate a system you own.
|
|
24
24
|
|
|
25
|
-
A
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
read on its own.
|
|
25
|
+
A run directory holds the span export beside raw event, stream and SDK logs that are
|
|
26
|
+
also `*.jsonl`. Only the OTLP files are read; the rest are listed with what they hold.
|
|
27
|
+
An `otlp/` subdirectory, when present, is read on its own.
|
|
29
28
|
|
|
30
29
|
Any section headed `inputs incomplete`, or carrying an `Inputs incomplete` line above
|
|
31
30
|
its table, is computed from a field the trace does not carry everywhere. Report those
|
|
@@ -73,6 +72,33 @@ traces improve --harness codex --current --latest-turn --workflow \
|
|
|
73
72
|
`improve` writes findings, evidence, a report, and spans.
|
|
74
73
|
It does not edit an agent, repository, memory store, or knowledge base.
|
|
75
74
|
|
|
75
|
+
Write one session's durable evidence directory for a later reader:
|
|
76
|
+
|
|
77
|
+
```bash
|
|
78
|
+
traces bundle --harness claude-code --session <id-or-path> --out .traces/bundle
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
`bundle` copies the transcript, the derived report and spans, the `.evolve` ledger rows inside the session window, and a `manifest.json` with a SHA-256 per file.
|
|
82
|
+
It spends no model call.
|
|
83
|
+
A missing transcript stops the assembly.
|
|
84
|
+
An absent optional input is recorded in `manifest.absent` with the probed path.
|
|
85
|
+
|
|
86
|
+
## Pick the view for the reader
|
|
87
|
+
|
|
88
|
+
That bundle is the FULL view (`manifest.view: "full"`), and it holds the whole session transcript.
|
|
89
|
+
Never give it to a writer that must not see an earlier conclusion: the transcript holds the text of every file the session wrote, so a check for the earlier report FILE passes while its CONTENT is still readable.
|
|
90
|
+
|
|
91
|
+
Project the writer's copy instead:
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
traces bundle-view .traces/bundle --view evidence-only --out .traces/writer
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
That view carries `derived/session-index.json`, `derived/evidence.jsonl`, and the structured `ledger/` records.
|
|
98
|
+
It excludes the transcripts, the report, the spans, and every prose ledger file by name, with rules and hashes in `manifest.excluded`.
|
|
99
|
+
Before writing, it compares each carried file against each excluded file for shared 8-word runs of prose, and drops any file that repeats one.
|
|
100
|
+
`manifest.view` names which copy you hold.
|
|
101
|
+
|
|
76
102
|
## Report
|
|
77
103
|
|
|
78
104
|
- State the source (`--otlp <path>` or the harness), selected task boundary, session and span counts, and integrity warnings.
|