@tangle-network/traces 0.10.3 → 0.11.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +220 -15
- package/dist/{chunk-LC66I5TE.js → chunk-UY3M7XUF.js} +3350 -689
- package/dist/cli.js +278 -30
- package/dist/index.d.ts +1217 -9
- package/dist/index.js +129 -3
- package/docs/replay-verify.md +55 -0
- package/package.json +6 -2
- package/skills/inspect-agent-traces/SKILL.md +31 -2
package/dist/cli.js
CHANGED
|
@@ -6,17 +6,20 @@ import {
|
|
|
6
6
|
appendAll,
|
|
7
7
|
buildPolicyEvidenceRecord,
|
|
8
8
|
buildSessionIndexFromRows,
|
|
9
|
+
cmdReplayVerify,
|
|
9
10
|
collectSessionSelection,
|
|
10
11
|
commandAnalyzer,
|
|
11
12
|
commandRedactor,
|
|
12
13
|
describeSessionRelationship,
|
|
13
14
|
executeUpload,
|
|
14
15
|
exportTraceEvidenceFile,
|
|
16
|
+
fileRunContextSupervisorRunReader,
|
|
15
17
|
haloAnalyzer,
|
|
16
18
|
hodoscopeAnalyzer,
|
|
17
19
|
importCodeTraceBench,
|
|
18
20
|
indexSessionIdsByTrace,
|
|
19
21
|
inspectSessionIndex,
|
|
22
|
+
isFileRunContextDir,
|
|
20
23
|
knownHarnesses,
|
|
21
24
|
loadTracesConfig,
|
|
22
25
|
locateSessions,
|
|
@@ -24,9 +27,12 @@ import {
|
|
|
24
27
|
parseSession,
|
|
25
28
|
parseSince,
|
|
26
29
|
planUpload,
|
|
30
|
+
readOtlpInput,
|
|
27
31
|
readSessionIndexFile,
|
|
28
32
|
renderInspectionReport,
|
|
33
|
+
renderValidation,
|
|
29
34
|
resolveAdapter,
|
|
35
|
+
resolveRunWatchTarget,
|
|
30
36
|
runTraceImprovement,
|
|
31
37
|
runTraceInvestigation,
|
|
32
38
|
saveReport,
|
|
@@ -39,18 +45,20 @@ import {
|
|
|
39
45
|
streamSessions,
|
|
40
46
|
totalAgenticFailureMessage,
|
|
41
47
|
traceStreamEventsFromSpans,
|
|
48
|
+
validationExitCode,
|
|
49
|
+
watchRunTarget,
|
|
42
50
|
watchSessions,
|
|
43
51
|
writeInspectionReportFile,
|
|
44
52
|
writeOtlpFile,
|
|
45
53
|
writePolicyEvidenceFile,
|
|
46
54
|
writeSessionIndexFile,
|
|
47
55
|
writeTraceEvidenceExportFile
|
|
48
|
-
} from "./chunk-
|
|
56
|
+
} from "./chunk-UY3M7XUF.js";
|
|
49
57
|
|
|
50
58
|
// src/cli.ts
|
|
51
59
|
import { readFileSync } from "fs";
|
|
52
|
-
import { readFile, stat } from "fs/promises";
|
|
53
|
-
import { basename, resolve } from "path";
|
|
60
|
+
import { readdir, readFile, stat } from "fs/promises";
|
|
61
|
+
import { basename, join, resolve } from "path";
|
|
54
62
|
import {
|
|
55
63
|
analyzeSupervisorRun,
|
|
56
64
|
findSupervisorRunDirs,
|
|
@@ -81,6 +89,7 @@ function parseArgs(argv) {
|
|
|
81
89
|
maxWorkflowSessions: 100,
|
|
82
90
|
llm: false,
|
|
83
91
|
interval: 5,
|
|
92
|
+
intervalExplicit: false,
|
|
84
93
|
window: 30,
|
|
85
94
|
minLoop: 3,
|
|
86
95
|
dryRun: false,
|
|
@@ -88,6 +97,7 @@ function parseArgs(argv) {
|
|
|
88
97
|
noContent: false,
|
|
89
98
|
analyzers: [],
|
|
90
99
|
replay: false,
|
|
100
|
+
once: false,
|
|
91
101
|
noSpans: false,
|
|
92
102
|
noFindings: false,
|
|
93
103
|
attrs: [],
|
|
@@ -149,6 +159,9 @@ function parseArgs(argv) {
|
|
|
149
159
|
case "--otlp":
|
|
150
160
|
a.otlp = next();
|
|
151
161
|
break;
|
|
162
|
+
case "--otlp-out":
|
|
163
|
+
a.otlpOut = next();
|
|
164
|
+
break;
|
|
152
165
|
case "--llm":
|
|
153
166
|
a.llm = true;
|
|
154
167
|
break;
|
|
@@ -174,6 +187,7 @@ function parseArgs(argv) {
|
|
|
174
187
|
}
|
|
175
188
|
case "--interval":
|
|
176
189
|
a.interval = Number(next());
|
|
190
|
+
a.intervalExplicit = true;
|
|
177
191
|
break;
|
|
178
192
|
case "--window":
|
|
179
193
|
a.window = Number(next());
|
|
@@ -188,7 +202,12 @@ function parseArgs(argv) {
|
|
|
188
202
|
a.noContent = true;
|
|
189
203
|
break;
|
|
190
204
|
case "--replay":
|
|
205
|
+
a.replay = true;
|
|
206
|
+
break;
|
|
207
|
+
// `stream --once` has always meant "one pass"; `watch --once` means the
|
|
208
|
+
// same thing over a run directory, so one flag drives both.
|
|
191
209
|
case "--once":
|
|
210
|
+
a.once = true;
|
|
192
211
|
a.replay = true;
|
|
193
212
|
break;
|
|
194
213
|
case "--no-spans":
|
|
@@ -227,6 +246,30 @@ function parseArgs(argv) {
|
|
|
227
246
|
}
|
|
228
247
|
return a;
|
|
229
248
|
}
|
|
249
|
+
var OTLP_INPUT_COMMANDS = /* @__PURE__ */ new Set(["analyze", "investigate", "improve", "stream", "validate"]);
|
|
250
|
+
function migrateWritingOtlpFlag(args) {
|
|
251
|
+
if (args.otlp === void 0 || OTLP_INPUT_COMMANDS.has(args.command)) return args;
|
|
252
|
+
if (args.otlpOut !== void 0) {
|
|
253
|
+
throw new Error(
|
|
254
|
+
`${args.command} was given both --otlp and --otlp-out. --otlp is the deprecated spelling of --otlp-out on writing commands; pass only --otlp-out.`
|
|
255
|
+
);
|
|
256
|
+
}
|
|
257
|
+
console.error(
|
|
258
|
+
`warning: --otlp is deprecated on \`${args.command}\` and will be removed in 0.12. --otlp now READS OTLP-JSONL; use --otlp-out <path> to write the artifact. Treating it as --otlp-out for this run.`
|
|
259
|
+
);
|
|
260
|
+
return { ...args, otlpOut: args.otlp, otlp: void 0 };
|
|
261
|
+
}
|
|
262
|
+
function validateOtlpSelection(raw) {
|
|
263
|
+
const args = migrateWritingOtlpFlag(raw);
|
|
264
|
+
if (args.otlp === void 0) return args;
|
|
265
|
+
if (args.input) throw new Error("--otlp cannot be combined with an input file");
|
|
266
|
+
if (args.session) throw new Error("--otlp reads a file and cannot be combined with --session");
|
|
267
|
+
if (args.supervisorRunDir) throw new Error("--otlp cannot be combined with --supervisor-run-dir");
|
|
268
|
+
if (args.workflow || args.latestTurn) {
|
|
269
|
+
throw new Error("--workflow and --latest-turn expand harness sessions and cannot be combined with --otlp");
|
|
270
|
+
}
|
|
271
|
+
return args;
|
|
272
|
+
}
|
|
230
273
|
var CURRENT_SESSION_COMMANDS = /* @__PURE__ */ new Set([
|
|
231
274
|
"analyze",
|
|
232
275
|
"investigate",
|
|
@@ -445,7 +488,45 @@ function selectedSessionSource(ref, spans, sessionIdOverride) {
|
|
|
445
488
|
} : {}
|
|
446
489
|
};
|
|
447
490
|
}
|
|
491
|
+
async function collectOtlpSpans(path) {
|
|
492
|
+
const input = await readOtlpInput(path);
|
|
493
|
+
if (input.spans.length === 0) {
|
|
494
|
+
throw new Error(
|
|
495
|
+
`no analyzable spans in ${path} (${input.files.length} file(s), ${input.rows.length} row(s), ${input.issues.length} unusable). Run \`traces validate ${path}\` for the per-finding reason.`
|
|
496
|
+
);
|
|
497
|
+
}
|
|
498
|
+
const fileByTrace = /* @__PURE__ */ new Map();
|
|
499
|
+
for (const file of input.files) {
|
|
500
|
+
for (const traceId of file.traceIds) if (!fileByTrace.has(traceId)) fileByTrace.set(traceId, file.path);
|
|
501
|
+
}
|
|
502
|
+
const byTrace = /* @__PURE__ */ new Map();
|
|
503
|
+
for (const span of input.spans) {
|
|
504
|
+
const grouped = byTrace.get(span.trace_id) ?? [];
|
|
505
|
+
grouped.push(span);
|
|
506
|
+
byTrace.set(span.trace_id, grouped);
|
|
507
|
+
}
|
|
508
|
+
return {
|
|
509
|
+
spans: [...input.spans],
|
|
510
|
+
harness: "otlp",
|
|
511
|
+
cwds: [],
|
|
512
|
+
sources: [...byTrace].map(([traceId, spans]) => selectedSessionSource({
|
|
513
|
+
harness: "otlp",
|
|
514
|
+
sessionId: traceId,
|
|
515
|
+
path: fileByTrace.get(traceId) ?? path,
|
|
516
|
+
cwd: null,
|
|
517
|
+
mtimeMs: 0
|
|
518
|
+
}, spans, traceId)),
|
|
519
|
+
conformance: input.validation,
|
|
520
|
+
conformanceSubject: path,
|
|
521
|
+
conformanceFiles: input.files,
|
|
522
|
+
conformanceSkipped: input.skipped,
|
|
523
|
+
conformanceIssues: input.issues,
|
|
524
|
+
conformanceWithheld: input.withheld,
|
|
525
|
+
conformanceUnreadable: input.unreadable
|
|
526
|
+
};
|
|
527
|
+
}
|
|
448
528
|
async function collectSpans(args) {
|
|
529
|
+
if (args.otlp) return collectOtlpSpans(args.otlp);
|
|
449
530
|
if (args.input) {
|
|
450
531
|
if (args.workflow) throw new Error("--workflow reads discovered sessions and cannot be combined with an input file");
|
|
451
532
|
return collectImportedSpans(args);
|
|
@@ -469,11 +550,37 @@ async function collectSpans(args) {
|
|
|
469
550
|
...selection.workflow ? { workflow: selection.workflow } : {}
|
|
470
551
|
};
|
|
471
552
|
}
|
|
553
|
+
async function cmdValidate(args) {
|
|
554
|
+
const path = args.otlp ?? args.input;
|
|
555
|
+
if (!path) {
|
|
556
|
+
throw new Error("validate needs an OTLP-JSONL file or directory: `traces validate <file|dir>`");
|
|
557
|
+
}
|
|
558
|
+
const input = await readOtlpInput(path);
|
|
559
|
+
const report = renderValidation(input.validation, {
|
|
560
|
+
subject: path,
|
|
561
|
+
files: input.files,
|
|
562
|
+
skipped: input.skipped,
|
|
563
|
+
issues: input.issues,
|
|
564
|
+
withheld: input.withheld,
|
|
565
|
+
unreadable: input.unreadable,
|
|
566
|
+
spans: input.spans
|
|
567
|
+
});
|
|
568
|
+
const validation = input.validation;
|
|
569
|
+
if (args.out) {
|
|
570
|
+
await saveReport(args.out, report);
|
|
571
|
+
console.log(
|
|
572
|
+
`conformance report \u2192 ${args.out} (${validation.findings.length} finding(s), ${validation.capabilities.filter((capability) => capability.available).length}/${validation.capabilities.length} capabilities available)`
|
|
573
|
+
);
|
|
574
|
+
} else {
|
|
575
|
+
console.log(report);
|
|
576
|
+
}
|
|
577
|
+
process.exitCode = validationExitCode(validation);
|
|
578
|
+
}
|
|
472
579
|
async function cmdConvert(args) {
|
|
473
580
|
const { spans, workflow } = await collectSpans(args);
|
|
474
581
|
if (spans.length === 0) throw new Error("no spans found for the given selection");
|
|
475
582
|
warnIncompleteWorkflow(workflow);
|
|
476
|
-
const path = await writeOtlpFile(spans, args.
|
|
583
|
+
const path = await writeOtlpFile(spans, args.otlpOut);
|
|
477
584
|
console.log(`wrote ${spans.length} spans \u2192 ${path}`);
|
|
478
585
|
}
|
|
479
586
|
function workflowIssueText(issue) {
|
|
@@ -507,7 +614,7 @@ async function cmdEvidence(args) {
|
|
|
507
614
|
const rows = selection.rows.filter((row) => row.spans.length > 0);
|
|
508
615
|
if (rows.length === 0) throw new Error("no spans found for the given selection");
|
|
509
616
|
warnIncompleteWorkflow(selection.workflow);
|
|
510
|
-
const otlpPath = args.
|
|
617
|
+
const otlpPath = args.otlpOut ? await writeOtlpFile(rows.flatMap((row) => row.spans), args.otlpOut) : void 0;
|
|
511
618
|
const generatedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
512
619
|
const records = await Promise.all(rows.map(
|
|
513
620
|
(row) => buildPolicyEvidenceRecord(row.ref, row.spans, {
|
|
@@ -566,7 +673,7 @@ async function cmdExport(args) {
|
|
|
566
673
|
if (!args.input) throw new Error("export needs an input file; run `traces export --help` for examples");
|
|
567
674
|
const format = traceEvidenceFormat(args.format);
|
|
568
675
|
const attributes = await loadExportAttributes(args);
|
|
569
|
-
const outPath = args.out ?? args.
|
|
676
|
+
const outPath = args.out ?? args.otlpOut;
|
|
570
677
|
if (outPath) {
|
|
571
678
|
const result2 = await writeTraceEvidenceExportFile(args.input, outPath, { format, attributes });
|
|
572
679
|
console.log(
|
|
@@ -668,9 +775,23 @@ async function cmdAnalyze(args) {
|
|
|
668
775
|
assertAgenticAnalystsRan(args, result.agenticPerAnalyst);
|
|
669
776
|
}
|
|
670
777
|
async function cmdAnalyzeSupervisorRun(runDir, args) {
|
|
671
|
-
const
|
|
778
|
+
const runContextDirs = await findFileRunContextDirs(runDir);
|
|
779
|
+
const nested = runContextDirs.length > 0 ? [] : await findSupervisorRunDirs(runDir);
|
|
672
780
|
let markdown;
|
|
673
|
-
if (
|
|
781
|
+
if (runContextDirs.length > 1) {
|
|
782
|
+
markdown = renderSupervisorRollupMarkdown(
|
|
783
|
+
rollupSupervisorRuns(
|
|
784
|
+
await Promise.all(
|
|
785
|
+
runContextDirs.map((dir) => analyzeSupervisorRun(fileRunContextSupervisorRunReader(dir)))
|
|
786
|
+
)
|
|
787
|
+
),
|
|
788
|
+
`Supervisor rollup \u2014 ${runDir}`
|
|
789
|
+
);
|
|
790
|
+
} else if (runContextDirs.length === 1) {
|
|
791
|
+
markdown = renderSupervisorRunMarkdown(
|
|
792
|
+
await analyzeSupervisorRun(fileRunContextSupervisorRunReader(runContextDirs[0]))
|
|
793
|
+
);
|
|
794
|
+
} else if (nested.length > 0) {
|
|
674
795
|
markdown = renderSupervisorRollupMarkdown(
|
|
675
796
|
rollupSupervisorRuns(await Promise.all(nested.map((dir) => analyzeSupervisorRun(dir)))),
|
|
676
797
|
`Supervisor rollup \u2014 ${runDir}`
|
|
@@ -679,20 +800,30 @@ async function cmdAnalyzeSupervisorRun(runDir, args) {
|
|
|
679
800
|
const report = await analyzeSupervisorRun(runDir);
|
|
680
801
|
if (isUnavailable(report.orchestration.workersSpawned)) {
|
|
681
802
|
throw new Error(
|
|
682
|
-
`no supervisor run found at ${runDir} \u2014 expected <runDir>/ws/.agent/supervisor/<id> (or the pre-rename <runDir>/ws/.loops/supervisor/<id>), or a parent directory containing such runs`
|
|
803
|
+
`no supervisor run found at ${runDir} \u2014 expected <runDir>/spawn-journal.jsonl (agent-runtime createFileRunContext), <runDir>/ws/.agent/supervisor/<id> (or the pre-rename <runDir>/ws/.loops/supervisor/<id>), or a parent directory containing such runs`
|
|
683
804
|
);
|
|
684
805
|
}
|
|
685
806
|
markdown = renderSupervisorRunMarkdown(report);
|
|
686
807
|
}
|
|
808
|
+
const runs = Math.max(runContextDirs.length, nested.length, 1);
|
|
687
809
|
if (args.out) {
|
|
688
810
|
await saveReport(args.out, markdown);
|
|
689
|
-
console.log(
|
|
690
|
-
`supervisor report \u2192 ${args.out} (${nested.length > 0 ? `${nested.length} runs` : "1 run"})`
|
|
691
|
-
);
|
|
811
|
+
console.log(`supervisor report \u2192 ${args.out} (${runs === 1 ? "1 run" : `${runs} runs`})`);
|
|
692
812
|
} else {
|
|
693
813
|
console.log(markdown);
|
|
694
814
|
}
|
|
695
815
|
}
|
|
816
|
+
async function findFileRunContextDirs(dir) {
|
|
817
|
+
if (await isFileRunContextDir(dir)) return [dir];
|
|
818
|
+
const entries = await readdir(dir, { withFileTypes: true }).catch(() => []);
|
|
819
|
+
const found = [];
|
|
820
|
+
for (const entry of entries) {
|
|
821
|
+
if (!entry.isDirectory()) continue;
|
|
822
|
+
const child = join(dir, entry.name);
|
|
823
|
+
if (await isFileRunContextDir(child)) found.push(child);
|
|
824
|
+
}
|
|
825
|
+
return found.sort();
|
|
826
|
+
}
|
|
696
827
|
async function collectImportedSpans(args) {
|
|
697
828
|
if (!args.input) throw new Error("input file missing");
|
|
698
829
|
const attributes = await loadExportAttributes(args);
|
|
@@ -744,8 +875,20 @@ async function cmdInvestigate(args) {
|
|
|
744
875
|
}
|
|
745
876
|
assertAgenticAnalystsRan(args, result.agenticPerAnalyst);
|
|
746
877
|
}
|
|
878
|
+
function conformanceOptions(collected) {
|
|
879
|
+
return {
|
|
880
|
+
...collected.conformance ? { conformance: collected.conformance } : {},
|
|
881
|
+
...collected.conformanceSubject ? { conformanceSubject: collected.conformanceSubject } : {},
|
|
882
|
+
...collected.conformanceFiles ? { conformanceFiles: collected.conformanceFiles } : {},
|
|
883
|
+
...collected.conformanceSkipped ? { conformanceSkipped: collected.conformanceSkipped } : {},
|
|
884
|
+
...collected.conformanceIssues ? { conformanceIssues: collected.conformanceIssues } : {},
|
|
885
|
+
...collected.conformanceWithheld ? { conformanceWithheld: collected.conformanceWithheld } : {},
|
|
886
|
+
...collected.conformanceUnreadable ? { conformanceUnreadable: collected.conformanceUnreadable } : {}
|
|
887
|
+
};
|
|
888
|
+
}
|
|
747
889
|
async function investigate(args, options = {}) {
|
|
748
|
-
const
|
|
890
|
+
const collected = await collectSpans(args);
|
|
891
|
+
const { spans, harness, cwds, sources, workflow } = collected;
|
|
749
892
|
if (spans.length === 0) throw new Error("no spans found for the given selection");
|
|
750
893
|
const config = args.config !== void 0 || options.loadDefaultConfig !== false ? await loadTracesConfig(args.config) : void 0;
|
|
751
894
|
const analystModel = args.model ?? process.env.TRACES_ANALYST_MODEL ?? DEFAULT_ANALYST_MODEL;
|
|
@@ -760,14 +903,16 @@ async function investigate(args, options = {}) {
|
|
|
760
903
|
engine,
|
|
761
904
|
model: args.llm ? analystModel : args.model,
|
|
762
905
|
budgetUsd: args.budget,
|
|
763
|
-
otlpOutPath: args.
|
|
906
|
+
otlpOutPath: args.otlpOut,
|
|
764
907
|
externalAnalyzers: externalAnalyzersFromArgs(args),
|
|
765
908
|
analyzerPrompt: args.analyzerPrompt,
|
|
766
|
-
log: analystLog
|
|
909
|
+
log: analystLog,
|
|
910
|
+
...conformanceOptions(collected)
|
|
767
911
|
}, config));
|
|
768
912
|
}
|
|
769
913
|
async function cmdImprove(args) {
|
|
770
|
-
const
|
|
914
|
+
const collected = await collectSpans(args);
|
|
915
|
+
const { spans, harness, cwds, sources, workflow } = collected;
|
|
771
916
|
if (spans.length === 0) throw new Error("no spans found for the given selection");
|
|
772
917
|
const config = await loadTracesConfig(args.config);
|
|
773
918
|
const analystModel = args.model ?? process.env.TRACES_ANALYST_MODEL ?? DEFAULT_ANALYST_MODEL;
|
|
@@ -783,10 +928,11 @@ async function cmdImprove(args) {
|
|
|
783
928
|
engine,
|
|
784
929
|
model: args.llm ? analystModel : args.model,
|
|
785
930
|
budgetUsd: args.budget,
|
|
786
|
-
otlpOutPath: args.
|
|
931
|
+
otlpOutPath: args.otlpOut,
|
|
787
932
|
externalAnalyzers: externalAnalyzersFromArgs(args),
|
|
788
933
|
analyzerPrompt: args.analyzerPrompt,
|
|
789
|
-
log: analystLog
|
|
934
|
+
log: analystLog,
|
|
935
|
+
...conformanceOptions(collected)
|
|
790
936
|
}, config),
|
|
791
937
|
outDir: args.dir ?? args.out
|
|
792
938
|
});
|
|
@@ -854,12 +1000,13 @@ async function streamExplicitSession(args, extraAnalysts) {
|
|
|
854
1000
|
async function cmdStream(args) {
|
|
855
1001
|
const config = await loadTracesConfig(args.config);
|
|
856
1002
|
const preset = streamPreset(args);
|
|
857
|
-
|
|
858
|
-
|
|
1003
|
+
const source = args.otlp ?? args.input;
|
|
1004
|
+
if (source) {
|
|
1005
|
+
const { spans, harness } = args.otlp ? await collectOtlpSpans(args.otlp) : await collectImportedSpans(args);
|
|
859
1006
|
const ref = {
|
|
860
1007
|
harness,
|
|
861
|
-
sessionId: basename(
|
|
862
|
-
path: resolve(
|
|
1008
|
+
sessionId: basename(source),
|
|
1009
|
+
path: resolve(source),
|
|
863
1010
|
cwd: args.cwd ?? null,
|
|
864
1011
|
mtimeMs: Date.now()
|
|
865
1012
|
};
|
|
@@ -911,6 +1058,8 @@ async function cmdStream(args) {
|
|
|
911
1058
|
});
|
|
912
1059
|
}
|
|
913
1060
|
async function cmdWatch(args) {
|
|
1061
|
+
const target = args.supervisorRunDir ?? args.input;
|
|
1062
|
+
if (target !== void 0) return cmdWatchRunTarget(target, args);
|
|
914
1063
|
const all = args.all || !args.harnessExplicit && !args.cwd;
|
|
915
1064
|
const controller = new AbortController();
|
|
916
1065
|
process.once("SIGINT", () => controller.abort());
|
|
@@ -939,6 +1088,26 @@ async function cmdWatch(args) {
|
|
|
939
1088
|
}
|
|
940
1089
|
});
|
|
941
1090
|
}
|
|
1091
|
+
async function cmdWatchRunTarget(dir, args) {
|
|
1092
|
+
const target = await resolveRunWatchTarget(resolve(dir));
|
|
1093
|
+
const controller = new AbortController();
|
|
1094
|
+
if (!args.once) {
|
|
1095
|
+
process.once("SIGINT", () => controller.abort());
|
|
1096
|
+
process.stderr.write(
|
|
1097
|
+
`traces watch: tailing ${target.requested} every ${args.interval}s; read-only, ${target.runDir === null ? "span source only \u2014 Ctrl-C to stop" : "stops when result.json lands. Ctrl-C to stop"}.
|
|
1098
|
+
`
|
|
1099
|
+
);
|
|
1100
|
+
}
|
|
1101
|
+
const snapshot = await watchRunTarget(target, {
|
|
1102
|
+
once: args.once,
|
|
1103
|
+
// A tree view wants a tighter poll than the session observer's 5s default:
|
|
1104
|
+
// a blocking question is worth surfacing the moment it lands.
|
|
1105
|
+
intervalMs: args.intervalExplicit ? Math.max(250, args.interval * 1e3) : 2e3,
|
|
1106
|
+
signal: controller.signal
|
|
1107
|
+
});
|
|
1108
|
+
const result = snapshot.journal?.result ?? null;
|
|
1109
|
+
if (result !== null && result.errorMessage !== null) process.exitCode = 1;
|
|
1110
|
+
}
|
|
942
1111
|
async function cmdUpload(args) {
|
|
943
1112
|
const sinceMs = args.since ? parseSince(args.since) : Date.now() - 24 * 60 * 60 * 1e3;
|
|
944
1113
|
const all = args.all || !args.harnessExplicit;
|
|
@@ -976,7 +1145,7 @@ Window: since ${new Date(sinceMs).toISOString()}
|
|
|
976
1145
|
return;
|
|
977
1146
|
}
|
|
978
1147
|
if (args.dryRun) {
|
|
979
|
-
const res2 = await executeUpload(plan, { dryRun: true, otlpOut: args.
|
|
1148
|
+
const res2 = await executeUpload(plan, { dryRun: true, otlpOut: args.otlpOut, stripContent: args.noContent, redactor });
|
|
980
1149
|
console.log(`dry run: ${candidates.length - res2.skippedSessions} session(s), ${totalRedactions} redaction(s). Redacted OTLP -> ${res2.otlpPath}`);
|
|
981
1150
|
console.log("No upload performed. Set TANGLE_INGEST_URL / TANGLE_INGEST_API_KEY / TANGLE_TENANT_ID and drop --dry-run to send.");
|
|
982
1151
|
return;
|
|
@@ -1039,24 +1208,75 @@ The command validates every native step, rejects annotation leakage, writes
|
|
|
1039
1208
|
<traj_id>.otlp.jsonl files, and records input and output SHA-256 hashes in
|
|
1040
1209
|
codetracebench-import.json. The output directory must not already exist.`);
|
|
1041
1210
|
}
|
|
1211
|
+
function usageValidate() {
|
|
1212
|
+
console.log(`traces validate: report what a trace can and cannot answer
|
|
1213
|
+
|
|
1214
|
+
Usage:
|
|
1215
|
+
traces validate <file.jsonl|dir> [--out conformance.md]
|
|
1216
|
+
|
|
1217
|
+
Reads OTLP-JSONL and prints the conformance report from
|
|
1218
|
+
@tangle-network/agent-trace-contract:
|
|
1219
|
+
|
|
1220
|
+
- findings by severity, each naming the CONSEQUENCE of the defect and the
|
|
1221
|
+
capabilities it blocks
|
|
1222
|
+
- the capability table: token-accounting, cost-attribution, tool-usage,
|
|
1223
|
+
loop-convergence, tree-comparison, steering-chain, latency-analysis \u2014
|
|
1224
|
+
available or not, with the trace's own reason
|
|
1225
|
+
- rows that could not be read at all, with the line and the reason
|
|
1226
|
+
|
|
1227
|
+
A directory reads the OTLP files under it, recursively \u2014 only the otlp/
|
|
1228
|
+
subdirectory when the producer made one. A run directory's raw event, stream and
|
|
1229
|
+
SDK logs are also *.jsonl; they are listed with what they actually hold, not read
|
|
1230
|
+
as several hundred broken spans. Identical re-declarations of a span across
|
|
1231
|
+
per-shot files are counted as repeats, not defects; only a repeat with DIFFERENT
|
|
1232
|
+
content is an error.
|
|
1233
|
+
|
|
1234
|
+
Exit code:
|
|
1235
|
+
0 no error findings, and at least one capability available (warnings mean
|
|
1236
|
+
thinner analysis, not wrong analysis)
|
|
1237
|
+
1 an error finding, or nothing analysable at all \u2014 an empty file, or one
|
|
1238
|
+
whose every line was unreadable, answers no question and is not a pass
|
|
1239
|
+
|
|
1240
|
+
Nothing is fixed up: a span with an unreadable timestamp is reported, never
|
|
1241
|
+
repaired, because a synthesized value puts invented work into a total.
|
|
1242
|
+
|
|
1243
|
+
Examples:
|
|
1244
|
+
traces validate spans.otlp.jsonl
|
|
1245
|
+
traces validate results/sessions --out conformance.md
|
|
1246
|
+
traces validate spans.otlp.jsonl && traces analyze --otlp spans.otlp.jsonl`);
|
|
1247
|
+
}
|
|
1042
1248
|
function usage() {
|
|
1043
|
-
console.log(`traces: analyze
|
|
1249
|
+
console.log(`traces: analyze agent traces \u2014 yours, or any system's
|
|
1250
|
+
|
|
1251
|
+
Two ways in:
|
|
1252
|
+
--otlp <file|dir> Spans a system already emitted in the
|
|
1253
|
+
@tangle-network/agent-trace-contract shape. No adapter, no
|
|
1254
|
+
translation. This is how you integrate a NEW system.
|
|
1255
|
+
--harness <id> Run an adapter over a coding agent's proprietary session
|
|
1256
|
+
store (Claude Code, Codex, OpenCode, \u2026) \u2014 the legacy edge,
|
|
1257
|
+
for formats we do not control.
|
|
1044
1258
|
|
|
1045
1259
|
Commands:
|
|
1260
|
+
validate Report what a trace can and cannot answer (exit 1 on error findings)
|
|
1046
1261
|
list List discovered sessions
|
|
1047
|
-
analyze Run analyst suite + loop/waste pipelines over
|
|
1262
|
+
analyze Run analyst suite + loop/waste pipelines over OTLP (--otlp), sessions,
|
|
1263
|
+
or an input file; names every analysis the spans could not support
|
|
1048
1264
|
(--supervisor-run-dir <dir> reports a supervision tree instead)
|
|
1049
1265
|
investigate Run typed investigation flow, including BYO config + evidence-backed actions
|
|
1050
1266
|
improve Write findings, evidence, report, and canonical trace artifacts
|
|
1051
|
-
convert Emit OTLP-JSONL only (HALO: use analyze --analyzer halo)
|
|
1267
|
+
convert Emit OTLP-JSONL only, to --otlp-out (HALO: use analyze --analyzer halo)
|
|
1052
1268
|
index Emit a reusable session index JSON for later investigation
|
|
1053
1269
|
inspect Read a session index and print ranked improvement findings
|
|
1054
1270
|
export Convert evidence/events files to OpenInference JSONL for HALO
|
|
1055
1271
|
import-codetracebench
|
|
1056
1272
|
Convert CodeTracer-normalized CodeTraceBench trajectories in bulk
|
|
1273
|
+
replay-verify
|
|
1274
|
+
Replay a trajectory prefix in a sandbox, execute step k both recorded
|
|
1275
|
+
and corrected, and emit an executed-proof verdict (--help for flags)
|
|
1057
1276
|
evidence Emit compact session-evidence JSONL for downstream policy miners
|
|
1058
1277
|
stream Emit JSONL trace stream events for live visualizers or replay
|
|
1059
1278
|
watch Online observer: tail active sessions, notify on loops + semantic findings
|
|
1279
|
+
(watch <target> tails ONE run tree instead: a run directory or a span file)
|
|
1060
1280
|
upload Redact + upload sessions in a time window to the Tangle Intelligence Platform
|
|
1061
1281
|
|
|
1062
1282
|
Options:
|
|
@@ -1074,15 +1294,26 @@ Options:
|
|
|
1074
1294
|
analyze: report a SUPERVISION TREE instead of harness sessions \u2014
|
|
1075
1295
|
steers, spawn waves, concurrency, idle wall, cost by role,
|
|
1076
1296
|
accepted vs rejected. Rolls up when the dir holds many runs.
|
|
1297
|
+
watch: same directory, live. Reads the loops layout, the
|
|
1298
|
+
agent-runtime createFileRunContext journal, and OTLP span files.
|
|
1077
1299
|
--since <t> upload: window, 30m / 2h / 7d or an ISO date (default 24h); analyze: ISO cutoff
|
|
1078
1300
|
--out <path> Write report to a file
|
|
1079
1301
|
--dir <path> improve: write artifacts to this directory
|
|
1080
|
-
--otlp <
|
|
1302
|
+
--otlp <file|dir> READ OTLP-JSONL emitted by any system, skipping the adapters.
|
|
1303
|
+
A directory reads the OTLP files under it \u2014 only the otlp/
|
|
1304
|
+
subdirectory when the producer made one \u2014 and names the JSONL
|
|
1305
|
+
that is not OTLP instead of reading it as broken spans.
|
|
1306
|
+
Supported by: validate, analyze, investigate, improve, stream.
|
|
1307
|
+
On a WRITING command it is the deprecated spelling of
|
|
1308
|
+
--otlp-out; it still works, with a warning, until 0.12.
|
|
1309
|
+
--otlp-out <path> WRITE the OTLP-JSONL artifact here (also evidence
|
|
1310
|
+
provenance / dry-run upload preview)
|
|
1081
1311
|
--format <kind> analyze/export: auto | policy-evidence | sandbox-events | openinference | intelligence-spans | chat-trajectory
|
|
1082
1312
|
--metadata <json> analyze/export file: attach JSON object fields as span attributes
|
|
1083
1313
|
--attr <k=v> analyze/export file: attach one span attribute (repeatable)
|
|
1084
1314
|
--mode <kind> stream: visualizer | findings | agent (default visualizer)
|
|
1085
1315
|
--replay, --once stream: scan once and exit (default for positional input / --session)
|
|
1316
|
+
--once watch <target>: print ONE snapshot and exit, for scripts and agents
|
|
1086
1317
|
--no-spans stream: omit per-span pulse events
|
|
1087
1318
|
--no-findings stream: omit semantic live-finding events
|
|
1088
1319
|
--llm Enable agentic RLM analysts. Runs on the Tangle router by
|
|
@@ -1096,7 +1327,7 @@ Options:
|
|
|
1096
1327
|
--budget <usd> USD cap for agentic analysts
|
|
1097
1328
|
--analyzer <id> analyze: also run halo, hodoscope, or an installed command (repeatable)
|
|
1098
1329
|
--analyzer-prompt <p> analyze: prompt passed to external analyzers (default: diagnose)
|
|
1099
|
-
--interval <s> watch/stream: poll interval seconds (
|
|
1330
|
+
--interval <s> watch/stream: poll interval seconds (sessions 5, run tree 2)
|
|
1100
1331
|
--window <m> watch/stream: only sessions active in the last N minutes (default 30)
|
|
1101
1332
|
--min-loop <n> Min identical repeated calls to flag a loop (default 3)
|
|
1102
1333
|
--dry-run upload: redact + dedup + preview, write OTLP, but do NOT send
|
|
@@ -1104,7 +1335,15 @@ Options:
|
|
|
1104
1335
|
--redactor <cmd> upload: external PII scrubber (JSON array stdin\u2192stdout) after the regex pass
|
|
1105
1336
|
--yes, -y upload: skip the confirmation prompt
|
|
1106
1337
|
--version, -v Print the installed traces version
|
|
1107
|
-
--help, -h Show help (use \`traces
|
|
1338
|
+
--help, -h Show help (use \`traces validate --help\`, \`traces export --help\`,
|
|
1339
|
+
or \`traces import-codetracebench --help\`)
|
|
1340
|
+
|
|
1341
|
+
Watch a run tree (any depth; driver spend and child spend stay separate,
|
|
1342
|
+
and a cost the harness never priced reads "unknown", never $0):
|
|
1343
|
+
traces watch runs/my-run live tree: status, budget, driver vs child spend
|
|
1344
|
+
traces watch runs/my-run --once one snapshot, then exit
|
|
1345
|
+
traces watch traces.otlp.jsonl OTLP spans only \u2014 works for ANY emitter
|
|
1346
|
+
traces analyze --supervisor-run-dir runs/my-run the full tree report
|
|
1108
1347
|
|
|
1109
1348
|
Upload env: TANGLE_INGEST_URL (or TANGLE_ORCHESTRATOR_URL), TANGLE_INGEST_API_KEY (or TANGLE_API_KEY), TANGLE_TENANT_ID`);
|
|
1110
1349
|
}
|
|
@@ -1114,18 +1353,24 @@ async function main() {
|
|
|
1114
1353
|
console.log(`traces ${packageVersion()}`);
|
|
1115
1354
|
return;
|
|
1116
1355
|
}
|
|
1356
|
+
if (rawArgs[0] === "replay-verify") {
|
|
1357
|
+
await cmdReplayVerify(rawArgs.slice(1));
|
|
1358
|
+
return;
|
|
1359
|
+
}
|
|
1117
1360
|
const parsedArgs = parseArgs(rawArgs);
|
|
1118
1361
|
if (parsedArgs.help) {
|
|
1119
1362
|
if (parsedArgs.command === "import-codetracebench" || parsedArgs.command === "help" && parsedArgs.input === "import-codetracebench") usageImportCodeTraceBench();
|
|
1120
1363
|
else if (parsedArgs.command === "export" || parsedArgs.command === "help" && parsedArgs.input === "export") usageExport();
|
|
1364
|
+
else if (parsedArgs.command === "validate" || parsedArgs.command === "help" && parsedArgs.input === "validate") usageValidate();
|
|
1121
1365
|
else usage();
|
|
1122
1366
|
return;
|
|
1123
1367
|
}
|
|
1124
|
-
const args = validateWorkflowSelection(applyCurrentSessionSelection(parsedArgs));
|
|
1368
|
+
const args = validateOtlpSelection(validateWorkflowSelection(applyCurrentSessionSelection(parsedArgs)));
|
|
1125
1369
|
switch (args.command) {
|
|
1126
1370
|
case "help":
|
|
1127
1371
|
if (args.input === "import-codetracebench") usageImportCodeTraceBench();
|
|
1128
1372
|
else if (args.input === "export") usageExport();
|
|
1373
|
+
else if (args.input === "validate") usageValidate();
|
|
1129
1374
|
else usage();
|
|
1130
1375
|
break;
|
|
1131
1376
|
case "list":
|
|
@@ -1134,6 +1379,9 @@ async function main() {
|
|
|
1134
1379
|
case "analyze":
|
|
1135
1380
|
await cmdAnalyze(args);
|
|
1136
1381
|
break;
|
|
1382
|
+
case "validate":
|
|
1383
|
+
await cmdValidate(args);
|
|
1384
|
+
break;
|
|
1137
1385
|
case "investigate":
|
|
1138
1386
|
await cmdInvestigate(args);
|
|
1139
1387
|
break;
|