@tangle-network/traces 0.8.13 → 0.8.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +48 -0
- package/dist/{chunk-PRRWLKSI.js → chunk-EOQY2RIK.js} +789 -118
- package/dist/cli.js +90 -193
- package/dist/index.d.ts +173 -3
- package/dist/index.js +19 -3
- package/package.json +1 -1
|
@@ -160,11 +160,11 @@ function serializeSpans(spans) {
|
|
|
160
160
|
`;
|
|
161
161
|
}
|
|
162
162
|
async function writeOtlpFile(spans, outPath) {
|
|
163
|
-
const { mkdtemp:
|
|
164
|
-
const { tmpdir:
|
|
165
|
-
const { join:
|
|
166
|
-
const path = outPath ??
|
|
167
|
-
await
|
|
163
|
+
const { mkdtemp: mkdtemp6, writeFile: writeFile7 } = await import("fs/promises");
|
|
164
|
+
const { tmpdir: tmpdir6 } = await import("os");
|
|
165
|
+
const { join: join20 } = await import("path");
|
|
166
|
+
const path = outPath ?? join20(await mkdtemp6(join20(tmpdir6(), "traces-")), "spans.otlp.jsonl");
|
|
167
|
+
await writeFile7(path, serializeSpans(spans), "utf8");
|
|
168
168
|
return path;
|
|
169
169
|
}
|
|
170
170
|
|
|
@@ -2286,13 +2286,13 @@ function extractAbsolutePaths(spans) {
|
|
|
2286
2286
|
let scannedChars = 0;
|
|
2287
2287
|
for (const span2 of spans) {
|
|
2288
2288
|
for (const value of [span2.name, span2.status.message, ...Object.values(span2.attributes)]) {
|
|
2289
|
-
for (const
|
|
2290
|
-
scannedChars +=
|
|
2289
|
+
for (const evidence2 of pathsFromUnknown(value)) {
|
|
2290
|
+
scannedChars += evidence2.path.length;
|
|
2291
2291
|
if (scannedChars > MAX_SPAN_TEXT_CHARS) return paths;
|
|
2292
|
-
const key = `${
|
|
2292
|
+
const key = `${evidence2.source}:${evidence2.path}`;
|
|
2293
2293
|
if (seen.has(key)) continue;
|
|
2294
2294
|
seen.add(key);
|
|
2295
|
-
paths.push(
|
|
2295
|
+
paths.push(evidence2);
|
|
2296
2296
|
if (paths.length >= MAX_SPAN_PATH_CANDIDATES) return paths;
|
|
2297
2297
|
}
|
|
2298
2298
|
}
|
|
@@ -2346,8 +2346,8 @@ async function resolveSessionRepoAttrs(cwd, spans) {
|
|
|
2346
2346
|
add(await repairDashedPath(cwd), "repaired-cwd");
|
|
2347
2347
|
add(cwd, "ref-cwd");
|
|
2348
2348
|
}
|
|
2349
|
-
for (const
|
|
2350
|
-
add(await repoCandidateForPath(
|
|
2349
|
+
for (const evidence2 of extractAbsolutePaths(spans)) {
|
|
2350
|
+
add(await repoCandidateForPath(evidence2.path), evidence2.source, evidence2.weight);
|
|
2351
2351
|
}
|
|
2352
2352
|
const resolved = [];
|
|
2353
2353
|
for (const candidate of candidates.values()) {
|
|
@@ -2531,7 +2531,7 @@ import { spawn } from "child_process";
|
|
|
2531
2531
|
function runCommand(command, args = [], opts = {}) {
|
|
2532
2532
|
const timeoutMs = opts.timeoutMs ?? 12e4;
|
|
2533
2533
|
const maxBuffer = opts.maxBuffer ?? 32 * 1024 * 1024;
|
|
2534
|
-
return new Promise((
|
|
2534
|
+
return new Promise((resolve2, reject) => {
|
|
2535
2535
|
let child;
|
|
2536
2536
|
try {
|
|
2537
2537
|
child = spawn(command, [...args], { signal: opts.signal });
|
|
@@ -2569,7 +2569,7 @@ function runCommand(command, args = [], opts = {}) {
|
|
|
2569
2569
|
});
|
|
2570
2570
|
child.on("close", (code) => {
|
|
2571
2571
|
clearTimeout(timer);
|
|
2572
|
-
if (!killed)
|
|
2572
|
+
if (!killed) resolve2({ stdout: out, stderr: err, code });
|
|
2573
2573
|
});
|
|
2574
2574
|
if (opts.input !== void 0) {
|
|
2575
2575
|
child.stdin?.on("error", () => {
|
|
@@ -3376,84 +3376,6 @@ async function writeInspectionReportFile(report, outPath) {
|
|
|
3376
3376
|
return path;
|
|
3377
3377
|
}
|
|
3378
3378
|
|
|
3379
|
-
// src/observer.ts
|
|
3380
|
-
var sleep = (ms, signal) => new Promise((resolve) => {
|
|
3381
|
-
const onAbort = () => {
|
|
3382
|
-
clearTimeout(t);
|
|
3383
|
-
resolve();
|
|
3384
|
-
};
|
|
3385
|
-
const t = setTimeout(() => {
|
|
3386
|
-
signal?.removeEventListener("abort", onAbort);
|
|
3387
|
-
resolve();
|
|
3388
|
-
}, ms);
|
|
3389
|
-
signal?.addEventListener("abort", onAbort, { once: true });
|
|
3390
|
-
});
|
|
3391
|
-
function loopKey(sessionId, toolName2, argHash) {
|
|
3392
|
-
return `${sessionId}\0${toolName2}\0${argHash}`;
|
|
3393
|
-
}
|
|
3394
|
-
async function watchSessions(opts) {
|
|
3395
|
-
const windowMs = opts.windowMs ?? 30 * 6e4;
|
|
3396
|
-
const intervalMs = Math.max(250, opts.intervalMs ?? 5e3);
|
|
3397
|
-
const minLoop = opts.minLoopOccurrences ?? 3;
|
|
3398
|
-
const seen = /* @__PURE__ */ new Map();
|
|
3399
|
-
while (!opts.signal?.aborted) {
|
|
3400
|
-
let sessionCount = 0;
|
|
3401
|
-
for await (const { adapter, ref, spans } of scanSessions({
|
|
3402
|
-
adapters: opts.adapters,
|
|
3403
|
-
all: opts.all,
|
|
3404
|
-
harnesses: opts.harnesses,
|
|
3405
|
-
cwd: opts.cwd,
|
|
3406
|
-
sinceMs: Date.now() - windowMs,
|
|
3407
|
-
signal: opts.signal,
|
|
3408
|
-
onError: opts.onError
|
|
3409
|
-
})) {
|
|
3410
|
-
let report;
|
|
3411
|
-
try {
|
|
3412
|
-
report = await runPipelines(spans, { minLoopOccurrences: minLoop });
|
|
3413
|
-
} catch (err) {
|
|
3414
|
-
opts.onError?.(err, ref);
|
|
3415
|
-
continue;
|
|
3416
|
-
}
|
|
3417
|
-
sessionCount += 1;
|
|
3418
|
-
if (opts.onReport) {
|
|
3419
|
-
try {
|
|
3420
|
-
await opts.onReport(ref, report);
|
|
3421
|
-
} catch (err) {
|
|
3422
|
-
opts.onError?.(err, ref);
|
|
3423
|
-
}
|
|
3424
|
-
}
|
|
3425
|
-
for (const f of report.stuckLoops.findings) {
|
|
3426
|
-
const key = loopKey(ref.sessionId, f.toolName, f.argHash);
|
|
3427
|
-
if (f.occurrences > (seen.get(key) ?? 0)) {
|
|
3428
|
-
seen.set(key, f.occurrences);
|
|
3429
|
-
if (opts.onLoop) {
|
|
3430
|
-
try {
|
|
3431
|
-
await opts.onLoop({
|
|
3432
|
-
harness: adapter.harness,
|
|
3433
|
-
sessionId: ref.sessionId,
|
|
3434
|
-
cwd: ref.cwd,
|
|
3435
|
-
toolName: f.toolName,
|
|
3436
|
-
argHash: f.argHash,
|
|
3437
|
-
occurrences: f.occurrences,
|
|
3438
|
-
windowMs: f.windowMs
|
|
3439
|
-
});
|
|
3440
|
-
} catch (err) {
|
|
3441
|
-
opts.onError?.(err, ref);
|
|
3442
|
-
}
|
|
3443
|
-
}
|
|
3444
|
-
}
|
|
3445
|
-
}
|
|
3446
|
-
}
|
|
3447
|
-
try {
|
|
3448
|
-
opts.onTick?.({ sessions: sessionCount });
|
|
3449
|
-
} catch (err) {
|
|
3450
|
-
opts.onError?.(err);
|
|
3451
|
-
}
|
|
3452
|
-
if (opts.signal?.aborted) break;
|
|
3453
|
-
await sleep(intervalMs, opts.signal);
|
|
3454
|
-
}
|
|
3455
|
-
}
|
|
3456
|
-
|
|
3457
3379
|
// src/reactions.ts
|
|
3458
3380
|
var REACTIONS = ["correction", "frustration", "jargon", "structure", "praise"];
|
|
3459
3381
|
function emptySignals() {
|
|
@@ -3540,10 +3462,746 @@ function analyzeReactions(spans) {
|
|
|
3540
3462
|
};
|
|
3541
3463
|
}
|
|
3542
3464
|
|
|
3543
|
-
// src/
|
|
3544
|
-
|
|
3465
|
+
// src/report.ts
|
|
3466
|
+
var SEVERITY_RANK = { critical: 0, high: 1, medium: 2, low: 3, info: 4 };
|
|
3467
|
+
var SEVERITY_BADGE = {
|
|
3468
|
+
critical: "\u{1F534} CRITICAL",
|
|
3469
|
+
high: "\u{1F7E0} HIGH",
|
|
3470
|
+
medium: "\u{1F7E1} MEDIUM",
|
|
3471
|
+
low: "\u{1F535} LOW",
|
|
3472
|
+
info: "\u2139\uFE0F INFO"
|
|
3473
|
+
};
|
|
3474
|
+
function deterministicSummaryText(summary) {
|
|
3475
|
+
const parts = [];
|
|
3476
|
+
if (summary.stuckLoops > 0) parts.push(`${summary.stuckLoops} stuck loop(s)`);
|
|
3477
|
+
if (summary.reactionSignals > 0) parts.push(`${summary.reactionSignals} human reaction signal(s)`);
|
|
3478
|
+
if (summary.toolErrorRuns > 0) parts.push(`${summary.toolErrorRuns} tool-error run(s)`);
|
|
3479
|
+
if (parts.length === 0) return "0 deterministic signals";
|
|
3480
|
+
if (parts.length === 1) return parts[0];
|
|
3481
|
+
return `${parts.slice(0, -1).join(", ")} and ${parts[parts.length - 1]}`;
|
|
3482
|
+
}
|
|
3483
|
+
function plural(count, singular, pluralForm = `${singular}s`) {
|
|
3484
|
+
return count === 1 ? singular : pluralForm;
|
|
3485
|
+
}
|
|
3486
|
+
function summarizeDeterministicSignals(pipelines, reactions) {
|
|
3487
|
+
const stuckLoops = pipelines.stuckLoops.findings.length;
|
|
3488
|
+
const reactionSignals = Object.values(reactions.signals).reduce((total, count) => total + count, 0);
|
|
3489
|
+
const toolErrorRuns2 = pipelines.toolUse.filter((run) => run.errorRate > 0).length;
|
|
3490
|
+
return {
|
|
3491
|
+
stuckLoops,
|
|
3492
|
+
reactionSignals,
|
|
3493
|
+
toolErrorRuns: toolErrorRuns2,
|
|
3494
|
+
totalSignals: stuckLoops + reactionSignals + toolErrorRuns2
|
|
3495
|
+
};
|
|
3496
|
+
}
|
|
3497
|
+
function renderReport(result, meta) {
|
|
3498
|
+
const lines = [];
|
|
3499
|
+
const findings = [...result.findings].sort(
|
|
3500
|
+
(a, b) => (SEVERITY_RANK[a.severity] ?? 9) - (SEVERITY_RANK[b.severity] ?? 9)
|
|
3501
|
+
);
|
|
3502
|
+
const deterministic = meta.deterministic ?? { stuckLoops: 0, reactionSignals: 0, toolErrorRuns: 0, totalSignals: 0 };
|
|
3503
|
+
const findingSummary = deterministic.totalSignals > 0 ? `${findings.length} analyst ${plural(findings.length, "finding")} + ${deterministic.totalSignals} deterministic ${plural(deterministic.totalSignals, "signal")}` : `${findings.length} ${plural(findings.length, "finding")}`;
|
|
3504
|
+
lines.push(`# Trace analysis \u2014 ${meta.harness}`);
|
|
3505
|
+
lines.push("");
|
|
3506
|
+
lines.push(
|
|
3507
|
+
`${meta.sessionCount} session(s), ${meta.spanCount} spans \u2192 **${findingSummary}** across ${result.per_analyst.length} analyst(s). Cost: $${result.total_cost_usd.toFixed(4)}.`
|
|
3508
|
+
);
|
|
3509
|
+
lines.push("");
|
|
3510
|
+
lines.push("| Analyst | Status | Findings | Latency |");
|
|
3511
|
+
lines.push("|---|---|---|---|");
|
|
3512
|
+
for (const s of result.per_analyst) {
|
|
3513
|
+
lines.push(`| \`${s.analyst_id}\` | ${s.status} | ${s.findings_count} | ${s.latency_ms}ms |`);
|
|
3514
|
+
}
|
|
3515
|
+
lines.push("");
|
|
3516
|
+
if (findings.length === 0 && deterministic.totalSignals === 0) {
|
|
3517
|
+
lines.push("_No findings \u2014 no behavioral inefficiencies or failure modes detected._");
|
|
3518
|
+
} else if (findings.length === 0) {
|
|
3519
|
+
lines.push(
|
|
3520
|
+
`_No analyst findings. Deterministic checks found ${deterministicSummaryText(deterministic)}; see sections below._`
|
|
3521
|
+
);
|
|
3522
|
+
} else {
|
|
3523
|
+
const byArea = /* @__PURE__ */ new Map();
|
|
3524
|
+
for (const f of findings) {
|
|
3525
|
+
const arr = byArea.get(f.area) ?? [];
|
|
3526
|
+
arr.push(f);
|
|
3527
|
+
byArea.set(f.area, arr);
|
|
3528
|
+
}
|
|
3529
|
+
for (const [area, arr] of byArea) {
|
|
3530
|
+
lines.push(`## ${area} (${arr.length})`);
|
|
3531
|
+
lines.push("");
|
|
3532
|
+
for (const f of arr) {
|
|
3533
|
+
lines.push(`### ${SEVERITY_BADGE[f.severity] ?? f.severity} \u2014 ${f.claim}`);
|
|
3534
|
+
lines.push("");
|
|
3535
|
+
if (f.subject) lines.push(`- **Subject:** \`${f.subject}\``);
|
|
3536
|
+
lines.push(`- **Confidence:** ${f.confidence}`);
|
|
3537
|
+
if (f.recommended_action) lines.push(`- **Fix:** ${f.recommended_action}`);
|
|
3538
|
+
for (const ev of f.evidence_refs.slice(0, 3)) {
|
|
3539
|
+
const excerpt = ev.excerpt ? ` \u2014 \`${ev.excerpt.slice(0, 200)}\`` : "";
|
|
3540
|
+
lines.push(`- **Evidence:** ${ev.kind} ${ev.uri}${excerpt}`);
|
|
3541
|
+
}
|
|
3542
|
+
lines.push("");
|
|
3543
|
+
}
|
|
3544
|
+
}
|
|
3545
|
+
}
|
|
3546
|
+
lines.push("---");
|
|
3547
|
+
lines.push(`OTLP artifact: \`${meta.otlpPath}\` \u2014 run external engines with \`traces analyze --analyzer halo\`.`);
|
|
3548
|
+
lines.push("");
|
|
3549
|
+
return lines.join("\n");
|
|
3550
|
+
}
|
|
3551
|
+
function renderPipelines(pr) {
|
|
3552
|
+
const lines = ["## loops & waste (deterministic)", ""];
|
|
3553
|
+
if (pr.stuckLoops.findings.length === 0) {
|
|
3554
|
+
lines.push("- **Stuck loops:** none (no tool called \u22653\xD7 with identical args).");
|
|
3555
|
+
} else {
|
|
3556
|
+
lines.push(`- **Stuck loops:** ${pr.stuckLoops.findings.length} (${(pr.stuckLoops.affectedRunRatio * 100).toFixed(0)}% of runs affected)`);
|
|
3557
|
+
for (const f of pr.stuckLoops.findings.sort((a, b) => b.occurrences - a.occurrences).slice(0, 10)) {
|
|
3558
|
+
lines.push(` - \u{1F501} \`${f.toolName}\` \xD7${f.occurrences} with identical args over ${(f.windowMs / 1e3).toFixed(1)}s`);
|
|
3559
|
+
}
|
|
3560
|
+
}
|
|
3561
|
+
for (const m of pr.toolUse) {
|
|
3562
|
+
if (m.totalCalls === 0) continue;
|
|
3563
|
+
lines.push(
|
|
3564
|
+
`- **Tool use** (${m.totalCalls} calls): ${(m.duplicateRate * 100).toFixed(0)}% duplicate, ${(m.retryRate * 100).toFixed(0)}% retry, ${(m.errorRate * 100).toFixed(0)}% error`
|
|
3565
|
+
);
|
|
3566
|
+
}
|
|
3567
|
+
lines.push("");
|
|
3568
|
+
return lines.join("\n");
|
|
3569
|
+
}
|
|
3570
|
+
var REACTION_BADGE = {
|
|
3571
|
+
correction: "correction",
|
|
3572
|
+
frustration: "frustration",
|
|
3573
|
+
jargon: "jargon-complaint",
|
|
3574
|
+
structure: "structure-complaint",
|
|
3575
|
+
praise: "praise"
|
|
3576
|
+
};
|
|
3577
|
+
function ratioStr(r) {
|
|
3578
|
+
if (r === null) return "n/a (no reaction signals)";
|
|
3579
|
+
if (!Number.isFinite(r)) return "\u221E (corrective with zero praise)";
|
|
3580
|
+
return `${r.toFixed(2)}:1`;
|
|
3581
|
+
}
|
|
3582
|
+
function oneLine(s, n) {
|
|
3583
|
+
const t = s.replace(/\s+/g, " ").trim();
|
|
3584
|
+
return t.length > n ? `${t.slice(0, n)}\u2026` : t;
|
|
3585
|
+
}
|
|
3586
|
+
function renderReactions(rr) {
|
|
3587
|
+
const lines = ["## user reactions (deterministic, human turns only)", ""];
|
|
3588
|
+
const total = Object.values(rr.signals).reduce((a, b) => a + b, 0);
|
|
3589
|
+
if (rr.humanReactionTurns === 0) {
|
|
3590
|
+
lines.push("- No human turns followed an assistant turn (nothing to classify).");
|
|
3591
|
+
lines.push("");
|
|
3592
|
+
return lines.join("\n");
|
|
3593
|
+
}
|
|
3594
|
+
lines.push(
|
|
3595
|
+
`- **Reaction turns:** ${rr.humanReactionTurns} human turn(s) followed an assistant turn; ${total} carried a reaction signal.`
|
|
3596
|
+
);
|
|
3597
|
+
lines.push(`- **Corrective-to-positive ratio:** ${ratioStr(rr.correctiveToPositiveRatio)}`);
|
|
3598
|
+
lines.push("");
|
|
3599
|
+
lines.push("| Signal | Count |");
|
|
3600
|
+
lines.push("|---|---|");
|
|
3601
|
+
for (const [k, v] of Object.entries(rr.signals)) {
|
|
3602
|
+
if (v > 0) lines.push(`| ${REACTION_BADGE[k] ?? k} | ${v} |`);
|
|
3603
|
+
}
|
|
3604
|
+
lines.push("");
|
|
3605
|
+
if (rr.triggerPairs.length > 0) {
|
|
3606
|
+
lines.push("### top trigger pairs (assistant prose \u2192 human reaction)");
|
|
3607
|
+
lines.push("");
|
|
3608
|
+
for (const p of rr.triggerPairs) {
|
|
3609
|
+
lines.push(`- **[${p.reactions.map((r) => REACTION_BADGE[r] ?? r).join(", ")}]**`);
|
|
3610
|
+
lines.push(` - assistant: \`${oneLine(p.assistant, 180)}\``);
|
|
3611
|
+
lines.push(` - human: \`${oneLine(p.human, 180)}\``);
|
|
3612
|
+
}
|
|
3613
|
+
lines.push("");
|
|
3614
|
+
}
|
|
3615
|
+
return lines.join("\n");
|
|
3616
|
+
}
|
|
3617
|
+
function renderAdoption(ar) {
|
|
3618
|
+
const lines = ["## skill & subagent adoption (deterministic)", ""];
|
|
3619
|
+
lines.push(
|
|
3620
|
+
`- **Skill penetration:** ${(ar.skillPenetration * 100).toFixed(0)}% (${ar.sessionsWithSkill}/${ar.sessionCount} session(s) invoked a skill explicitly)`
|
|
3621
|
+
);
|
|
3622
|
+
lines.push(
|
|
3623
|
+
`- **Explicit skill invocations:** ${ar.totalSkillInvocations} (Skill tool spans) \xB7 **Loop-dispatched runs:** ${ar.totalLoopDispatchedRuns} (from ${ar.skillRunFilesRead} \`.evolve/skill-runs.jsonl\` file(s))`
|
|
3624
|
+
);
|
|
3625
|
+
lines.push(
|
|
3626
|
+
`- **Subagent spawns:** ${ar.totalSubagentSpawns} across ${ar.sessionsWithSubagent} session(s)`
|
|
3627
|
+
);
|
|
3628
|
+
lines.push("");
|
|
3629
|
+
const skillRows = Object.entries(ar.skillInvocations).sort((a, b) => b[1] - a[1]);
|
|
3630
|
+
const loopRows = Object.entries(ar.loopDispatchedRuns).sort((a, b) => b[1] - a[1]);
|
|
3631
|
+
if (skillRows.length > 0 || loopRows.length > 0) {
|
|
3632
|
+
lines.push("| Skill | Explicit invocations | Loop-dispatched runs |");
|
|
3633
|
+
lines.push("|---|---|---|");
|
|
3634
|
+
const names = [.../* @__PURE__ */ new Set([...skillRows.map((r) => r[0]), ...loopRows.map((r) => r[0])])].sort(
|
|
3635
|
+
(a, b) => (ar.skillInvocations[b] ?? 0) + (ar.loopDispatchedRuns[b] ?? 0) - (ar.skillInvocations[a] ?? 0) - (ar.loopDispatchedRuns[a] ?? 0)
|
|
3636
|
+
);
|
|
3637
|
+
for (const n of names) {
|
|
3638
|
+
lines.push(`| \`${n}\` | ${ar.skillInvocations[n] ?? 0} | ${ar.loopDispatchedRuns[n] ?? 0} |`);
|
|
3639
|
+
}
|
|
3640
|
+
lines.push("");
|
|
3641
|
+
}
|
|
3642
|
+
const agentRows = Object.entries(ar.subagentSpawns).sort((a, b) => b[1] - a[1]);
|
|
3643
|
+
if (agentRows.length > 0) {
|
|
3644
|
+
lines.push("| Subagent | Spawns |");
|
|
3645
|
+
lines.push("|---|---|");
|
|
3646
|
+
for (const [n, c] of agentRows) lines.push(`| \`${n}\` | ${c} |`);
|
|
3647
|
+
lines.push("");
|
|
3648
|
+
}
|
|
3649
|
+
return lines.join("\n");
|
|
3650
|
+
}
|
|
3651
|
+
|
|
3652
|
+
// src/improvement.ts
|
|
3653
|
+
import { access, mkdir, mkdtemp as mkdtemp4, writeFile as writeFile4 } from "fs/promises";
|
|
3545
3654
|
import { tmpdir as tmpdir4 } from "os";
|
|
3546
|
-
import { dirname as dirname2, join as join16 } from "path";
|
|
3655
|
+
import { dirname as dirname2, join as join16, resolve } from "path";
|
|
3656
|
+
import { pathToFileURL } from "url";
|
|
3657
|
+
import {
|
|
3658
|
+
AnalystRegistry,
|
|
3659
|
+
buildDefaultAnalystRegistry as buildDefaultAnalystRegistry2,
|
|
3660
|
+
makeFinding
|
|
3661
|
+
} from "@tangle-network/agent-eval/analyst";
|
|
3662
|
+
var severityRank2 = {
|
|
3663
|
+
critical: 0,
|
|
3664
|
+
high: 1,
|
|
3665
|
+
medium: 2,
|
|
3666
|
+
low: 3,
|
|
3667
|
+
info: 4
|
|
3668
|
+
};
|
|
3669
|
+
function slug(input) {
|
|
3670
|
+
return input.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 72) || "item";
|
|
3671
|
+
}
|
|
3672
|
+
function json(value) {
|
|
3673
|
+
return `${JSON.stringify(value, null, 2)}
|
|
3674
|
+
`;
|
|
3675
|
+
}
|
|
3676
|
+
function jsonl(rows) {
|
|
3677
|
+
return rows.length === 0 ? "" : `${rows.map((row) => JSON.stringify(row)).join("\n")}
|
|
3678
|
+
`;
|
|
3679
|
+
}
|
|
3680
|
+
function evidence(kind, uri, excerpt) {
|
|
3681
|
+
return excerpt ? { kind, uri, excerpt } : { kind, uri };
|
|
3682
|
+
}
|
|
3683
|
+
function totalReactionSignals(reactions) {
|
|
3684
|
+
return Object.values(reactions.signals).reduce((sum, count) => sum + count, 0);
|
|
3685
|
+
}
|
|
3686
|
+
function toolErrorRuns(pipelines) {
|
|
3687
|
+
return pipelines.toolUse.filter((run) => run.errorRate > 0).length;
|
|
3688
|
+
}
|
|
3689
|
+
function deterministicFindings(pipelines, reactions, adoption) {
|
|
3690
|
+
const findings = [];
|
|
3691
|
+
const loops = pipelines.stuckLoops.findings;
|
|
3692
|
+
if (loops.length > 0) {
|
|
3693
|
+
const top = [...loops].sort((a, b) => b.occurrences - a.occurrences)[0];
|
|
3694
|
+
findings.push(makeFinding({
|
|
3695
|
+
analyst_id: "traces-deterministic",
|
|
3696
|
+
area: "tool-use",
|
|
3697
|
+
claim: `${loops.length} repeated tool-call loop(s) were observed`,
|
|
3698
|
+
severity: loops.length >= 10 ? "high" : "medium",
|
|
3699
|
+
rationale: "Repeated identical tool calls usually mean the agent is retrying without new information or a stop rule.",
|
|
3700
|
+
evidence_refs: [
|
|
3701
|
+
evidence("metric", "pipelines.stuck_loop_count", `${loops.length} loop(s)`),
|
|
3702
|
+
...top ? [evidence("metric", `tool.${top.toolName}.repeated_calls`, `${top.toolName} repeated ${top.occurrences} time(s)`)] : []
|
|
3703
|
+
],
|
|
3704
|
+
recommended_action: "Add a loop breaker: when the same tool and arguments fail or repeat, force a state check, alternate plan, or stop condition before retrying.",
|
|
3705
|
+
validation_plan: "Rerun traces over fresh sessions and require repeated-call loops to drop to zero or explain each remaining loop with changed state.",
|
|
3706
|
+
confidence: 0.95,
|
|
3707
|
+
metadata: { source: "traces.pipeline.stuckLoopView", loopCount: loops.length }
|
|
3708
|
+
}));
|
|
3709
|
+
}
|
|
3710
|
+
const erroredRuns = toolErrorRuns(pipelines);
|
|
3711
|
+
if (erroredRuns > 0) {
|
|
3712
|
+
findings.push(makeFinding({
|
|
3713
|
+
analyst_id: "traces-deterministic",
|
|
3714
|
+
area: "tool-use",
|
|
3715
|
+
claim: `${erroredRuns} run(s) included tool errors`,
|
|
3716
|
+
severity: erroredRuns >= 5 ? "high" : "medium",
|
|
3717
|
+
rationale: "Tool errors are expensive when the agent keeps planning as if the call succeeded.",
|
|
3718
|
+
evidence_refs: [evidence("metric", "pipelines.tool_error_runs", `${erroredRuns} run(s) with errorRate > 0`)],
|
|
3719
|
+
recommended_action: "Fix the highest-frequency failing tool path first, then make the agent inspect failure output before retrying.",
|
|
3720
|
+
validation_plan: "Rerun the same trace window and require the tool-error run count and repeated retries to decrease.",
|
|
3721
|
+
confidence: 0.9,
|
|
3722
|
+
metadata: { source: "traces.pipeline.computeToolUseMetrics", erroredRuns }
|
|
3723
|
+
}));
|
|
3724
|
+
}
|
|
3725
|
+
const corrective = reactions.signals.correction + reactions.signals.frustration + reactions.signals.jargon + reactions.signals.structure;
|
|
3726
|
+
if (corrective > 0) {
|
|
3727
|
+
findings.push(makeFinding({
|
|
3728
|
+
analyst_id: "traces-deterministic",
|
|
3729
|
+
area: "communication",
|
|
3730
|
+
claim: `${corrective} corrective human reaction signal(s) followed assistant turns`,
|
|
3731
|
+
severity: corrective >= 10 ? "high" : "medium",
|
|
3732
|
+
rationale: "Human corrections are direct product feedback: the agent either missed the task, overexplained, used unclear language, or failed to adapt.",
|
|
3733
|
+
evidence_refs: reactions.triggerPairs.slice(0, 3).map((pair, index) => evidence("event", `reaction.trigger_pair.${index + 1}`, `${pair.reactions.join(", ")}: ${pair.human.slice(0, 180)}`)),
|
|
3734
|
+
recommended_action: "Turn the top correction pattern into an agent profile rule or analyst skill, then rerun traces and check the corrective-to-positive ratio.",
|
|
3735
|
+
validation_plan: "Rerun traces on fresh sessions and require corrective reaction signals per human reaction turn to decrease.",
|
|
3736
|
+
confidence: 0.85,
|
|
3737
|
+
metadata: { source: "traces.reactions", correctiveSignals: corrective, totalSignals: totalReactionSignals(reactions) }
|
|
3738
|
+
}));
|
|
3739
|
+
}
|
|
3740
|
+
const skillRuns = adoption.totalSkillInvocations + adoption.totalLoopDispatchedRuns;
|
|
3741
|
+
if (adoption.sessionCount > 0 && skillRuns === 0) {
|
|
3742
|
+
findings.push(makeFinding({
|
|
3743
|
+
analyst_id: "traces-deterministic",
|
|
3744
|
+
area: "workflow",
|
|
3745
|
+
claim: "No skill usage was observed in the selected sessions",
|
|
3746
|
+
severity: "low",
|
|
3747
|
+
rationale: "Repeatable work without explicit skills is harder to improve because the trace has no reusable policy boundary to patch.",
|
|
3748
|
+
evidence_refs: [evidence("metric", "adoption.skill_runs", "0 explicit or loop-dispatched skill runs")],
|
|
3749
|
+
recommended_action: "Create or invoke a narrow skill for the repeated workflow, then let traces compare future sessions against this baseline.",
|
|
3750
|
+
validation_plan: "Rerun traces and require skill-run attribution to appear for repeated workflows.",
|
|
3751
|
+
confidence: 0.75,
|
|
3752
|
+
metadata: { source: "traces.adoption", sessionCount: adoption.sessionCount }
|
|
3753
|
+
}));
|
|
3754
|
+
}
|
|
3755
|
+
return findings;
|
|
3756
|
+
}
|
|
3757
|
+
function validSeverity(value) {
|
|
3758
|
+
return value === "critical" || value === "high" || value === "medium" || value === "low" || value === "info";
|
|
3759
|
+
}
|
|
3760
|
+
function validEvidenceRef(value) {
|
|
3761
|
+
if (!value || typeof value !== "object") return false;
|
|
3762
|
+
const row = value;
|
|
3763
|
+
return (row.kind === "span" || row.kind === "event" || row.kind === "artifact" || row.kind === "finding" || row.kind === "metric") && typeof row.uri === "string" && (row.excerpt === void 0 || typeof row.excerpt === "string");
|
|
3764
|
+
}
|
|
3765
|
+
function externalRows(output) {
|
|
3766
|
+
const trimmed = output.trim();
|
|
3767
|
+
if (!trimmed.startsWith("{") && !trimmed.startsWith("[")) return [];
|
|
3768
|
+
try {
|
|
3769
|
+
const parsed = JSON.parse(trimmed);
|
|
3770
|
+
if (Array.isArray(parsed)) return parsed;
|
|
3771
|
+
if (parsed && typeof parsed === "object") {
|
|
3772
|
+
const row = parsed;
|
|
3773
|
+
if (Array.isArray(row.findings)) return row.findings;
|
|
3774
|
+
if (Array.isArray(row.issues)) return row.issues;
|
|
3775
|
+
if (Array.isArray(row.recommendations)) return row.recommendations;
|
|
3776
|
+
}
|
|
3777
|
+
} catch {
|
|
3778
|
+
return [];
|
|
3779
|
+
}
|
|
3780
|
+
return [];
|
|
3781
|
+
}
|
|
3782
|
+
function coerceExternalFinding(analyzer, row, index) {
|
|
3783
|
+
if (!row || typeof row !== "object") return null;
|
|
3784
|
+
const item = row;
|
|
3785
|
+
const claim = item.claim ?? item.title ?? item.issue ?? item.recommendation;
|
|
3786
|
+
if (typeof claim !== "string" || claim.trim().length === 0) return null;
|
|
3787
|
+
const evidenceRefs = Array.isArray(item.evidence_refs) ? item.evidence_refs.filter(validEvidenceRef) : Array.isArray(item.evidenceRefs) ? item.evidenceRefs.filter(validEvidenceRef) : [];
|
|
3788
|
+
return makeFinding({
|
|
3789
|
+
analyst_id: `external:${analyzer}`,
|
|
3790
|
+
area: typeof item.area === "string" && item.area ? item.area : "external",
|
|
3791
|
+
claim,
|
|
3792
|
+
severity: validSeverity(item.severity) ? item.severity : "medium",
|
|
3793
|
+
rationale: typeof item.rationale === "string" ? item.rationale : void 0,
|
|
3794
|
+
evidence_refs: evidenceRefs,
|
|
3795
|
+
recommended_action: typeof item.recommended_action === "string" ? item.recommended_action : typeof item.action === "string" ? item.action : void 0,
|
|
3796
|
+
validation_plan: typeof item.validation_plan === "string" ? item.validation_plan : void 0,
|
|
3797
|
+
confidence: typeof item.confidence === "number" && Number.isFinite(item.confidence) ? item.confidence : 0.65,
|
|
3798
|
+
subject: typeof item.subject === "string" ? item.subject : void 0,
|
|
3799
|
+
metadata: { source: "traces.external", analyzer, rowIndex: index }
|
|
3800
|
+
});
|
|
3801
|
+
}
|
|
3802
|
+
function normalizeExternalFindings(results) {
|
|
3803
|
+
const out = [];
|
|
3804
|
+
for (const result of results) {
|
|
3805
|
+
if (result.findings) out.push(...result.findings);
|
|
3806
|
+
externalRows(result.output).forEach((row, index) => {
|
|
3807
|
+
const finding = coerceExternalFinding(result.analyzer, row, index);
|
|
3808
|
+
if (finding) out.push(finding);
|
|
3809
|
+
});
|
|
3810
|
+
}
|
|
3811
|
+
return out;
|
|
3812
|
+
}
|
|
3813
|
+
function sourceOf(finding) {
|
|
3814
|
+
if (finding.analyst_id.startsWith("external:")) return "external";
|
|
3815
|
+
if (finding.analyst_id === "traces-deterministic") return "deterministic";
|
|
3816
|
+
return "analyst";
|
|
3817
|
+
}
|
|
3818
|
+
function recommendationTitle(finding) {
|
|
3819
|
+
if (finding.recommended_action) return finding.recommended_action.replace(/\.$/, "");
|
|
3820
|
+
return `Investigate ${finding.area}: ${finding.claim}`.replace(/\.$/, "");
|
|
3821
|
+
}
|
|
3822
|
+
function buildRecommendations(findings) {
|
|
3823
|
+
return [...findings].filter((finding) => finding.severity !== "info").sort((a, b) => severityRank2[a.severity] - severityRank2[b.severity] || b.confidence - a.confidence || a.finding_id.localeCompare(b.finding_id)).slice(0, 20).map((finding, index) => ({
|
|
3824
|
+
schemaVersion: 1,
|
|
3825
|
+
id: `rec-${index + 1}-${slug(finding.area)}`,
|
|
3826
|
+
severity: finding.severity,
|
|
3827
|
+
title: recommendationTitle(finding),
|
|
3828
|
+
action: finding.recommended_action ?? `Review and fix: ${finding.claim}`,
|
|
3829
|
+
rationale: finding.rationale ?? finding.claim,
|
|
3830
|
+
evidenceRefs: finding.evidence_refs,
|
|
3831
|
+
findingIds: [finding.finding_id],
|
|
3832
|
+
validationPlan: finding.validation_plan ?? "Rerun `traces improve` on a fresh comparable session window and confirm this finding disappears or drops in severity.",
|
|
3833
|
+
source: sourceOf(finding)
|
|
3834
|
+
}));
|
|
3835
|
+
}
|
|
3836
|
+
function buildClaims(findings) {
|
|
3837
|
+
return findings.map((finding, index) => ({
|
|
3838
|
+
schemaVersion: 1,
|
|
3839
|
+
id: `claim-${index + 1}-${slug(finding.area)}`,
|
|
3840
|
+
severity: finding.severity,
|
|
3841
|
+
text: finding.claim,
|
|
3842
|
+
area: finding.area,
|
|
3843
|
+
sourceFindingId: finding.finding_id,
|
|
3844
|
+
evidenceRefs: finding.evidence_refs,
|
|
3845
|
+
validationPlan: finding.validation_plan
|
|
3846
|
+
}));
|
|
3847
|
+
}
|
|
3848
|
+
function buildEvidenceRows(findings) {
|
|
3849
|
+
return findings.flatMap((finding) => finding.evidence_refs.map((ref) => ({
|
|
3850
|
+
schemaVersion: 1,
|
|
3851
|
+
kind: "traces.improvement_evidence",
|
|
3852
|
+
findingId: finding.finding_id,
|
|
3853
|
+
severity: finding.severity,
|
|
3854
|
+
area: finding.area,
|
|
3855
|
+
claim: finding.claim,
|
|
3856
|
+
evidence: ref
|
|
3857
|
+
})));
|
|
3858
|
+
}
|
|
3859
|
+
function renderRecommendations(recommendations) {
|
|
3860
|
+
const lines = ["## recommendations", ""];
|
|
3861
|
+
if (recommendations.length === 0) {
|
|
3862
|
+
lines.push("- No recommendations emitted.");
|
|
3863
|
+
lines.push("");
|
|
3864
|
+
return lines.join("\n");
|
|
3865
|
+
}
|
|
3866
|
+
for (const rec of recommendations.slice(0, 10)) {
|
|
3867
|
+
lines.push(`### ${rec.severity.toUpperCase()} \u2014 ${rec.title}`);
|
|
3868
|
+
lines.push("");
|
|
3869
|
+
lines.push(`- **Action:** ${rec.action}`);
|
|
3870
|
+
lines.push(`- **Why:** ${rec.rationale}`);
|
|
3871
|
+
lines.push(`- **Check:** ${rec.validationPlan}`);
|
|
3872
|
+
for (const ref of rec.evidenceRefs.slice(0, 3)) {
|
|
3873
|
+
lines.push(`- **Evidence:** ${ref.kind} ${ref.uri}${ref.excerpt ? ` \u2014 \`${ref.excerpt.slice(0, 180)}\`` : ""}`);
|
|
3874
|
+
}
|
|
3875
|
+
lines.push("");
|
|
3876
|
+
}
|
|
3877
|
+
return lines.join("\n");
|
|
3878
|
+
}
|
|
3879
|
+
function renderFindingPacket(packet, title = "Trace finding packet") {
|
|
3880
|
+
const lines = [`# ${title}`, ""];
|
|
3881
|
+
lines.push(`${packet.findings.length} finding(s), ${packet.recommendations.length} recommendation(s), ${packet.claims.length} claim(s).`);
|
|
3882
|
+
lines.push("");
|
|
3883
|
+
lines.push(renderRecommendations(packet.recommendations));
|
|
3884
|
+
if (packet.findings.length > 0) {
|
|
3885
|
+
lines.push("## findings");
|
|
3886
|
+
lines.push("");
|
|
3887
|
+
for (const finding of [...packet.findings].sort((a, b) => severityRank2[a.severity] - severityRank2[b.severity] || a.area.localeCompare(b.area) || a.claim.localeCompare(b.claim))) {
|
|
3888
|
+
lines.push(`### ${finding.severity.toUpperCase()} \u2014 ${finding.claim}`);
|
|
3889
|
+
lines.push("");
|
|
3890
|
+
lines.push(`- **Area:** ${finding.area}`);
|
|
3891
|
+
lines.push(`- **Analyst:** ${finding.analyst_id}`);
|
|
3892
|
+
if (finding.recommended_action) lines.push(`- **Action:** ${finding.recommended_action}`);
|
|
3893
|
+
if (finding.validation_plan) lines.push(`- **Check:** ${finding.validation_plan}`);
|
|
3894
|
+
for (const ref of finding.evidence_refs.slice(0, 3)) {
|
|
3895
|
+
lines.push(`- **Evidence:** ${ref.kind} ${ref.uri}${ref.excerpt ? ` \u2014 \`${ref.excerpt.slice(0, 180)}\`` : ""}`);
|
|
3896
|
+
}
|
|
3897
|
+
lines.push("");
|
|
3898
|
+
}
|
|
3899
|
+
}
|
|
3900
|
+
return lines.join("\n");
|
|
3901
|
+
}
|
|
3902
|
+
function buildTraceFindingPacket(opts) {
|
|
3903
|
+
const generatedAt = opts.generatedAt ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
3904
|
+
const packet = {
|
|
3905
|
+
schemaVersion: 1,
|
|
3906
|
+
kind: "traces.finding_packet",
|
|
3907
|
+
generatedAt,
|
|
3908
|
+
source: opts.source ?? "traces",
|
|
3909
|
+
findings: opts.findings,
|
|
3910
|
+
recommendations: buildRecommendations(opts.findings),
|
|
3911
|
+
claims: buildClaims(opts.findings)
|
|
3912
|
+
};
|
|
3913
|
+
return { ...packet, report: renderFindingPacket(packet, opts.title) };
|
|
3914
|
+
}
|
|
3915
|
+
function renderExternal(results) {
|
|
3916
|
+
if (results.length === 0) return "";
|
|
3917
|
+
const lines = ["## external analyzers", ""];
|
|
3918
|
+
for (const result of results) {
|
|
3919
|
+
lines.push(`### ${result.analyzer}`);
|
|
3920
|
+
lines.push("");
|
|
3921
|
+
lines.push(result.ok ? result.output || "(no output)" : `failed: ${result.error}`);
|
|
3922
|
+
lines.push("");
|
|
3923
|
+
}
|
|
3924
|
+
return lines.join("\n");
|
|
3925
|
+
}
|
|
3926
|
+
function renderInvestigationReport(result, analystResult) {
|
|
3927
|
+
const base = `${renderReport({ ...analystResult, findings: [...result.findings] }, {
|
|
3928
|
+
harness: result.harness,
|
|
3929
|
+
sessionCount: result.sessionCount,
|
|
3930
|
+
spanCount: result.spanCount,
|
|
3931
|
+
otlpPath: result.otlpPath,
|
|
3932
|
+
deterministic: summarizeDeterministicSignals(result.pipelines, result.reactions)
|
|
3933
|
+
})}
|
|
3934
|
+
${renderRecommendations(result.recommendations)}
|
|
3935
|
+
${renderPipelines(result.pipelines)}
|
|
3936
|
+
${renderReactions(result.reactions)}
|
|
3937
|
+
${renderAdoption(result.adoption)}`;
|
|
3938
|
+
const external = renderExternal(result.external);
|
|
3939
|
+
return external ? `${base}
|
|
3940
|
+
${external}` : base;
|
|
3941
|
+
}
|
|
3942
|
+
function registryFromConfig(config) {
|
|
3943
|
+
if (!config) return void 0;
|
|
3944
|
+
if (config.registry) return config.registry;
|
|
3945
|
+
if (!config.analysts || config.analysts.length === 0) return void 0;
|
|
3946
|
+
const registry = new AnalystRegistry();
|
|
3947
|
+
for (const analyst of config.analysts) registry.register(analyst);
|
|
3948
|
+
return registry;
|
|
3949
|
+
}
|
|
3950
|
+
async function existingConfigPath(path) {
|
|
3951
|
+
const candidates = path ? [path] : ["traces.config.mjs", "traces.config.js", "traces.config.cjs"];
|
|
3952
|
+
for (const candidate of candidates) {
|
|
3953
|
+
const resolved = resolve(candidate);
|
|
3954
|
+
try {
|
|
3955
|
+
await access(resolved);
|
|
3956
|
+
return resolved;
|
|
3957
|
+
} catch {
|
|
3958
|
+
if (path) return void 0;
|
|
3959
|
+
}
|
|
3960
|
+
}
|
|
3961
|
+
return void 0;
|
|
3962
|
+
}
|
|
3963
|
+
async function loadTracesConfig(path) {
|
|
3964
|
+
const resolved = await existingConfigPath(path);
|
|
3965
|
+
if (!resolved) return void 0;
|
|
3966
|
+
let mod;
|
|
3967
|
+
try {
|
|
3968
|
+
mod = await import(pathToFileURL(resolved).href);
|
|
3969
|
+
} catch (err) {
|
|
3970
|
+
if (resolved.endsWith(".ts")) {
|
|
3971
|
+
throw new Error(`traces config is TypeScript (${resolved}); use traces.config.mjs/js, or run the CLI through a TS loader`);
|
|
3972
|
+
}
|
|
3973
|
+
throw err;
|
|
3974
|
+
}
|
|
3975
|
+
const value = mod.default ?? mod.config;
|
|
3976
|
+
if (value === void 0) return void 0;
|
|
3977
|
+
if (!value || typeof value !== "object") throw new Error(`${path} must export a config object`);
|
|
3978
|
+
return value;
|
|
3979
|
+
}
|
|
3980
|
+
function mergeTracesConfig(opts, config) {
|
|
3981
|
+
if (!config) return opts;
|
|
3982
|
+
return {
|
|
3983
|
+
...opts,
|
|
3984
|
+
registry: opts.registry ?? registryFromConfig(config),
|
|
3985
|
+
externalAnalyzers: [...config.externalAnalyzers ?? [], ...opts.externalAnalyzers ?? []]
|
|
3986
|
+
};
|
|
3987
|
+
}
|
|
3988
|
+
async function runTraceInvestigation(opts) {
|
|
3989
|
+
if (opts.spans.length === 0) throw new Error("runTraceInvestigation: no spans to analyze");
|
|
3990
|
+
const generatedAt = opts.generatedAt ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
3991
|
+
const [analysis, pipelines, reactions, adoption] = await Promise.all([
|
|
3992
|
+
analyzeSpans(opts.spans, {
|
|
3993
|
+
ai: opts.ai,
|
|
3994
|
+
model: opts.model,
|
|
3995
|
+
budgetUsd: opts.budgetUsd,
|
|
3996
|
+
registry: opts.registry,
|
|
3997
|
+
otlpOutPath: opts.otlpOutPath,
|
|
3998
|
+
runId: `traces-investigation-${Date.parse(generatedAt) || Date.now()}`,
|
|
3999
|
+
log: opts.log
|
|
4000
|
+
}),
|
|
4001
|
+
runPipelines(opts.spans, { minLoopOccurrences: opts.minLoopOccurrences }),
|
|
4002
|
+
Promise.resolve(analyzeReactions(opts.spans)),
|
|
4003
|
+
analyzeAdoption(opts.spans, { cwds: opts.cwds })
|
|
4004
|
+
]);
|
|
4005
|
+
const external = opts.externalAnalyzers?.length ? await runExternalAnalyzers(analysis.otlpPath, opts.externalAnalyzers, { prompt: opts.analyzerPrompt }) : [];
|
|
4006
|
+
const findings = [
|
|
4007
|
+
...analysis.result.findings,
|
|
4008
|
+
...deterministicFindings(pipelines, reactions, adoption),
|
|
4009
|
+
...normalizeExternalFindings(external)
|
|
4010
|
+
];
|
|
4011
|
+
const recommendations = buildRecommendations(findings);
|
|
4012
|
+
const claims = buildClaims(findings);
|
|
4013
|
+
const partial = {
|
|
4014
|
+
schemaVersion: 1,
|
|
4015
|
+
kind: "traces.investigation",
|
|
4016
|
+
generatedAt,
|
|
4017
|
+
harness: opts.harness,
|
|
4018
|
+
sessionCount: opts.sessionCount,
|
|
4019
|
+
spanCount: opts.spans.length,
|
|
4020
|
+
otlpPath: analysis.otlpPath,
|
|
4021
|
+
findings,
|
|
4022
|
+
recommendations,
|
|
4023
|
+
claims,
|
|
4024
|
+
pipelines,
|
|
4025
|
+
reactions,
|
|
4026
|
+
adoption,
|
|
4027
|
+
external
|
|
4028
|
+
};
|
|
4029
|
+
const result = { ...partial, report: "" };
|
|
4030
|
+
return { ...result, report: renderInvestigationReport(result, analysis.result) };
|
|
4031
|
+
}
|
|
4032
|
+
async function runTraceStoreInvestigation(opts) {
|
|
4033
|
+
const generatedAt = opts.generatedAt ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
4034
|
+
const registry = opts.registry ?? buildDefaultAnalystRegistry2({
|
|
4035
|
+
ai: opts.ai,
|
|
4036
|
+
model: opts.model,
|
|
4037
|
+
registry: { log: opts.log }
|
|
4038
|
+
});
|
|
4039
|
+
const runId = opts.runId ?? `traces-store-investigation-${Date.parse(generatedAt) || Date.now()}`;
|
|
4040
|
+
const analystResult = await registry.run(runId, { traceStore: opts.traceStore }, {
|
|
4041
|
+
budget: opts.budgetUsd != null ? { totalUsd: opts.budgetUsd } : void 0
|
|
4042
|
+
});
|
|
4043
|
+
const packet = buildTraceFindingPacket({
|
|
4044
|
+
findings: analystResult.findings,
|
|
4045
|
+
generatedAt,
|
|
4046
|
+
source: "trace-store",
|
|
4047
|
+
title: "Trace store investigation"
|
|
4048
|
+
});
|
|
4049
|
+
return {
|
|
4050
|
+
...packet,
|
|
4051
|
+
kind: "traces.store_investigation",
|
|
4052
|
+
analystResult
|
|
4053
|
+
};
|
|
4054
|
+
}
|
|
4055
|
+
function replayProof(result, proposals) {
|
|
4056
|
+
const deterministic = summarizeDeterministicSignals(result.pipelines, result.reactions);
|
|
4057
|
+
return {
|
|
4058
|
+
schemaVersion: 1,
|
|
4059
|
+
kind: "traces.improvement_replay",
|
|
4060
|
+
generatedAt: result.generatedAt,
|
|
4061
|
+
status: proposals.length > 0 ? "proposal-only" : "not-run",
|
|
4062
|
+
candidateApplied: false,
|
|
4063
|
+
baseline: {
|
|
4064
|
+
spanCount: result.spanCount,
|
|
4065
|
+
findingCount: result.findings.length,
|
|
4066
|
+
recommendationCount: result.recommendations.length,
|
|
4067
|
+
stuckLoops: deterministic.stuckLoops,
|
|
4068
|
+
reactionSignals: deterministic.reactionSignals,
|
|
4069
|
+
toolErrorRuns: deterministic.toolErrorRuns
|
|
4070
|
+
},
|
|
4071
|
+
proposals,
|
|
4072
|
+
note: proposals.length > 0 ? "traces produced reviewable proposals only; apply one and rerun traces on a comparable window to populate after metrics." : "no proposal adapter was configured, so no candidate was applied; recommendations remain reviewable guidance."
|
|
4073
|
+
};
|
|
4074
|
+
}
|
|
4075
|
+
async function writeTraceImprovementArtifacts(result, outDir) {
|
|
4076
|
+
const directory = outDir ? resolve(outDir) : await mkdtemp4(join16(tmpdir4(), "traces-improvement-"));
|
|
4077
|
+
await mkdir(directory, { recursive: true });
|
|
4078
|
+
const paths = {
|
|
4079
|
+
directory,
|
|
4080
|
+
findings: join16(directory, "findings.json"),
|
|
4081
|
+
recommendations: join16(directory, "recommendations.json"),
|
|
4082
|
+
evidence: join16(directory, "evidence.jsonl"),
|
|
4083
|
+
claims: join16(directory, "claims.json"),
|
|
4084
|
+
report: join16(directory, "report.md"),
|
|
4085
|
+
replay: join16(directory, "replay-before-after.json")
|
|
4086
|
+
};
|
|
4087
|
+
await Promise.all([
|
|
4088
|
+
writeFile4(paths.findings, json(result.findings), "utf8"),
|
|
4089
|
+
writeFile4(paths.recommendations, json(result.recommendations), "utf8"),
|
|
4090
|
+
writeFile4(paths.evidence, jsonl(buildEvidenceRows(result.findings)), "utf8"),
|
|
4091
|
+
writeFile4(paths.claims, json(result.claims), "utf8"),
|
|
4092
|
+
writeFile4(paths.report, result.report, "utf8"),
|
|
4093
|
+
writeFile4(paths.replay, json(result.replay), "utf8")
|
|
4094
|
+
]);
|
|
4095
|
+
return paths;
|
|
4096
|
+
}
|
|
4097
|
+
async function runTraceImprovementLoop(opts) {
|
|
4098
|
+
const investigation = await runTraceInvestigation(opts);
|
|
4099
|
+
const proposals = opts.adapter ? await opts.adapter.propose({
|
|
4100
|
+
findings: investigation.findings,
|
|
4101
|
+
recommendations: investigation.recommendations,
|
|
4102
|
+
claims: investigation.claims,
|
|
4103
|
+
spans: opts.spans,
|
|
4104
|
+
otlpPath: investigation.otlpPath
|
|
4105
|
+
}) : [];
|
|
4106
|
+
const replay = replayProof(investigation, proposals);
|
|
4107
|
+
const result = {
|
|
4108
|
+
...investigation,
|
|
4109
|
+
kind: "traces.improvement",
|
|
4110
|
+
proposals,
|
|
4111
|
+
replay
|
|
4112
|
+
};
|
|
4113
|
+
return {
|
|
4114
|
+
...result,
|
|
4115
|
+
artifacts: await writeTraceImprovementArtifacts(result, opts.outDir)
|
|
4116
|
+
};
|
|
4117
|
+
}
|
|
4118
|
+
async function saveReport(path, report) {
|
|
4119
|
+
await mkdir(dirname2(resolve(path)), { recursive: true });
|
|
4120
|
+
await writeFile4(path, report, "utf8");
|
|
4121
|
+
}
|
|
4122
|
+
|
|
4123
|
+
// src/observer.ts
|
|
4124
|
+
var sleep = (ms, signal) => new Promise((resolve2) => {
|
|
4125
|
+
const onAbort = () => {
|
|
4126
|
+
clearTimeout(t);
|
|
4127
|
+
resolve2();
|
|
4128
|
+
};
|
|
4129
|
+
const t = setTimeout(() => {
|
|
4130
|
+
signal?.removeEventListener("abort", onAbort);
|
|
4131
|
+
resolve2();
|
|
4132
|
+
}, ms);
|
|
4133
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
4134
|
+
});
|
|
4135
|
+
function loopKey(sessionId, toolName2, argHash) {
|
|
4136
|
+
return `${sessionId}\0${toolName2}\0${argHash}`;
|
|
4137
|
+
}
|
|
4138
|
+
async function watchSessions(opts) {
|
|
4139
|
+
const windowMs = opts.windowMs ?? 30 * 6e4;
|
|
4140
|
+
const intervalMs = Math.max(250, opts.intervalMs ?? 5e3);
|
|
4141
|
+
const minLoop = opts.minLoopOccurrences ?? 3;
|
|
4142
|
+
const seen = /* @__PURE__ */ new Map();
|
|
4143
|
+
while (!opts.signal?.aborted) {
|
|
4144
|
+
let sessionCount = 0;
|
|
4145
|
+
for await (const { adapter, ref, spans } of scanSessions({
|
|
4146
|
+
adapters: opts.adapters,
|
|
4147
|
+
all: opts.all,
|
|
4148
|
+
harnesses: opts.harnesses,
|
|
4149
|
+
cwd: opts.cwd,
|
|
4150
|
+
sinceMs: Date.now() - windowMs,
|
|
4151
|
+
signal: opts.signal,
|
|
4152
|
+
onError: opts.onError
|
|
4153
|
+
})) {
|
|
4154
|
+
let report;
|
|
4155
|
+
try {
|
|
4156
|
+
report = await runPipelines(spans, { minLoopOccurrences: minLoop });
|
|
4157
|
+
} catch (err) {
|
|
4158
|
+
opts.onError?.(err, ref);
|
|
4159
|
+
continue;
|
|
4160
|
+
}
|
|
4161
|
+
sessionCount += 1;
|
|
4162
|
+
if (opts.onReport) {
|
|
4163
|
+
try {
|
|
4164
|
+
await opts.onReport(ref, report);
|
|
4165
|
+
} catch (err) {
|
|
4166
|
+
opts.onError?.(err, ref);
|
|
4167
|
+
}
|
|
4168
|
+
}
|
|
4169
|
+
for (const f of report.stuckLoops.findings) {
|
|
4170
|
+
const key = loopKey(ref.sessionId, f.toolName, f.argHash);
|
|
4171
|
+
if (f.occurrences > (seen.get(key) ?? 0)) {
|
|
4172
|
+
seen.set(key, f.occurrences);
|
|
4173
|
+
if (opts.onLoop) {
|
|
4174
|
+
try {
|
|
4175
|
+
await opts.onLoop({
|
|
4176
|
+
harness: adapter.harness,
|
|
4177
|
+
sessionId: ref.sessionId,
|
|
4178
|
+
cwd: ref.cwd,
|
|
4179
|
+
toolName: f.toolName,
|
|
4180
|
+
argHash: f.argHash,
|
|
4181
|
+
occurrences: f.occurrences,
|
|
4182
|
+
windowMs: f.windowMs
|
|
4183
|
+
});
|
|
4184
|
+
} catch (err) {
|
|
4185
|
+
opts.onError?.(err, ref);
|
|
4186
|
+
}
|
|
4187
|
+
}
|
|
4188
|
+
}
|
|
4189
|
+
}
|
|
4190
|
+
}
|
|
4191
|
+
try {
|
|
4192
|
+
opts.onTick?.({ sessions: sessionCount });
|
|
4193
|
+
} catch (err) {
|
|
4194
|
+
opts.onError?.(err);
|
|
4195
|
+
}
|
|
4196
|
+
if (opts.signal?.aborted) break;
|
|
4197
|
+
await sleep(intervalMs, opts.signal);
|
|
4198
|
+
}
|
|
4199
|
+
}
|
|
4200
|
+
|
|
4201
|
+
// src/session-index.ts
|
|
4202
|
+
import { mkdtemp as mkdtemp5, readdir as readdir11, readFile as readFile14, stat as stat12, writeFile as writeFile5 } from "fs/promises";
|
|
4203
|
+
import { tmpdir as tmpdir5 } from "os";
|
|
4204
|
+
import { dirname as dirname3, join as join17 } from "path";
|
|
3547
4205
|
function indexRow(record) {
|
|
3548
4206
|
return {
|
|
3549
4207
|
session: record.session,
|
|
@@ -3613,13 +4271,13 @@ async function findContextRoot(cwd) {
|
|
|
3613
4271
|
let current = cwd;
|
|
3614
4272
|
const s = await pathStat2(current);
|
|
3615
4273
|
if (!s) return null;
|
|
3616
|
-
if (!s.isDirectory()) current =
|
|
4274
|
+
if (!s.isDirectory()) current = dirname3(current);
|
|
3617
4275
|
for (; ; ) {
|
|
3618
|
-
if (await pathStat2(
|
|
4276
|
+
if (await pathStat2(join17(current, ".git")) || await dirExists(join17(current, ".evolve")) || await fileExists(join17(current, "AGENTS.md")) || await fileExists(join17(current, "CLAUDE.md"))) {
|
|
3619
4277
|
return current;
|
|
3620
4278
|
}
|
|
3621
|
-
const parent =
|
|
3622
|
-
if (parent === current) return s.isDirectory() ? cwd :
|
|
4279
|
+
const parent = dirname3(current);
|
|
4280
|
+
if (parent === current) return s.isDirectory() ? cwd : dirname3(cwd);
|
|
3623
4281
|
current = parent;
|
|
3624
4282
|
}
|
|
3625
4283
|
}
|
|
@@ -3660,12 +4318,12 @@ async function summarizeFile(path, kind) {
|
|
|
3660
4318
|
try {
|
|
3661
4319
|
const text = await readFile14(path, "utf8");
|
|
3662
4320
|
const lines = countLines(text);
|
|
3663
|
-
const
|
|
4321
|
+
const jsonl2 = path.endsWith(".jsonl") ? summarizeJsonl(text) : void 0;
|
|
3664
4322
|
return {
|
|
3665
4323
|
...base,
|
|
3666
4324
|
lines,
|
|
3667
4325
|
...path.endsWith(".md") ? { markdown: summarizeMarkdown(text) } : {},
|
|
3668
|
-
...
|
|
4326
|
+
...jsonl2 ? { jsonlRows: jsonl2.rows, jsonl: jsonl2 } : {}
|
|
3669
4327
|
};
|
|
3670
4328
|
} catch {
|
|
3671
4329
|
return base;
|
|
@@ -3683,7 +4341,7 @@ async function walkMarkdown(root, kind, maxFiles = 100) {
|
|
|
3683
4341
|
}
|
|
3684
4342
|
for (const entry of entries) {
|
|
3685
4343
|
if (out.length >= maxFiles) return;
|
|
3686
|
-
const path =
|
|
4344
|
+
const path = join17(dir, entry.name);
|
|
3687
4345
|
if (entry.isDirectory()) {
|
|
3688
4346
|
await walk(path);
|
|
3689
4347
|
} else if (entry.isFile() && entry.name.endsWith(".md")) {
|
|
@@ -3698,20 +4356,20 @@ async function walkMarkdown(root, kind, maxFiles = 100) {
|
|
|
3698
4356
|
async function collectContextRoot(root) {
|
|
3699
4357
|
const files = [];
|
|
3700
4358
|
for (const name of ["AGENTS.md", "CLAUDE.md", "GEMINI.md"]) {
|
|
3701
|
-
const summary = await summarizeFile(
|
|
4359
|
+
const summary = await summarizeFile(join17(root, name), "instruction-doc");
|
|
3702
4360
|
if (summary) files.push(summary);
|
|
3703
4361
|
}
|
|
3704
|
-
const evolve =
|
|
4362
|
+
const evolve = join17(root, ".evolve");
|
|
3705
4363
|
for (const name of ["skill-runs.jsonl", "governor.jsonl", "experiments.jsonl"]) {
|
|
3706
|
-
const summary = await summarizeFile(
|
|
4364
|
+
const summary = await summarizeFile(join17(evolve, name), "evolve-jsonl");
|
|
3707
4365
|
if (summary) files.push(summary);
|
|
3708
4366
|
}
|
|
3709
4367
|
for (const name of ["scorecard.json"]) {
|
|
3710
|
-
const summary = await summarizeFile(
|
|
4368
|
+
const summary = await summarizeFile(join17(evolve, name), "evolve-json");
|
|
3711
4369
|
if (summary) files.push(summary);
|
|
3712
4370
|
}
|
|
3713
|
-
files.push(...await walkMarkdown(
|
|
3714
|
-
files.push(...await walkMarkdown(
|
|
4371
|
+
files.push(...await walkMarkdown(join17(evolve, "reflections"), "reflection"));
|
|
4372
|
+
files.push(...await walkMarkdown(join17(evolve, "handoffs"), "handoff"));
|
|
3715
4373
|
return { root, files: files.sort((a, b) => a.path.localeCompare(b.path)) };
|
|
3716
4374
|
}
|
|
3717
4375
|
async function collectContextIndex(records) {
|
|
@@ -3757,19 +4415,19 @@ function serializeSessionIndex(index) {
|
|
|
3757
4415
|
`;
|
|
3758
4416
|
}
|
|
3759
4417
|
async function writeSessionIndexFile(index, outPath) {
|
|
3760
|
-
const path = outPath ??
|
|
3761
|
-
await
|
|
4418
|
+
const path = outPath ?? join17(await mkdtemp5(join17(tmpdir5(), "traces-index-")), "session-index.json");
|
|
4419
|
+
await writeFile5(path, serializeSessionIndex(index), "utf8");
|
|
3762
4420
|
return path;
|
|
3763
4421
|
}
|
|
3764
4422
|
|
|
3765
4423
|
// src/upload-state.ts
|
|
3766
4424
|
import { createHash as createHash2 } from "crypto";
|
|
3767
|
-
import { mkdir, readFile as readFile15, writeFile as
|
|
4425
|
+
import { mkdir as mkdir2, readFile as readFile15, writeFile as writeFile6 } from "fs/promises";
|
|
3768
4426
|
import { homedir as homedir11 } from "os";
|
|
3769
|
-
import { dirname as
|
|
4427
|
+
import { dirname as dirname4, join as join18 } from "path";
|
|
3770
4428
|
function stateFile() {
|
|
3771
|
-
const base = process.env.XDG_STATE_HOME ??
|
|
3772
|
-
return
|
|
4429
|
+
const base = process.env.XDG_STATE_HOME ?? join18(homedir11(), ".local", "state");
|
|
4430
|
+
return join18(base, "tangle-traces", "uploaded.json");
|
|
3773
4431
|
}
|
|
3774
4432
|
function uploadKey(harness, sessionId) {
|
|
3775
4433
|
return `${harness}:${sessionId}`;
|
|
@@ -3791,8 +4449,8 @@ async function loadState() {
|
|
|
3791
4449
|
}
|
|
3792
4450
|
async function saveState(state) {
|
|
3793
4451
|
const f = stateFile();
|
|
3794
|
-
await
|
|
3795
|
-
await
|
|
4452
|
+
await mkdir2(dirname4(f), { recursive: true });
|
|
4453
|
+
await writeFile6(f, `${JSON.stringify(state, null, 2)}
|
|
3796
4454
|
`, "utf8");
|
|
3797
4455
|
}
|
|
3798
4456
|
function alreadyUploaded(state, harness, sessionId, hash) {
|
|
@@ -3803,7 +4461,7 @@ function alreadyUploaded(state, harness, sessionId, hash) {
|
|
|
3803
4461
|
import { readFile as readFile16 } from "fs/promises";
|
|
3804
4462
|
import { createRequire } from "module";
|
|
3805
4463
|
import { hostname } from "os";
|
|
3806
|
-
import { basename as basename9, join as
|
|
4464
|
+
import { basename as basename9, join as join19 } from "path";
|
|
3807
4465
|
import { hostedClientFromEnv } from "@tangle-network/agent-eval/hosted";
|
|
3808
4466
|
import { REDACTION_VERSION } from "@tangle-network/agent-eval/traces";
|
|
3809
4467
|
var require2 = createRequire(import.meta.url);
|
|
@@ -3827,7 +4485,7 @@ async function planUpload(opts = {}) {
|
|
|
3827
4485
|
async function gitBranch(cwd) {
|
|
3828
4486
|
if (!cwd) return void 0;
|
|
3829
4487
|
try {
|
|
3830
|
-
const head = await readFile16(
|
|
4488
|
+
const head = await readFile16(join19(cwd, ".git", "HEAD"), "utf8");
|
|
3831
4489
|
const m = head.match(/^ref:\s*refs\/heads\/(.+)$/m);
|
|
3832
4490
|
return m?.[1]?.trim();
|
|
3833
4491
|
} catch {
|
|
@@ -4001,9 +4659,22 @@ export {
|
|
|
4001
4659
|
serializeInspectionReport,
|
|
4002
4660
|
readSessionIndexFile,
|
|
4003
4661
|
writeInspectionReportFile,
|
|
4004
|
-
watchSessions,
|
|
4005
4662
|
REACTIONS,
|
|
4006
4663
|
analyzeReactions,
|
|
4664
|
+
summarizeDeterministicSignals,
|
|
4665
|
+
renderReport,
|
|
4666
|
+
renderPipelines,
|
|
4667
|
+
renderReactions,
|
|
4668
|
+
renderAdoption,
|
|
4669
|
+
buildTraceFindingPacket,
|
|
4670
|
+
loadTracesConfig,
|
|
4671
|
+
mergeTracesConfig,
|
|
4672
|
+
runTraceInvestigation,
|
|
4673
|
+
runTraceStoreInvestigation,
|
|
4674
|
+
writeTraceImprovementArtifacts,
|
|
4675
|
+
runTraceImprovementLoop,
|
|
4676
|
+
saveReport,
|
|
4677
|
+
watchSessions,
|
|
4007
4678
|
buildSessionIndex,
|
|
4008
4679
|
buildSessionIndexFromRows,
|
|
4009
4680
|
collectSessionIndex,
|