glassbox-trace 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -26
- package/dist/glassbox.html +592 -67
- package/package.json +57 -54
- package/src/cli.mjs +217 -31
- package/src/tail.mjs +141 -0
- package/src/trace-core.js +426 -50
- package/src/viewer.html +166 -17
package/src/trace-core.js
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
})(typeof globalThis !== 'undefined' ? globalThis : this, function () {
|
|
10
10
|
'use strict';
|
|
11
11
|
|
|
12
|
-
const VERSION = '0.
|
|
12
|
+
const VERSION = '0.5.0';
|
|
13
13
|
|
|
14
14
|
// ---------------------------------------------------------------------------
|
|
15
15
|
// Rate card (USD per million tokens). Prefix-matched against model ids so dated
|
|
@@ -57,6 +57,13 @@
|
|
|
57
57
|
longTurnTools: 30,
|
|
58
58
|
failedToolRate: 0.5,
|
|
59
59
|
failedToolMinCalls: 3,
|
|
60
|
+
failedToolWarnMinErrors: 2,
|
|
61
|
+
failedToolWarnRate: 0.2,
|
|
62
|
+
cacheMissMinShare: 0.05,
|
|
63
|
+
longGenerationMaxTokPerSec: 15,
|
|
64
|
+
duplicateReadAgents: 3,
|
|
65
|
+
duplicateReadWarnChars: 100000,
|
|
66
|
+
groupRepeatsMin: 3, // slow-tool / oversized-result: this many from one tool become one finding
|
|
60
67
|
};
|
|
61
68
|
|
|
62
69
|
// ---------------------------------------------------------------------------
|
|
@@ -66,18 +73,31 @@
|
|
|
66
73
|
const WRITE_TOOLS = new Set(['Write', 'Edit', 'MultiEdit', 'NotebookEdit']);
|
|
67
74
|
const EXEC_TOOLS = new Set(['Bash', 'BashOutput', 'KillShell']);
|
|
68
75
|
const AGENT_TOOLS = new Set(['Agent', 'Task', 'Workflow']);
|
|
69
|
-
const USER_TOOLS = new Set(['AskUserQuestion']);
|
|
76
|
+
const USER_TOOLS = new Set(['AskUserQuestion', 'SendUserMessage', 'SendUserFile', 'ExitPlanMode', 'EnterPlanMode']);
|
|
77
|
+
// Cowork: Artifact publishes a page (a write), Skill loads instructions (a read); Task* are bookkeeping (other).
|
|
78
|
+
const COWORK_READ = new Set(['Skill', 'ListSkills', 'SearchSkills', 'TaskList', 'TaskGet']);
|
|
79
|
+
const COWORK_WRITE = new Set(['Artifact']);
|
|
80
|
+
// Blocking reads of a background task's output: the duration is the task's run time, and re-issuing the
|
|
81
|
+
// same call after a "still running" timeout is polling, not a retry.
|
|
82
|
+
const WAIT_TOOLS = new Set(['TaskOutput', 'AgentOutputTool']);
|
|
83
|
+
const isWait = (c) => WAIT_TOOLS.has(c.name) && !(c.input && c.input.block === false);
|
|
84
|
+
const waitTarget = (c) => { const i = c.input || {}; return String(i.task_id || i.agentId || i.bash_id || ''); };
|
|
85
|
+
const hasRange = (c) => !!(c.input && (c.input.limit != null || c.input.offset != null || c.input.head_limit != null));
|
|
86
|
+
// MCP leaf names are verb_noun or noun_verb ("memory_read", "device_list_dir", "send_message"): match the verb as a word.
|
|
87
|
+
const MCP_READ = /(^|_)(get|list|search|read|find|fetch|query|describe|show|stage|screenshot|context|status|info|check)(_|$)/i;
|
|
88
|
+
const MCP_WRITE = /(^|_)(write|set|put|create|update|delete|remove|commit|send|reply|forward|upload|str_replace|append|label|move|rename|save)(_|$)/i;
|
|
70
89
|
|
|
71
90
|
function toolCategory(name) {
|
|
72
91
|
if (!name) return 'other';
|
|
73
|
-
if (READ_TOOLS.has(name)) return 'read';
|
|
74
|
-
if (WRITE_TOOLS.has(name)) return 'write';
|
|
92
|
+
if (READ_TOOLS.has(name) || COWORK_READ.has(name)) return 'read';
|
|
93
|
+
if (WRITE_TOOLS.has(name) || COWORK_WRITE.has(name)) return 'write';
|
|
75
94
|
if (EXEC_TOOLS.has(name) || /device_bash$/.test(name)) return 'exec';
|
|
76
95
|
if (AGENT_TOOLS.has(name)) return 'agent';
|
|
77
96
|
if (USER_TOOLS.has(name)) return 'user';
|
|
78
97
|
if (name.startsWith('mcp__')) {
|
|
79
98
|
const leaf = name.split('__').pop() || '';
|
|
80
|
-
if (
|
|
99
|
+
if (MCP_READ.test(leaf)) return 'read';
|
|
100
|
+
if (MCP_WRITE.test(leaf)) return 'write';
|
|
81
101
|
return 'mcp';
|
|
82
102
|
}
|
|
83
103
|
return 'other';
|
|
@@ -143,6 +163,26 @@
|
|
|
143
163
|
return !u.input && !u.cacheRead && !u.cacheWrite && !u.output;
|
|
144
164
|
}
|
|
145
165
|
|
|
166
|
+
// A tool call that never ran because the human or a permission rule stopped it. Claude Code
|
|
167
|
+
// records toolDenialKind (user-rejected, permission-rule, automode-blocked, interrupted); older
|
|
168
|
+
// transcripts only have the result text. A shell's own "Permission denied" is a real failure.
|
|
169
|
+
function denialKind(rec, text) {
|
|
170
|
+
if (rec && rec.toolDenialKind) return String(rec.toolDenialKind);
|
|
171
|
+
const t = String(text || '').trim();
|
|
172
|
+
if (/^\[Request interrupted by user/.test(t)) return 'interrupted';
|
|
173
|
+
if (/^The user doesn't want to proceed with this tool use|^User rejected tool use|^Denied by voice/.test(t)) return 'user-rejected';
|
|
174
|
+
if (/^Permission (for this action was|to use \S+ has been) denied/.test(t)) return 'permission-rule';
|
|
175
|
+
return null;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
// One API response is written as several records (one per content block). Subagent transcripts
|
|
179
|
+
// rewrite output_tokens as the stream progresses, so the first record can be far below the final
|
|
180
|
+
// count; the other fields repeat. Keep the largest value seen for each field.
|
|
181
|
+
function maxUsage(a, b) {
|
|
182
|
+
for (const k of Object.keys(a)) if ((b[k] || 0) > a[k]) a[k] = b[k];
|
|
183
|
+
return a;
|
|
184
|
+
}
|
|
185
|
+
|
|
146
186
|
// ---------------------------------------------------------------------------
|
|
147
187
|
// Line parsing
|
|
148
188
|
// ---------------------------------------------------------------------------
|
|
@@ -233,7 +273,7 @@
|
|
|
233
273
|
const meta = {
|
|
234
274
|
sessionId: null, version: null, cwd: null, gitBranch: null, entrypoint: null, title: null,
|
|
235
275
|
models: [], start: null, end: null, hasTimestamps, files: fileMeta, format: 'claude-code',
|
|
236
|
-
reportedCost: null, reportedUsage: null, recordCounts: {},
|
|
276
|
+
reportedCost: null, reportedUsage: null, reportedTurns: null, reportedDurationMs: null, recordCounts: {},
|
|
237
277
|
};
|
|
238
278
|
|
|
239
279
|
const agents = new Map(); // id -> agent
|
|
@@ -243,6 +283,7 @@
|
|
|
243
283
|
agents.set(id, {
|
|
244
284
|
id, type: id === 'main' ? 'main' : 'subagent', description: null, parentToolUseId: null,
|
|
245
285
|
start: null, end: null, usage: emptyUsage(), requestCount: 0, toolCount: 0, model: null,
|
|
286
|
+
workflowRun: (String(rec._file || '').match(/workflows[\\/](wf_[^\\/]+)[\\/]/) || [])[1] || null,
|
|
246
287
|
});
|
|
247
288
|
}
|
|
248
289
|
const a = agents.get(id);
|
|
@@ -293,13 +334,17 @@
|
|
|
293
334
|
if (!meta.gitBranch && rec.gitBranch) meta.gitBranch = rec.gitBranch;
|
|
294
335
|
if (!meta.entrypoint && rec.entrypoint) meta.entrypoint = rec.entrypoint;
|
|
295
336
|
const t = ts(rec);
|
|
296
|
-
|
|
337
|
+
// Only conversation records bound the session: bookkeeping records (frame-link, file-history…)
|
|
338
|
+
// can be written days later and would stretch the wall clock.
|
|
339
|
+
if (t != null && (type === 'user' || type === 'assistant' || type === 'system')) { if (meta.start == null || t < meta.start) meta.start = t; if (meta.end == null || t > meta.end) meta.end = t; }
|
|
297
340
|
|
|
298
341
|
if (type === 'summary') { if (rec.summary && !meta.title) meta.title = rec.summary; events.push({ kind: 'summary', at: t, detail: rec.summary }); continue; }
|
|
299
342
|
if (type === 'attachment') { attachments.count++; attachments.chars += JSON.stringify(rec.attachment || '').length; continue; }
|
|
300
343
|
if (type === 'result') {
|
|
301
344
|
meta.format = 'stream-json';
|
|
302
345
|
if (typeof rec.total_cost_usd === 'number') meta.reportedCost = rec.total_cost_usd;
|
|
346
|
+
if (typeof rec.num_turns === 'number') meta.reportedTurns = rec.num_turns;
|
|
347
|
+
if (typeof rec.duration_ms === 'number') meta.reportedDurationMs = rec.duration_ms;
|
|
303
348
|
if (rec.usage) meta.reportedUsage = usageFrom(rec.usage);
|
|
304
349
|
if (rec.is_error) events.push({ kind: 'api_error', at: t, detail: textOf(rec.result) || 'result is_error' });
|
|
305
350
|
continue;
|
|
@@ -345,6 +390,7 @@
|
|
|
345
390
|
if (structured && typeof structured === 'object' && !text && !imgs.length) call.resultChars = JSON.stringify(structured).length;
|
|
346
391
|
// stream-json puts is_error only in block; Claude Code also stores toolUseResult as an Error string
|
|
347
392
|
if (!tr.is_error && typeof structured === 'string' && /^Error/i.test(structured)) { call.status = 'error'; call.isError = true; }
|
|
393
|
+
if (call.isError) call.denial = denialKind(rec, text);
|
|
348
394
|
} else {
|
|
349
395
|
toolCalls.push(mkOrphanResult(tr, rec, a.id, text, turns.length ? turns[turns.length - 1].index : null));
|
|
350
396
|
}
|
|
@@ -357,6 +403,7 @@
|
|
|
357
403
|
let kind = 'human';
|
|
358
404
|
if (rec.isCompactSummary) kind = 'compact';
|
|
359
405
|
else if (rec.isMeta) kind = 'meta';
|
|
406
|
+
else if (/^\[Request interrupted by user/.test(text.trim())) kind = 'interrupt';
|
|
360
407
|
else if (/^<(system-reminder|local-command|command-name|bash-input|bash-stdout)/.test(text.trim())) kind = 'meta';
|
|
361
408
|
ensureTurn(rec, kind, text);
|
|
362
409
|
continue;
|
|
@@ -379,16 +426,10 @@
|
|
|
379
426
|
turn.requestIds.push(req.id);
|
|
380
427
|
a.requestCount++;
|
|
381
428
|
if (msg.model) { pushModel(meta, msg.model); if (!a.model) a.model = msg.model; }
|
|
382
|
-
const u = usageFrom(msg.usage);
|
|
383
|
-
if (!isEmptyUsage(u)) { req.usage = u; req.contextTokens = u.input + u.cacheRead + u.cacheWrite; addUsage(turn.usage, u); addUsage(a.usage, u); }
|
|
384
429
|
} else {
|
|
385
430
|
if (t != null) { if (req.start == null || t < req.start) req.start = t; if (req.end == null || t > req.end) req.end = t; }
|
|
386
|
-
// usage: if the first record lacked it and a later one has it, take it once
|
|
387
|
-
if (isEmptyUsage(req.usage)) {
|
|
388
|
-
const u = usageFrom(msg.usage);
|
|
389
|
-
if (!isEmptyUsage(u)) { req.usage = u; req.contextTokens = u.input + u.cacheRead + u.cacheWrite; const turn = turns[req.turnIndex]; addUsage(turn.usage, u); addUsage(a.usage, u); }
|
|
390
|
-
}
|
|
391
431
|
}
|
|
432
|
+
maxUsage(req.usage, usageFrom(msg.usage)); // turn and agent totals are summed once, after the loop
|
|
392
433
|
req.records++;
|
|
393
434
|
if (msg.stop_reason) req.stopReason = msg.stop_reason;
|
|
394
435
|
if (rec.isApiErrorMessage) req.isApiError = true;
|
|
@@ -401,7 +442,7 @@
|
|
|
401
442
|
id: b.id || (key + ':' + blockIndex), agent: a.id, requestId: req.id, name: b.name || 'unknown', input: b.input || {},
|
|
402
443
|
start: t, end: null, durationMs: null, status: 'orphan', isError: false, resultText: '', resultChars: 0,
|
|
403
444
|
resultStructured: null, category: toolCategory(b.name), turnIndex: req.turnIndex, subagentId: null,
|
|
404
|
-
inputChars: JSON.stringify(b.input || {}).length, resultImages: 0, imageBytes: 0,
|
|
445
|
+
inputChars: JSON.stringify(b.input || {}).length, resultImages: 0, imageBytes: 0, denial: null,
|
|
405
446
|
};
|
|
406
447
|
toolCalls.push(call);
|
|
407
448
|
toolCallById.set(call.id, call);
|
|
@@ -423,7 +464,12 @@
|
|
|
423
464
|
}
|
|
424
465
|
|
|
425
466
|
// Finalise
|
|
426
|
-
for (const req of requests)
|
|
467
|
+
for (const req of requests) {
|
|
468
|
+
req.blocks.sort((x, y) => x.index - y.index);
|
|
469
|
+
req.contextTokens = req.usage.input + req.usage.cacheRead + req.usage.cacheWrite;
|
|
470
|
+
addUsage(turns[req.turnIndex].usage, req.usage);
|
|
471
|
+
addUsage(agents.get(req.agent).usage, req.usage);
|
|
472
|
+
}
|
|
427
473
|
for (const turn of turns) if (turn.end == null) turn.end = meta.end;
|
|
428
474
|
// Response timing. Assistant record timestamps are written when a block finishes streaming,
|
|
429
475
|
// so a request's "start" is really its first completed block. We measure from the moment the
|
|
@@ -455,9 +501,14 @@
|
|
|
455
501
|
const parent = toolCalls.find((c) => c.category === 'agent' && c.agent === 'main' && ((c.resultText && c.resultText.includes(ag.id)) || (c.resultStructured && JSON.stringify(c.resultStructured).includes(ag.id))));
|
|
456
502
|
if (parent) ag.parentToolUseId = parent.id;
|
|
457
503
|
}
|
|
504
|
+
// Workflow agents live in subagents/workflows/<runId>/; the Workflow call's result names that runId.
|
|
505
|
+
if (!ag.parentToolUseId && ag.workflowRun) {
|
|
506
|
+
const parent = toolCalls.find((c) => c.name === 'Workflow' && ((c.resultStructured && c.resultStructured.runId === ag.workflowRun) || (c.resultText && c.resultText.includes(ag.workflowRun))));
|
|
507
|
+
if (parent) ag.parentToolUseId = parent.id;
|
|
508
|
+
}
|
|
458
509
|
if (ag.parentToolUseId) {
|
|
459
510
|
const parent = toolCallById.get(ag.parentToolUseId);
|
|
460
|
-
if (parent) { parent.subagentId = ag.id; if (!ag.description && parent.input && parent.input.description) ag.description = parent.input.description; if (!ag.agentType && parent.input && parent.input.subagent_type) ag.agentType = parent.input.subagent_type; }
|
|
511
|
+
if (parent) { if (!parent.subagentId) parent.subagentId = ag.id; (parent.subagentIds || (parent.subagentIds = [])).push(ag.id); if (!ag.description && parent.input && parent.input.description) ag.description = parent.input.description; if (!ag.agentType && parent.input && parent.input.subagent_type) ag.agentType = parent.input.subagent_type; }
|
|
461
512
|
}
|
|
462
513
|
}
|
|
463
514
|
|
|
@@ -465,12 +516,15 @@
|
|
|
465
516
|
const usage = emptyUsage();
|
|
466
517
|
for (const r of requests) addUsage(usage, r.usage);
|
|
467
518
|
const humanIdleMs = turns.reduce((s, tn) => s + (tn.idleBeforeMs || 0), 0);
|
|
468
|
-
|
|
519
|
+
// stream-json runs carry no per-record timestamps and no human prompt record, but the result
|
|
520
|
+
// record reports duration_ms and num_turns; use those rather than showing "—" and 0.
|
|
521
|
+
const wallMs = (meta.start != null && meta.end != null) ? meta.end - meta.start : (meta.reportedDurationMs != null ? meta.reportedDurationMs : null);
|
|
522
|
+
const humanTurns = turns.filter((tn) => tn.promptKind === 'human' && tn.agent === 'main').length;
|
|
469
523
|
const totals = {
|
|
470
524
|
usage, wallMs, humanIdleMs, activeMs: wallMs != null ? Math.max(0, wallMs - humanIdleMs) : null,
|
|
471
525
|
requests: requests.length, toolCalls: toolCalls.length, toolErrors: toolCalls.filter((c) => c.isError).length,
|
|
472
526
|
orphans: toolCalls.filter((c) => c.status === 'orphan').length,
|
|
473
|
-
turns:
|
|
527
|
+
turns: humanTurns || (meta.reportedTurns != null ? meta.reportedTurns : 0),
|
|
474
528
|
toolTimeMs: toolCalls.filter((c) => c.agent === 'main').reduce((s, c) => s + (c.durationMs || 0), 0),
|
|
475
529
|
modelTimeMs: agents.get('main').modelTimeMs || 0,
|
|
476
530
|
cacheHitRatio: (usage.cacheRead + usage.cacheWrite + usage.input) ? usage.cacheRead / (usage.cacheRead + usage.cacheWrite + usage.input) : null,
|
|
@@ -508,9 +562,20 @@
|
|
|
508
562
|
if (!byTurnSig.has(sig)) byTurnSig.set(sig, []);
|
|
509
563
|
byTurnSig.get(sig).push(c);
|
|
510
564
|
}
|
|
565
|
+
const callIndex = new Map(calls.map((c, i) => [c, i]));
|
|
511
566
|
for (const group of byTurnSig.values()) {
|
|
512
567
|
if (group.length >= o.retryLoopMin) {
|
|
513
568
|
const errs = group.filter((c) => c.isError).length;
|
|
569
|
+
if (isWait(group[0]) && errs < 2) continue; // polling a running task; failing polls still count
|
|
570
|
+
if (errs < 2) {
|
|
571
|
+
// Re-running a test or taking another screenshot after something changed is observing, not
|
|
572
|
+
// looping: only flag successful repeats that returned the same thing with nothing in between.
|
|
573
|
+
const inGroup = new Set(group);
|
|
574
|
+
const between = calls.slice(callIndex.get(group[0]) + 1, callIndex.get(group[group.length - 1]));
|
|
575
|
+
const changed = between.some((c) => c.agent === group[0].agent && !inGroup.has(c) && c.category !== 'read');
|
|
576
|
+
const same = group.every((c) => c.resultText === group[0].resultText && c.imageBytes === group[0].imageBytes);
|
|
577
|
+
if (changed || !same) continue;
|
|
578
|
+
}
|
|
514
579
|
push({
|
|
515
580
|
id: 'retry-loop', severity: errs >= 2 ? 'error' : 'warn',
|
|
516
581
|
title: `${group[0].name} called ${group.length}× with identical input`,
|
|
@@ -520,14 +585,16 @@
|
|
|
520
585
|
}
|
|
521
586
|
}
|
|
522
587
|
|
|
523
|
-
// failed-tool: per tool error rate
|
|
588
|
+
// failed-tool: per tool error rate. Calls the human or a permission rule stopped are not the
|
|
589
|
+
// tool's failure (see permission-denied); one stray error in a long session is not a pattern.
|
|
524
590
|
const byTool = new Map();
|
|
525
|
-
for (const c of calls) { if (c.unmatched) continue; if (!byTool.has(c.name)) byTool.set(c.name, []); byTool.get(c.name).push(c); }
|
|
591
|
+
for (const c of calls) { if (c.unmatched || c.denial) continue; if (!byTool.has(c.name)) byTool.set(c.name, []); byTool.get(c.name).push(c); }
|
|
526
592
|
for (const [name, list] of byTool) {
|
|
527
593
|
const errs = list.filter((c) => c.isError);
|
|
528
594
|
if (!errs.length) continue;
|
|
529
595
|
const rate = errs.length / list.length;
|
|
530
596
|
const bad = list.length >= o.failedToolMinCalls && rate >= o.failedToolRate;
|
|
597
|
+
if (!bad && (errs.length < o.failedToolWarnMinErrors || rate < o.failedToolWarnRate)) continue;
|
|
531
598
|
push({
|
|
532
599
|
id: 'failed-tool', severity: bad ? 'error' : 'warn',
|
|
533
600
|
title: `${name} failed ${errs.length} of ${list.length} calls (${Math.round(rate * 100)}%)`,
|
|
@@ -536,6 +603,18 @@
|
|
|
536
603
|
});
|
|
537
604
|
}
|
|
538
605
|
|
|
606
|
+
// permission-denied: calls that never ran because the human rejected them, a permission rule or
|
|
607
|
+
// the auto-mode classifier blocked them, or the human interrupted.
|
|
608
|
+
const denied = calls.filter((c) => c.denial);
|
|
609
|
+
if (denied.length) {
|
|
610
|
+
const kinds = {}; for (const c of denied) kinds[c.denial] = (kinds[c.denial] || 0) + 1;
|
|
611
|
+
push({
|
|
612
|
+
id: 'permission-denied', severity: 'info', title: `${denied.length} tool call${denied.length > 1 ? 's' : ''} denied or interrupted`,
|
|
613
|
+
detail: Object.entries(kinds).map(([k, n]) => `${k} ×${n}`).join(', ') + ' — ' + summariseNames(denied),
|
|
614
|
+
evidence: { toolCallIds: denied.map((c) => c.id) }, metric: denied.length,
|
|
615
|
+
});
|
|
616
|
+
}
|
|
617
|
+
|
|
539
618
|
// orphan-tool
|
|
540
619
|
const orphans = calls.filter((c) => c.status === 'orphan');
|
|
541
620
|
if (orphans.length) push({
|
|
@@ -560,18 +639,52 @@
|
|
|
560
639
|
};
|
|
561
640
|
for (const c of calls) {
|
|
562
641
|
if (c.agent !== ag.id || c.unmatched) continue;
|
|
563
|
-
|
|
642
|
+
// MCP calls that aren't reads are actions (browser navigate/click, device commands) and end a run;
|
|
643
|
+
// agent and user calls don't.
|
|
644
|
+
if (c.category === 'read') run.push(c); else if (c.category === 'write' || c.category === 'exec' || c.category === 'mcp') flush();
|
|
564
645
|
}
|
|
565
646
|
flush();
|
|
566
647
|
}
|
|
567
648
|
|
|
568
|
-
//
|
|
649
|
+
// duplicate-subagent-read: the same file read by several agents, each paying for it in its own context
|
|
650
|
+
const sharedReads = new Set(); // reported here, so oversized-result below skips them
|
|
651
|
+
const readsByFile = new Map();
|
|
569
652
|
for (const c of calls) {
|
|
570
|
-
if (c.
|
|
653
|
+
if (c.name !== 'Read' || c.isError || !c.input || !c.input.file_path) continue;
|
|
654
|
+
const k = String(c.input.file_path).replace(/\\/g, '/').toLowerCase();
|
|
655
|
+
if (!readsByFile.has(k)) readsByFile.set(k, []);
|
|
656
|
+
readsByFile.get(k).push(c);
|
|
657
|
+
}
|
|
658
|
+
for (const list of readsByFile.values()) {
|
|
659
|
+
const agentIds = new Set(list.map((c) => c.agent));
|
|
660
|
+
if (agentIds.size < o.duplicateReadAgents) continue;
|
|
661
|
+
const chars = list.reduce((s, c) => s + c.resultChars, 0);
|
|
662
|
+
for (const c of list) sharedReads.add(c);
|
|
663
|
+
const largest = Math.max(...list.map((c) => c.resultChars));
|
|
664
|
+
push({
|
|
665
|
+
id: 'duplicate-subagent-read', severity: largest >= o.oversizedResultError ? 'error' : chars >= o.duplicateReadWarnChars ? 'warn' : 'info',
|
|
666
|
+
title: `Same file read by ${agentIds.size} agents (${fmtInt(chars)} chars)`, // path only in detail, which --redact blanks
|
|
667
|
+
detail: `${list.length} reads of ${list[0].input.file_path} across ${agentIds.size} agents; each copy is billed in that agent's context on every later request.`,
|
|
668
|
+
evidence: { toolCallIds: list.map((c) => c.id) }, metric: chars,
|
|
669
|
+
});
|
|
670
|
+
}
|
|
671
|
+
|
|
672
|
+
// oversized-result: one line per call, or one per tool when a tool does it repeatedly
|
|
673
|
+
const bigByTool = new Map();
|
|
674
|
+
for (const c of calls) if (c.resultChars >= o.oversizedResultWarn && !sharedReads.has(c)) { if (!bigByTool.has(c.name)) bigByTool.set(c.name, []); bigByTool.get(c.name).push(c); }
|
|
675
|
+
for (const [name, list] of bigByTool) {
|
|
676
|
+
const largest = Math.max(...list.map((c) => c.resultChars)), total = list.reduce((s, c) => s + c.resultChars, 0);
|
|
677
|
+
const ranged = list.every(hasRange);
|
|
678
|
+
const detail = 'Large tool results stay in context for the rest of the session. ' + (ranged ? 'The call already set a range; long lines or a wide window still cost, so narrow it or grep first.' : 'Consider head/limit, grep, or a subagent.');
|
|
679
|
+
if (list.length >= o.groupRepeatsMin) push({
|
|
680
|
+
id: 'oversized-result', severity: largest >= o.oversizedResultError ? 'error' : 'warn',
|
|
681
|
+
title: `${name} returned over ${fmtInt(o.oversizedResultWarn)} chars ${list.length} times (largest ${fmtInt(largest)}, ${fmtInt(total)} in all)`,
|
|
682
|
+
detail, evidence: { toolCallIds: list.map((c) => c.id) }, metric: largest,
|
|
683
|
+
});
|
|
684
|
+
else for (const c of list) push({
|
|
571
685
|
id: 'oversized-result', severity: c.resultChars >= o.oversizedResultError ? 'error' : 'warn',
|
|
572
686
|
title: `${c.name} returned ${fmtInt(c.resultChars)} chars`,
|
|
573
|
-
detail:
|
|
574
|
-
evidence: { toolCallIds: [c.id], turnIndex: c.turnIndex }, metric: c.resultChars,
|
|
687
|
+
detail, evidence: { toolCallIds: [c.id], turnIndex: c.turnIndex }, metric: c.resultChars,
|
|
575
688
|
});
|
|
576
689
|
}
|
|
577
690
|
|
|
@@ -591,21 +704,45 @@
|
|
|
591
704
|
if (r.contextTokens >= o.contextBloatWarn && !crossed) crossed = r;
|
|
592
705
|
if (!peak || r.contextTokens > peak.contextTokens) peak = r;
|
|
593
706
|
}
|
|
594
|
-
if (crossed)
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
707
|
+
if (crossed) {
|
|
708
|
+
// What the large context cost: every request (any agent) sent with more than the threshold.
|
|
709
|
+
let above = 0, all = 0, n = 0;
|
|
710
|
+
for (const r of reqs) { const c = requestCost(r.usage, rateFor(r.model, o.rates)); if (c == null) continue; all += c; if (r.contextTokens >= o.contextBloatWarn) { above += c; n++; } }
|
|
711
|
+
const spend = all > 0 ? ` · ${usd(above)} (${Math.round(above / all * 100)}% of cost) spent above ${fmtInt(o.contextBloatWarn)}` : '';
|
|
712
|
+
push({
|
|
713
|
+
id: 'context-bloat', severity: peak.contextTokens >= o.contextBloatError ? 'error' : 'warn',
|
|
714
|
+
title: `Context reached ${fmtInt(peak.contextTokens)} tokens${spend}`,
|
|
715
|
+
detail: `First crossed ${fmtInt(o.contextBloatWarn)} at request #${reqs.indexOf(crossed) + 1} (turn ${crossed.turnIndex + 1}); ${n} request${n === 1 ? '' : 's'} ran above it. Every one of them re-reads that prefix.`,
|
|
716
|
+
evidence: { requestIds: [crossed.id, peak.id], turnIndex: crossed.turnIndex }, metric: peak.contextTokens, cost: all > 0 ? above : null,
|
|
717
|
+
});
|
|
718
|
+
}
|
|
600
719
|
|
|
601
|
-
// cache-churn
|
|
602
|
-
|
|
720
|
+
// cache-churn / idle-cache-expiry: a request that re-wrote prefix the previous request of the same
|
|
721
|
+
// agent had already cached. A big cache write on its own is usually just a new tool result being
|
|
722
|
+
// cached, which is expected. After a gap longer than the cache lifetime the miss is expiry, not
|
|
723
|
+
// invalidation; right after a compaction it is an expected rebuild.
|
|
724
|
+
const prevByAgent = new Map(); const ttlByAgent = new Map();
|
|
603
725
|
for (const r of reqs) {
|
|
604
|
-
|
|
605
|
-
if (r.usage.
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
726
|
+
const prev = prevByAgent.get(r.agent); prevByAgent.set(r.agent, r);
|
|
727
|
+
if (r.usage.cacheWrite1h > 0) ttlByAgent.set(r.agent, 3600e3); else if (r.usage.cacheWrite5m > 0 && !ttlByAgent.has(r.agent)) ttlByAgent.set(r.agent, 300e3);
|
|
728
|
+
if (!prev) continue;
|
|
729
|
+
const cached = prev.usage.cacheRead + prev.usage.cacheWrite;
|
|
730
|
+
const missed = cached - r.usage.cacheRead;
|
|
731
|
+
if (missed < o.cacheChurnMin || missed < cached * o.cacheMissMinShare) continue;
|
|
732
|
+
const sent = r.responseFrom != null ? r.responseFrom : r.start;
|
|
733
|
+
if (prev.end != null && sent != null && trace.events.some((e) => e.kind === 'compact' && e.at != null && e.at >= prev.end && e.at <= sent)) continue;
|
|
734
|
+
const idle = prev.end != null && sent != null ? sent - prev.end : null;
|
|
735
|
+
const ttl = ttlByAgent.get(r.agent) || 300e3;
|
|
736
|
+
const n = reqs.indexOf(r) + 1;
|
|
737
|
+
if (idle != null && idle > ttl) push({
|
|
738
|
+
id: 'idle-cache-expiry', severity: 'info', title: `${fmtInt(missed)} tokens re-cached after ${fmtDur(idle)} idle`,
|
|
739
|
+
detail: `Request #${n}${r.agent !== 'main' ? ' (subagent)' : ''}: the gap was longer than the ${ttl >= 3600e3 ? '1-hour' : '5-minute'} cache lifetime, so the whole prefix was written again.`,
|
|
740
|
+
evidence: { requestIds: [prev.id, r.id], turnIndex: r.turnIndex }, metric: missed,
|
|
741
|
+
});
|
|
742
|
+
else push({
|
|
743
|
+
id: 'cache-churn', severity: 'info', title: `${fmtInt(missed)} cached tokens re-written at request #${n}`,
|
|
744
|
+
detail: `The previous request had ${fmtInt(cached)} tokens cached; this one read only ${fmtInt(r.usage.cacheRead)} back${idle != null ? ` ${fmtDur(idle)} later` : ''}. Something early in the prompt changed (system prompt, tool list, early messages).`,
|
|
745
|
+
evidence: { requestIds: [prev.id, r.id], turnIndex: r.turnIndex }, metric: missed,
|
|
609
746
|
});
|
|
610
747
|
}
|
|
611
748
|
|
|
@@ -616,14 +753,40 @@
|
|
|
616
753
|
evidence: {}, metric: trace.totals.cacheHitRatio,
|
|
617
754
|
});
|
|
618
755
|
|
|
619
|
-
// slow-tool
|
|
756
|
+
// slow-tool: blocking waits roll up per background task, other slow calls per tool when repeated
|
|
757
|
+
const waits = new Map(), slowByTool = new Map();
|
|
620
758
|
for (const c of calls) {
|
|
621
|
-
if (c.
|
|
759
|
+
if (isWait(c)) { const k = c.agent + '|' + c.name + '|' + waitTarget(c); if (!waits.has(k)) waits.set(k, []); waits.get(k).push(c); continue; }
|
|
760
|
+
// Time spent on a question to the human, or on a permission prompt that ended in a denial, is the human's.
|
|
761
|
+
if (c.category === 'user' || c.denial) continue;
|
|
762
|
+
if (c.durationMs != null && c.durationMs >= o.slowToolInfoMs) { if (!slowByTool.has(c.name)) slowByTool.set(c.name, []); slowByTool.get(c.name).push(c); }
|
|
763
|
+
}
|
|
764
|
+
for (const [name, list] of slowByTool) {
|
|
765
|
+
const longest = Math.max(...list.map((c) => c.durationMs)), total = list.reduce((s, c) => s + c.durationMs, 0);
|
|
766
|
+
const detail = list[0].category === 'agent' ? 'Subagent run time.' : 'Long-running tool call.';
|
|
767
|
+
if (list.length >= o.groupRepeatsMin) push({
|
|
768
|
+
id: 'slow-tool', severity: longest >= o.slowToolWarnMs ? 'warn' : 'info',
|
|
769
|
+
title: `${name} took over ${fmtDur(o.slowToolInfoMs)} ${list.length} times (longest ${fmtDur(longest)}, ${fmtDur(total)} in all)`,
|
|
770
|
+
detail, evidence: { toolCallIds: list.map((c) => c.id) }, metric: total,
|
|
771
|
+
});
|
|
772
|
+
else for (const c of list) push({
|
|
622
773
|
id: 'slow-tool', severity: c.durationMs >= o.slowToolWarnMs ? 'warn' : 'info',
|
|
623
|
-
title: `${c.name} took ${fmtDur(c.durationMs)}`, detail
|
|
774
|
+
title: `${c.name} took ${fmtDur(c.durationMs)}`, detail,
|
|
624
775
|
evidence: { toolCallIds: [c.id], turnIndex: c.turnIndex }, metric: c.durationMs,
|
|
625
776
|
});
|
|
626
777
|
}
|
|
778
|
+
// One finding per waited-on task, always info: the time belongs to the background job.
|
|
779
|
+
for (const group of waits.values()) {
|
|
780
|
+
const ms = group.reduce((s, c) => s + (c.durationMs || 0), 0);
|
|
781
|
+
if (ms < o.slowToolInfoMs) continue;
|
|
782
|
+
const target = waitTarget(group[0]);
|
|
783
|
+
push({
|
|
784
|
+
id: 'slow-tool', severity: 'info',
|
|
785
|
+
title: `${group[0].name} waited ${fmtDur(ms)} on ${target ? 'task ' + target : 'a background task'}${group.length > 1 ? ` (${group.length} polls)` : ''}`,
|
|
786
|
+
detail: 'Blocking wait on background work: the time is the task\'s run time, not a slow tool.',
|
|
787
|
+
evidence: { toolCallIds: group.map((c) => c.id), turnIndex: group[0].turnIndex }, metric: ms,
|
|
788
|
+
});
|
|
789
|
+
}
|
|
627
790
|
|
|
628
791
|
// slow-model: a long response that produced little — API latency, rate limiting or a stall,
|
|
629
792
|
// as opposed to a long response that was simply writing a lot (reported separately as info).
|
|
@@ -636,9 +799,10 @@
|
|
|
636
799
|
detail: `Request #${n} (${r.tokensPerSec != null ? r.tokensPerSec.toFixed(1) : '?'} tok/s): API latency, rate limiting or a stalled stream.`,
|
|
637
800
|
evidence: { requestIds: [r.id], turnIndex: r.turnIndex }, metric: r.responseMs,
|
|
638
801
|
});
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
802
|
+
// A long response that wrote a lot at a normal rate is just a big output. Only a slow rate is worth a line.
|
|
803
|
+
else if (r.tokensPerSec != null && r.tokensPerSec < o.longGenerationMaxTokPerSec) push({
|
|
804
|
+
id: 'long-generation', severity: 'info', title: `${fmtDur(r.responseMs)} writing ${fmtInt(r.usage.output)} tokens (${r.tokensPerSec.toFixed(1)} tok/s)`,
|
|
805
|
+
detail: `Request #${n} (${r.blocksSummary}) streamed well below the usual rate: a partial stall or a slow API period.`,
|
|
642
806
|
evidence: { requestIds: [r.id], turnIndex: r.turnIndex }, metric: r.responseMs,
|
|
643
807
|
});
|
|
644
808
|
}
|
|
@@ -670,10 +834,18 @@
|
|
|
670
834
|
detail: 'Most of the spend happened in delegated work.', evidence: {}, metric: subTokens / allTokens,
|
|
671
835
|
});
|
|
672
836
|
|
|
673
|
-
// long-turn
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
837
|
+
// long-turn: main conversation only (a subagent's whole run is one turn by design), numbered like
|
|
838
|
+
// the viewer's T1, T2… (human prompts).
|
|
839
|
+
// Meta turns (injected reminders, interrupts) between two prompts belong to the stretch before them.
|
|
840
|
+
const stretches = [];
|
|
841
|
+
for (const tn of trace.turns) {
|
|
842
|
+
if (tn.agent !== 'main') continue;
|
|
843
|
+
if (tn.promptKind === 'human' || !stretches.length) stretches.push({ first: tn, no: stretches.length + 1, ids: [] });
|
|
844
|
+
stretches[stretches.length - 1].ids.push(...tn.toolCallIds);
|
|
845
|
+
}
|
|
846
|
+
for (const st of stretches) if (st.ids.length >= o.longTurnTools) push({
|
|
847
|
+
id: 'long-turn', severity: 'info', title: `Turn ${st.no} made ${st.ids.length} tool calls`,
|
|
848
|
+
detail: 'A very long autonomous stretch. Worth a skim for wasted work.', evidence: { turnIndex: st.first.index, toolCallIds: st.ids }, metric: st.ids.length,
|
|
677
849
|
});
|
|
678
850
|
|
|
679
851
|
const order = { error: 0, warn: 1, info: 2 };
|
|
@@ -736,6 +908,210 @@
|
|
|
736
908
|
};
|
|
737
909
|
}
|
|
738
910
|
|
|
911
|
+
// ---------------------------------------------------------------------------
|
|
912
|
+
// Advice — one sentence per rule on what to do differently. Mechanical, like the rules.
|
|
913
|
+
// ---------------------------------------------------------------------------
|
|
914
|
+
const ADVICE = {
|
|
915
|
+
'retry-loop': 'When a call fails, change something before calling again: read the error, fix the input or the environment, or ask the user — never repeat an identical call.',
|
|
916
|
+
'failed-tool': 'Check the first error message of the failing tool before retrying; if the tool itself is broken, switch tools or tell the user instead of pushing on.',
|
|
917
|
+
'orphan-tool': 'A call with no result means the session ended mid-call; if you resume, re-run it or confirm its side effects before building on it.',
|
|
918
|
+
'exploration-run': 'Batch exploration: use one Grep or Glob with a wider pattern, or delegate a broad search to a subagent, then commit to an edit or command sooner.',
|
|
919
|
+
'oversized-result': 'Ask tools for less: head/limit/offset on reads, a tighter grep pattern, a file listing instead of a dump, or a subagent that returns a summary.',
|
|
920
|
+
'image-heavy': 'Take fewer screenshots and crop or downscale them; read text with a page-text tool where one exists, since every image is re-billed on each later request.',
|
|
921
|
+
'context-bloat': 'Keep the prompt small: summarise long results into notes, avoid re-reading files already in context, and hand large investigations to subagents.',
|
|
922
|
+
'cache-churn': 'Avoid changing anything at the top of the prompt mid-session (system prompt, tool list, early messages), because it invalidates the cached prefix and re-bills it.',
|
|
923
|
+
'low-cache-hit': 'Long stable prefixes cache well; frequent tool-list or system-prompt changes and many tiny sessions do not.',
|
|
924
|
+
'slow-tool': 'For calls that take minutes, run them in the background, tighten the command, or set a timeout, instead of blocking the whole turn.',
|
|
925
|
+
'slow-model': 'A slow, small response is usually API latency or a stall; nothing to fix in the session itself, but note it when reporting session time.',
|
|
926
|
+
'long-generation': 'Large single outputs are normal for file writes; split very large files into parts if a stall (low tok/s) shows up.',
|
|
927
|
+
'max-tokens': 'The output was truncated: write large files in sections and keep single responses under the output limit.',
|
|
928
|
+
'api-error': 'An API error interrupted the session; check what was lost and re-run the affected step rather than assuming it completed.',
|
|
929
|
+
'hook-error': 'A hook failed; read the hook error, fix the hook command or its permissions, then re-run the step it guarded.',
|
|
930
|
+
'compaction': 'Context was compacted; state that matters later (decisions, file paths, test status) should be written to a file or note before it is summarised away.',
|
|
931
|
+
'thinking-heavy': 'Most output tokens were thinking; fine for hard problems, but for routine tool loops a lower effort setting is cheaper and just as good.',
|
|
932
|
+
'subagent-share': 'Most spend was in subagents; give them narrower briefs and ask for short structured returns so the parent pays for less.',
|
|
933
|
+
'long-turn': 'A very long autonomous turn; check in with a short progress summary at natural checkpoints so wasted work is caught earlier.',
|
|
934
|
+
'permission-denied': 'Before a call the user may not want (installs, network, deleting, publishing), say what it will do and ask; for calls they always allow, suggest a permissions allow rule so the prompt goes away.',
|
|
935
|
+
'duplicate-subagent-read': 'When several subagents need the same file, put the relevant excerpt in their briefs or have one agent summarise it once, instead of every agent reading it into its own context.',
|
|
936
|
+
'idle-cache-expiry': 'The cache expired while the session sat idle; resume sooner, use the 1-hour cache lifetime where available, or start a fresh session with a summary instead of re-sending the whole history.',
|
|
937
|
+
};
|
|
938
|
+
|
|
939
|
+
// Advice for a concrete finding: the rule's line, except where the evidence says otherwise.
|
|
940
|
+
function adviceFor(f, trace) {
|
|
941
|
+
if (f.id === 'slow-tool') {
|
|
942
|
+
const ids = (f.evidence && f.evidence.toolCallIds) || [];
|
|
943
|
+
const c = trace.toolCalls.find((x) => x.id === ids[0]);
|
|
944
|
+
if (c && c.category === 'user') return null; // waiting on the human is not the agent's time
|
|
945
|
+
if (c && c.category === 'agent') return 'Subagent run time: give it a narrower brief and ask for a short structured return so the parent is not blocked for as long.';
|
|
946
|
+
if (c && isWait(c)) return 'Waiting on a long background task: keep doing independent work while it runs, or tell the user it is still going, instead of blocking the turn on it.';
|
|
947
|
+
}
|
|
948
|
+
if (f.id === 'oversized-result') {
|
|
949
|
+
const c = trace.toolCalls.find((x) => x.id === ((f.evidence && f.evidence.toolCallIds) || [])[0]);
|
|
950
|
+
if (c && hasRange(c)) return 'The read already had a range and still returned a lot: grep for the part you need first, or read a narrower window.';
|
|
951
|
+
}
|
|
952
|
+
return ADVICE[f.id] || null;
|
|
953
|
+
}
|
|
954
|
+
|
|
955
|
+
function inputSummary(c) {
|
|
956
|
+
const i = (c && c.input) || {};
|
|
957
|
+
if (i.command) return String(i.command);
|
|
958
|
+
if (i.file_path) return String(i.file_path) + (i.pattern ? ' ' + i.pattern : '');
|
|
959
|
+
if (i.pattern) return String(i.pattern);
|
|
960
|
+
if (i.query) return String(i.query);
|
|
961
|
+
if (i.description) return String(i.description);
|
|
962
|
+
if (i.url) return String(i.url);
|
|
963
|
+
if (i.prompt) return String(i.prompt);
|
|
964
|
+
if (i.path) return String(i.path);
|
|
965
|
+
const s = JSON.stringify(i);
|
|
966
|
+
return s === '{}' ? '' : s;
|
|
967
|
+
}
|
|
968
|
+
|
|
969
|
+
// ---------------------------------------------------------------------------
|
|
970
|
+
// Markdown report — written to be handed back to the agent as well as read by a human.
|
|
971
|
+
// opts: { redact, advice=true, instruction=true, maxEvidence=6, title }
|
|
972
|
+
// ---------------------------------------------------------------------------
|
|
973
|
+
function usd(v) { return v == null ? '—' : v < 0.01 ? '$' + v.toFixed(4) : v < 1 ? '$' + v.toFixed(3) : '$' + v.toFixed(2); }
|
|
974
|
+
function pctStr(x) { return x == null ? '—' : Math.round(x * 100) + '%'; }
|
|
975
|
+
function oneLine(s, n) { s = String(s == null ? '' : s).replace(/\s+/g, ' ').trim(); return s.length > n ? s.slice(0, n - 1) + '…' : s; }
|
|
976
|
+
function blank(s) { return '«' + String(s == null ? '' : s).length + ' chars»'; }
|
|
977
|
+
|
|
978
|
+
function toolTable(trace) {
|
|
979
|
+
const rows = new Map();
|
|
980
|
+
for (const c of trace.toolCalls) { if (c.unmatched) continue; if (!rows.has(c.name)) rows.set(c.name, { name: c.name, calls: 0, errors: 0, time: 0 }); const r = rows.get(c.name); r.calls++; if (c.isError) r.errors++; r.time += c.durationMs || 0; }
|
|
981
|
+
return Array.from(rows.values()).sort((a, b) => b.time - a.time || b.calls - a.calls);
|
|
982
|
+
}
|
|
983
|
+
|
|
984
|
+
// Rules whose detail quotes tool output or model text (everything else is names and numbers).
|
|
985
|
+
const CONTENT_DETAIL = new Set(['failed-tool', 'api-error', 'hook-error', 'duplicate-subagent-read']);
|
|
986
|
+
function redactDetail(f) { return CONTENT_DETAIL.has(f.id) ? blank(f.detail) : f.detail; }
|
|
987
|
+
|
|
988
|
+
function reportMarkdown(trace, findings, cost, opts) {
|
|
989
|
+
opts = opts || {};
|
|
990
|
+
const redact = !!opts.redact, maxEv = opts.maxEvidence || 6;
|
|
991
|
+
const T = trace.totals, m = trace.meta, u = T.usage;
|
|
992
|
+
const R = (s) => redact ? blank(s) : s;
|
|
993
|
+
const total = cost ? (cost.reported != null ? cost.reported : cost.total) : null;
|
|
994
|
+
const turnNo = (idx) => idx != null ? idx + 1 : null; // same numbering as the finding details and the viewer drawer
|
|
995
|
+
const advised = new Set();
|
|
996
|
+
const out = [];
|
|
997
|
+
out.push(`# Glassbox report — ${oneLine(R(m.title || opts.title || 'session'), 90)}`, '');
|
|
998
|
+
out.push(`- session: ${m.sessionId || '—'} · model: ${m.models.join(', ') || '—'}${m.start ? ' · started: ' + new Date(m.start).toISOString() : ''}`);
|
|
999
|
+
out.push(`- wall ${fmtDur(T.wallMs)} (active ${fmtDur(T.activeMs)}) · ${T.turns} turn${T.turns === 1 ? '' : 's'} · ${T.requests} requests · ${T.toolCalls} tool calls (${T.toolErrors} failed${T.orphans ? ', ' + T.orphans + ' unanswered' : ''})`);
|
|
1000
|
+
out.push(`- tokens: ${fmtInt(u.input + u.cacheRead + u.cacheWrite)} context served (${pctStr(T.cacheHitRatio)} cached) · peak prompt ${fmtInt(Math.max(0, ...trace.requests.map((r) => r.contextTokens)))} · ${fmtInt(u.output)} output (${fmtInt(u.thinking)} thinking) · est. cost ${usd(total)}${cost && cost.source === 'reported' ? ' (reported)' : ''}`, '');
|
|
1001
|
+
if (!findings.length) out.push('## Findings', '', 'Nothing flagged — a clean session, or a very short one.', '');
|
|
1002
|
+
else {
|
|
1003
|
+
out.push(`## Findings (${findings.length})`, '');
|
|
1004
|
+
findings.forEach((f, i) => {
|
|
1005
|
+
out.push(`### ${i + 1}. ${f.severity.toUpperCase()} \`${f.id}\` — ${f.title}`, '', redact ? redactDetail(f) : f.detail, '');
|
|
1006
|
+
const ev = f.evidence || {};
|
|
1007
|
+
const lines = [];
|
|
1008
|
+
for (const id of ev.toolCallIds || []) { const c = trace.toolCalls.find((x) => x.id === id); if (!c) continue; const tn = turnNo(c.turnIndex); lines.push(`- ${c.name}${c.isError ? ' (error)' : ''}${c.durationMs != null ? ' · ' + fmtDur(c.durationMs) : ''}${tn != null ? ' · turn ' + tn : ''}${c.agent !== 'main' ? ' · subagent' : ''}: \`${oneLine(R(inputSummary(c)), 100).replace(/`/g, "'") || '(no input)'}\``); }
|
|
1009
|
+
for (const id of ev.requestIds || []) { const r = trace.requests.find((x) => x.id === id); if (!r) continue; const n = trace.requests.indexOf(r) + 1; const tn = turnNo(r.turnIndex); lines.push(`- request #${n}${tn != null ? ' · turn ' + tn : ''}: context ${fmtInt(r.contextTokens)} · output ${fmtInt(r.usage.output)}${r.responseMs != null ? ' · ' + fmtDur(r.responseMs) : ''}`); }
|
|
1010
|
+
if (lines.length) { out.push('Evidence:', ''); out.push(...lines.slice(0, maxEv)); if (lines.length > maxEv) out.push(`- … ${lines.length - maxEv} more`); out.push(''); }
|
|
1011
|
+
const adv = opts.advice !== false && !advised.has(f.id) ? adviceFor(f, trace) : null;
|
|
1012
|
+
if (adv) { advised.add(f.id); out.push(`Next time: ${adv}`, ''); }
|
|
1013
|
+
});
|
|
1014
|
+
}
|
|
1015
|
+
const tools = toolTable(trace);
|
|
1016
|
+
if (tools.length) { out.push('## Tools', '', '| tool | calls | errors | total time |', '|---|---|---|---|'); for (const r of tools) out.push(`| ${r.name} | ${r.calls} | ${r.errors} | ${fmtDur(r.time)} |`); out.push(''); }
|
|
1017
|
+
if (opts.instruction !== false) out.push('## What to do with this', '', 'This is a mechanical review of the session transcript, generated by Glassbox (no AI involved). If you are the agent that ran this session: acknowledge the findings that apply, say in one or two sentences what you will do differently next session, and do not redo any work. If you are a human: the evidence lines are permalinks into the Glassbox viewer for this session.', '');
|
|
1018
|
+
out.push('_Generated by Glassbox ' + VERSION + '._', '');
|
|
1019
|
+
return out.join('\n');
|
|
1020
|
+
}
|
|
1021
|
+
|
|
1022
|
+
// ---------------------------------------------------------------------------
|
|
1023
|
+
// Compare two analysed sessions ({trace, findings, cost}).
|
|
1024
|
+
// ---------------------------------------------------------------------------
|
|
1025
|
+
const METRICS = [
|
|
1026
|
+
{ key: 'wallMs', label: 'Wall time', fmt: 'dur', dir: 'low', get: (s) => s.trace.totals.wallMs },
|
|
1027
|
+
{ key: 'activeMs', label: 'Active time', fmt: 'dur', dir: 'low', get: (s) => s.trace.totals.activeMs },
|
|
1028
|
+
{ key: 'turns', label: 'Turns', fmt: 'int', dir: null, get: (s) => s.trace.totals.turns },
|
|
1029
|
+
{ key: 'requests', label: 'API requests', fmt: 'int', dir: 'low', get: (s) => s.trace.totals.requests },
|
|
1030
|
+
{ key: 'toolCalls', label: 'Tool calls', fmt: 'int', dir: 'low', get: (s) => s.trace.totals.toolCalls },
|
|
1031
|
+
{ key: 'toolErrors', label: 'Tool errors', fmt: 'int', dir: 'low', get: (s) => s.trace.totals.toolErrors },
|
|
1032
|
+
{ key: 'toolTimeMs', label: 'Time in tools', fmt: 'dur', dir: 'low', get: (s) => s.trace.meta.hasTimestamps ? s.trace.totals.toolTimeMs : null },
|
|
1033
|
+
{ key: 'modelTimeMs', label: 'Time generating', fmt: 'dur', dir: 'low', get: (s) => s.trace.meta.hasTimestamps ? s.trace.totals.modelTimeMs : null },
|
|
1034
|
+
{ key: 'contextServed', label: 'Context served', fmt: 'tok', dir: 'low', get: (s) => { const u = s.trace.totals.usage; const v = u.input + u.cacheRead + u.cacheWrite; return v || (s.trace.requests.length ? 0 : null); } },
|
|
1035
|
+
{ key: 'peakContext', label: 'Peak prompt', fmt: 'tok', dir: 'low', get: (s) => s.trace.requests.length ? Math.max(0, ...s.trace.requests.map((r) => r.contextTokens)) : null },
|
|
1036
|
+
{ key: 'cacheHitRatio', label: 'Cache hit', fmt: 'pct', dir: 'high', get: (s) => s.trace.totals.cacheHitRatio },
|
|
1037
|
+
{ key: 'output', label: 'Output tokens', fmt: 'tok', dir: 'low', get: (s) => s.trace.requests.length ? s.trace.totals.usage.output : null },
|
|
1038
|
+
{ key: 'thinkingShare', label: 'Thinking share', fmt: 'pct', dir: null, get: (s) => { const u = s.trace.totals.usage; return u.output ? u.thinking / u.output : null; } },
|
|
1039
|
+
{ key: 'cost', label: 'Est. cost', fmt: 'usd', dir: 'low', get: (s) => s.cost ? (s.cost.reported != null ? s.cost.reported : s.cost.total) : null },
|
|
1040
|
+
{ key: 'findings', label: 'Findings', fmt: 'int', dir: 'low', get: (s) => s.findings.length },
|
|
1041
|
+
{ key: 'errorFindings', label: 'Error-level findings', fmt: 'int', dir: 'low', get: (s) => s.findings.filter((f) => f.severity === 'error').length },
|
|
1042
|
+
];
|
|
1043
|
+
function fmtMetric(v, fmt) {
|
|
1044
|
+
if (v == null) return '—';
|
|
1045
|
+
if (fmt === 'dur') return fmtDur(v);
|
|
1046
|
+
if (fmt === 'pct') return pctStr(v);
|
|
1047
|
+
if (fmt === 'usd') return usd(v);
|
|
1048
|
+
return fmtInt(v);
|
|
1049
|
+
}
|
|
1050
|
+
// "×2.4" when B is bigger, "÷2.4" when smaller; never "×0.00".
|
|
1051
|
+
function fmtRatio(r) {
|
|
1052
|
+
if (r == null || !Number.isFinite(r)) return '—';
|
|
1053
|
+
if (r === 1) return '×1.00';
|
|
1054
|
+
const v = r >= 1 ? r : 1 / r; const t = v >= 100 ? Math.round(v).toString() : v >= 10 ? v.toFixed(1) : v.toFixed(2);
|
|
1055
|
+
return (r >= 1 ? '×' : '÷') + t;
|
|
1056
|
+
}
|
|
1057
|
+
// Change of B relative to A: a signed percentage where that reads well, ×N for big increases,
|
|
1058
|
+
// the absolute delta when A is 0 (no ratio) or B is 0 (no "÷∞").
|
|
1059
|
+
function fmtChange(m) {
|
|
1060
|
+
if (m.delta == null) return '—';
|
|
1061
|
+
const r = m.ratio;
|
|
1062
|
+
if (r == null || !m.a || !m.b) return (m.delta > 0 ? '+' : m.delta < 0 ? '−' : '') + fmtMetric(Math.abs(m.delta), m.fmt);
|
|
1063
|
+
if (r === 1) return '0%';
|
|
1064
|
+
if (r < 1) { const p = (1 - r) * 100; return '−' + (p < 10 || p > 99 ? p.toFixed(1) : Math.round(p)) + '%'; }
|
|
1065
|
+
if (r < 10) { const p = (r - 1) * 100; return '+' + (p < 10 ? p.toFixed(1) : Math.round(p)) + '%'; }
|
|
1066
|
+
return fmtRatio(r);
|
|
1067
|
+
}
|
|
1068
|
+
function compare(A, B) {
|
|
1069
|
+
const metrics = METRICS.map((d) => {
|
|
1070
|
+
const a = d.get(A), b = d.get(B);
|
|
1071
|
+
const ok = a != null && b != null && Number.isFinite(a) && Number.isFinite(b);
|
|
1072
|
+
const delta = ok ? b - a : null;
|
|
1073
|
+
const ratio = ok && a ? b / a : (ok && !a && !b ? 1 : null);
|
|
1074
|
+
let better = null;
|
|
1075
|
+
if (ok && d.dir && a !== b) better = (d.dir === 'low') === (a < b) ? 'a' : 'b';
|
|
1076
|
+
return { key: d.key, label: d.label, fmt: d.fmt, dir: d.dir, a, b, delta, ratio, better, aText: fmtMetric(a, d.fmt), bText: fmtMetric(b, d.fmt) };
|
|
1077
|
+
});
|
|
1078
|
+
const ta = toolTable(A.trace), tb = toolTable(B.trace);
|
|
1079
|
+
const names = new Set([...ta.map((t) => t.name), ...tb.map((t) => t.name)]);
|
|
1080
|
+
const zero = { calls: 0, errors: 0, time: 0 };
|
|
1081
|
+
const tools = Array.from(names).map((name) => { const a = ta.find((t) => t.name === name) || zero, b = tb.find((t) => t.name === name) || zero; return { name, a: { calls: a.calls, errors: a.errors, time: a.time }, b: { calls: b.calls, errors: b.errors, time: b.time }, deltaCalls: b.calls - a.calls, deltaTime: b.time - a.time }; });
|
|
1082
|
+
tools.sort((x, y) => Math.abs(y.deltaCalls) - Math.abs(x.deltaCalls) || (y.a.calls + y.b.calls) - (x.a.calls + x.b.calls) || x.name.localeCompare(y.name));
|
|
1083
|
+
const byId = (fs) => { const m = new Map(); for (const f of fs) { if (!m.has(f.id)) m.set(f.id, []); m.get(f.id).push(f); } return m; };
|
|
1084
|
+
const fa = byId(A.findings), fb = byId(B.findings);
|
|
1085
|
+
const slim = (f) => ({ id: f.id, severity: f.severity, title: f.title });
|
|
1086
|
+
const onlyA = [], onlyB = [], both = [];
|
|
1087
|
+
for (const [id, list] of fa) { if (fb.has(id)) both.push({ id, a: list.map(slim), b: fb.get(id).map(slim) }); else onlyA.push(...list.map(slim)); }
|
|
1088
|
+
for (const [id, list] of fb) if (!fa.has(id)) onlyB.push(...list.map(slim));
|
|
1089
|
+
const pick = (key) => { const m = metrics.find((x) => x.key === key); return m ? m.better : null; };
|
|
1090
|
+
const cleaner = (() => { const ea = A.findings.filter((f) => f.severity === 'error').length, eb = B.findings.filter((f) => f.severity === 'error').length; if (ea !== eb) return ea < eb ? 'a' : 'b'; const wa = A.findings.length, wb = B.findings.length; return wa === wb ? null : (wa < wb ? 'a' : 'b'); })();
|
|
1091
|
+
const verdict = { cheaper: pick('cost'), faster: pick('wallMs') || pick('activeMs'), cleaner };
|
|
1092
|
+
return { metrics, tools, findings: { onlyA, onlyB, both }, verdict, a: { sessionId: A.trace.meta.sessionId, title: A.trace.meta.title, models: A.trace.meta.models }, b: { sessionId: B.trace.meta.sessionId, title: B.trace.meta.title, models: B.trace.meta.models } };
|
|
1093
|
+
}
|
|
1094
|
+
function compareMarkdown(c, opts) {
|
|
1095
|
+
opts = opts || {};
|
|
1096
|
+
const la = opts.labelA || 'A', lb = opts.labelB || 'B';
|
|
1097
|
+
const out = [`# Glassbox compare — ${la} vs ${lb}`, ''];
|
|
1098
|
+
out.push(`- ${la}: ${c.a.sessionId || '—'}${c.a.title ? ' · ' + oneLine(c.a.title, 70) : ''}${c.a.models.length ? ' · ' + c.a.models.join(', ') : ''}`);
|
|
1099
|
+
out.push(`- ${lb}: ${c.b.sessionId || '—'}${c.b.title ? ' · ' + oneLine(c.b.title, 70) : ''}${c.b.models.length ? ' · ' + c.b.models.join(', ') : ''}`, '');
|
|
1100
|
+
const v = c.verdict; const who = (x) => x === 'a' ? la : x === 'b' ? lb : 'tie';
|
|
1101
|
+
out.push(`**Verdict** — cheaper: ${who(v.cheaper)} · faster: ${who(v.faster)} · cleaner: ${who(v.cleaner)}`, '');
|
|
1102
|
+
out.push(`| metric | ${la} | ${lb} | change |`, '|---|---|---|---|');
|
|
1103
|
+
for (const m of c.metrics) { const ch = fmtChange(m); out.push(`| ${m.label} | ${m.aText} | ${m.bText} | ${ch}${m.better ? ' (' + who(m.better) + ' better)' : ''} |`); }
|
|
1104
|
+
out.push('');
|
|
1105
|
+
if (c.tools.length) { out.push('## Tools', '', `| tool | ${la} calls (errors) | ${lb} calls (errors) | Δ calls |`, '|---|---|---|---|'); for (const t of c.tools) out.push(`| ${t.name} | ${t.a.calls} (${t.a.errors}) | ${t.b.calls} (${t.b.errors}) | ${t.deltaCalls > 0 ? '+' : ''}${t.deltaCalls} |`); out.push(''); }
|
|
1106
|
+
out.push('## Findings', '');
|
|
1107
|
+
if (!c.findings.onlyA.length && !c.findings.onlyB.length && !c.findings.both.length) out.push('Neither session was flagged.', '');
|
|
1108
|
+
if (c.findings.onlyA.length) { out.push(`Only in ${la}:`, ''); for (const f of c.findings.onlyA) out.push(`- **${f.severity.toUpperCase()}** \`${f.id}\` — ${f.title}`); out.push(''); }
|
|
1109
|
+
if (c.findings.onlyB.length) { out.push(`Only in ${lb}:`, ''); for (const f of c.findings.onlyB) out.push(`- **${f.severity.toUpperCase()}** \`${f.id}\` — ${f.title}`); out.push(''); }
|
|
1110
|
+
if (c.findings.both.length) { out.push('In both:', ''); for (const f of c.findings.both) out.push(`- \`${f.id}\` — ${la}: ${f.a.length}, ${lb}: ${f.b.length}`); out.push(''); }
|
|
1111
|
+
out.push('_Generated by Glassbox ' + VERSION + '._', '');
|
|
1112
|
+
return out.join('\n');
|
|
1113
|
+
}
|
|
1114
|
+
|
|
739
1115
|
// ---------------------------------------------------------------------------
|
|
740
1116
|
// Redaction — keep structure, timestamps, usage, tool names; blank every string.
|
|
741
1117
|
// ---------------------------------------------------------------------------
|
|
@@ -764,5 +1140,5 @@
|
|
|
764
1140
|
}
|
|
765
1141
|
function toJsonl(records) { return records.map((r) => JSON.stringify(r)).join('\n') + '\n'; }
|
|
766
1142
|
|
|
767
|
-
return { VERSION, RATES, DEFAULTS, parseLines, parseTrace, diagnose, estimateCost, rateFor, redact, toJsonl, toolCategory, textOf, fmtDur, fmtInt, stableStringify };
|
|
1143
|
+
return { VERSION, RATES, DEFAULTS, ADVICE, METRICS, parseLines, parseTrace, diagnose, estimateCost, rateFor, redact, toJsonl, toolCategory, textOf, fmtDur, fmtInt, fmtMetric, fmtRatio, fmtChange, stableStringify, inputSummary, adviceFor, redactDetail, reportMarkdown, compare, compareMarkdown, toolTable };
|
|
768
1144
|
});
|