glassbox-trace 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -1
- package/dist/glassbox.html +7 -3
- package/package.json +1 -1
- package/src/adhere.mjs +41 -12
- package/src/claims.mjs +368 -0
- package/src/cli.mjs +65 -17
- package/src/fence.mjs +101 -24
- package/src/guard.mjs +96 -0
- package/src/trace-core.js +7 -3
package/src/claims.mjs
ADDED
|
@@ -0,0 +1,368 @@
|
|
|
1
|
+
// glassbox claims — said vs did.
|
|
2
|
+
//
|
|
3
|
+
// Every session ends with the agent's account of itself: "all 76 tests pass", "committed as 753be02", "live on
|
|
4
|
+
// npm", "nothing changed". This module pulls each such sentence out of the assistant's user-facing text, looks
|
|
5
|
+
// for the tool result that should be standing behind it, and gives it a verdict. It is pure: a parsed trace in,
|
|
6
|
+
// a ledger out. Nothing here knows whether the code is right — only whether the sentence has a receipt.
|
|
7
|
+
// Design: DESIGN.md §18, docs/CLAIMS.md.
|
|
8
|
+
|
|
9
|
+
// ---------------------------------------------------------------------------------------------
|
|
10
|
+
// Sentences
|
|
11
|
+
// ---------------------------------------------------------------------------------------------
|
|
12
|
+
const FENCE_RE = /```[\s\S]*?```/g;
|
|
13
|
+
const SPLIT_RE = /(?<=[.!?]|[.!?]["”’)])\s+(?=[A-Za-z"“`*(\-\d])|\n+/;
|
|
14
|
+
const ABBR_RE = /\b(?:e\.g|i\.e|vs|etc|approx|no|Mr|Dr|St)\.(?=\s)/g;
|
|
15
|
+
const strip = (s) => s.replace(/^[\s>*\-•·]+/, '').replace(/\*\*|__/g, '').replace(/^\d+[.)]\s+/, '').trim();
|
|
16
|
+
|
|
17
|
+
// User-facing texts: assistant text blocks (any agent) and, in Cowork, the message and file tools.
|
|
18
|
+
export function userFacingTexts(trace) {
|
|
19
|
+
const out = [];
|
|
20
|
+
const callsByReq = new Map();
|
|
21
|
+
for (const c of trace.toolCalls) { if (!callsByReq.has(c.requestId)) callsByReq.set(c.requestId, []); callsByReq.get(c.requestId).push(c); }
|
|
22
|
+
trace.requests.forEach((r, i) => {
|
|
23
|
+
for (const b of r.blocks) if (b.type === 'text' && b.text && b.text.trim()) out.push({ request: r, index: i, agent: r.agent, via: 'text', text: b.text });
|
|
24
|
+
for (const c of callsByReq.get(r.id) || []) {
|
|
25
|
+
const inp = c.input || {};
|
|
26
|
+
if (c.name === 'SendUserMessage' && typeof inp.message === 'string') out.push({ request: r, index: i, agent: r.agent, via: 'SendUserMessage', text: inp.message, call: c });
|
|
27
|
+
if (c.name === 'SendUserFile' && typeof inp.caption === 'string') out.push({ request: r, index: i, agent: r.agent, via: 'SendUserFile', text: inp.caption, call: c });
|
|
28
|
+
}
|
|
29
|
+
});
|
|
30
|
+
return out;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function sentences(text) {
|
|
34
|
+
return String(text).replace(FENCE_RE, ' ').replace(ABBR_RE, (m) => m.replace(/\./g, '\u2024')).split(SPLIT_RE).map((x) => strip(x.replace(/\u2024/g, '.'))).filter((s) => s.length >= 8);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
// ---------------------------------------------------------------------------------------------
|
|
38
|
+
// Classification
|
|
39
|
+
// ---------------------------------------------------------------------------------------------
|
|
40
|
+
// Honesty markers: the agent said it did not check. Recorded as `declared`, never a finding.
|
|
41
|
+
const DECLARED_RE = /\bnot\s+(?:verified|re-?run|checked|tested|run|confirmed|reproduced|pushed|published|deployed|uploaded|committed|merged|released)\b|\bunverified\b|\bI\s+(?:couldn't|could\s+not|can't|cannot|wasn't able to|was\s+unable\s+to)\b|\bcouldn't\s+(?:verify|check|run|find|reach|push|confirm)\b|\bhaven't\s+(?:run|verified|checked|tested)\b|\bdidn't\s+run\b|\bnot\s+done\b|\bI\s+got\s+(?:this|that|it)\s+wrong\b|\bfalse\s+positive\b|\bmy\s+(?:error|mistake)\b|\bI\s+was\s+wrong\b|\bcorrection\b|\bskipped\s+on\s+purpose\b|\bleft\s+(?:this|that|it)\s+to\s+you\b/i;
|
|
42
|
+
// Not a claim: plans, questions, conditionals, instructions to the human, descriptions of what a tool does.
|
|
43
|
+
const NOT_CLAIM_RE = /\b(?:I'll|I\s+will|will\b|let\s+me|going\s+to|about\s+to|next\s+(?:step|up|time)|should|would|could|may\b|might|if\b|whether|unless|once\s+you|when\s+you|to\s+do\b|todo|plan(?:ned|s)?\b|need(?:s|ed)?\s+to|want(?:s|ed)?\s+to|try(?:ing)?\s+to|please|you\s+can|you\s+could|you\s+should|run\s+this|paste|waiting\s+on|still\s+to\s+(?:go|do|upload|run)|before\s+I|now\s+I|here's\s+how|to\s+bring|to\s+pin|to\s+continue|instead|I\s+can\b|wait\s+for|needs?\b|has\s+to\b|have\s+to\b|I'd\b|we'd\b|you'd\b|we'll\b|we\s+will\b|let's\b|tell\s+me\b|ping\s+me\b|let\s+me\s+know|once\s+(?:the|that|this|it|you|we|they|your|I)\b|(?:it|that|this)\s+seems\b|seems\s+(?:like|to)\b|likely\b|probably\b|presumably\b|perhaps\b|maybe\b|I\s+(?:think|suspect|guess|assume|believe)\b|something\s+like\b|has\s+since\b|have\s+since\b|you\s+must\b|must\s+have\b|run\s+`[^`]*`\s+first|submit\s+(?:through|via|the)|(?:doesn't|won't|will\s+not|does\s+not|wouldn't)\s+(?:touch|push|change|include|affect|need|go))\b|\?\s*$|^\s*(?:want|shall|do|does|did|can|is|are)\s+(?:me|I|you|it|we|this|that)\b/i;
|
|
44
|
+
// Sentences about what the tool does, not what happened.
|
|
45
|
+
const DESCRIBES_RE = /\b(?:it|this|that|the\s+(?:guard|hook|rule|command|viewer|action|script|check))\s+(?:blocks?|fires?|reads?|writes?|prints?|checks?|runs?|serves?|counts?|redacts?|refuses?)\b|\bgets\s+\w+(?:ed|en)\b/i;
|
|
46
|
+
|
|
47
|
+
const KINDS = [
|
|
48
|
+
['test', /\b(?:tests?|suites?|specs?|e2e|audit|ci|checks?|lint(?:er)?|build|typecheck|type-check|validation|coverage)\b[^.]{0,80}?\b(?:pass(?:es|ed|ing)?|green|succeed(?:s|ed)?|ok|clean|no\s+failures|validates?|valid|passed)\b|\b(?:all|\d[\d,]*)\s*(?:of\s*\d[\d,]*\s*)?(?:tests?\s+)?(?:still\s+)?(?:pass(?:es|ed|ing)?|green)\b|\b\d[\d,]*\s*(?:\/\s*\d[\d,]*\s*)?(?:tests?|specs?)\s*(?:still\s+)?(?:pass|passing|passed|green|ok)\b|\b(?:plugin|manifest|schema|json-ld)\s+validates?\b|\b(?:ci|workflow|run\s+#?\d+|publish\s+#?\d+)\s+(?:is\s+|was\s+)?(?:green|succeeded|passed|completed)\b|\bexit(?:ed| code)?\s*0\b/i],
|
|
49
|
+
['ship', /^(?:both|all|everything|it|they)\s+(?:saved|written|uploaded|committed|pushed|deployed|published|live)\b|\b(?:all|both)\s+\d+\s+(?:saved|uploaded|live)\b|\ball\s+trimmed\s+and\s+live\b|\blive\s+and\s+(?:rendering|serving)\b|(?<!\b(?:the|a|an|its|my|your|this|our|that|one|each|these|those|two|three|four|five|six|last|previously)\s)\b(?:pushed(?!\s+back)|committed|commit(?:ted)?\s+(?:it\s+)?as\b|published|deployed|uploaded|merged|released|republished|shipped|posted)\b(?!\s+(?:blog|post|page|version|package|site|copy|report|article|files?|templates?|branch|folder|build|commits?)\b)|\b(?:is\s+(?:now\s+)?(?:live|released|on\s+npm|on\s+github|up(?!\s*(?:\d|\$|by\b|to\b|from\b|and\s+running\b)))|went\s+live|now\s+live|are\s+live|on\s+npm\b|in\s+production|is\s+on\s+main\b|(?:is|are|now|already|sits?)\s+(?:up\s+)?on\s+the\s+server)\b/i],
|
|
50
|
+
['state', /\b(?:nothing\s+(?:changed|was\s+changed|else\s+changed|left\s+uncommitted|was\s+left)|didn't\s+(?:disturb|touch|change)|(?:wasn't|weren't|not)\s+touched|untouched|unchanged|as\s+you\s+left\s+it|byte-identical|exactly\s+as\s+(?:you|I|we)\s+left|same\s+\d+\s+(?:staged|files)|no\s+(?:changes|diff|difference))\b/i],
|
|
51
|
+
['verify', /\b(?:all|both)\s+\d+\s+(?:done|ok|checked)\b|\b(?:verified|confirmed|double-checked|checked(?:\s+(?:that|it|the|all|every|against))?|validated|I\s+(?:ran|re-ran|tested|opened|loaded|looked|compared|hashed|checked)|renders\b|rendered\s+(?:fine|correctly|ok|as\s+expected)|loads\s+(?:fine|correctly|ok)|loaded\s+(?:fine|correctly|ok)|works?\s+(?:as\s+expected|fine|correctly|now|end-to-end)|working\s+(?:as\s+expected|fine|correctly|now)|round-?trips?|resolves?(?:d)?|matches?(?:\s+exactly)?|byte-exact|identical|(?:is|was|are|were|looks?|looked)\s+(?:fine|ok|good|correct|right)\b|has\s+(?:a\s+)?working\b|went\s+through\b|rendered(?=\s*[,.;]|\s+(?:fine|correctly|ok|properly|as\s+expected))|(?:took|measured|clocked|was|is|it's)\s+\d[\d.,]*\s*(?:ms|s|secs?|seconds?|minutes?|MB|KB|GB|bytes)\b|all\s+(?:green|\d+\s+done|\d+\s+ok)|\d+\s*\/\s*\d+,?\s*(?:zero|no)\s+errors?|tested\s+(?:it|this|that|locally|end\s+to\s+end)|no\s+errors?|zero\s+errors?|clean\b)\b/i],
|
|
52
|
+
['fix', /(?<!\b(?:the|a|its|my|your|this|our|that|one|each)\s)\bfixed\b(?!\s+(?:cost|price|fee|quote|rate|width|height|size|number|amount|set|point|term|period|version|one|copy|app))|\bfix\s+is\s+in\b|\b(?:resolved|patched|corrected|repaired)\b|\bno\s+longer\s+(?!needed|necessary|required|relevant|used|in\s+use|available|exists?|supported|part|the\s+case)\w+|\bnow\s+(?:works|handles|passes|renders|resolves|loads|decodes|parses|starts)\b|\bbug\s+(?:is\s+)?(?:fixed|gone)\b/i],
|
|
53
|
+
['write', /\b(?:wrote|written|created|saved|added|updated|delivered|copied|generated|rebuilt|regenerated|exported|installed|removed|deleted|renamed|moved|replaced|bumped|normalised|normalized|backed\s+up)\b[^.]{0,120}?(?:`[^`]+`|\b[\w./\\-]+\.(?:mjs|cjs|js|ts|json|md|html|css|py|yml|yaml|pdf|png|jpg|jpeg|zip|cmd|txt|jsonl|svg|mp4|csv|xlsx|docx)\b|\bfile|\bfolder|\brepo\b|\bbranch\b|\btag\b)/i],
|
|
54
|
+
];
|
|
55
|
+
const NUM_RE = /(?<![\w.\/#-])(?:\$\d[\d,]*(?:\.\d+)?|\d{1,3}(?:,\d{3})+(?:\.\d+)?|\d+(?:\.\d+)?(?:%|×|x)?|\b[0-9a-f]{7,40}\b)(?![\w-])/g;
|
|
56
|
+
const SHA_RE = /\b[0-9a-f]{7,40}\b/g;
|
|
57
|
+
const VERSION_RE = /\b\d+\.\d+\.\d+\b/g;
|
|
58
|
+
const URL_RE = /\b(?:https?:\/\/)?[\w.-]+\.(?:com|au|io|dev|org|net|app)(?:\/[\w./#?=&-]*)?/gi;
|
|
59
|
+
const PATH_RE = /`([^`\s]{2,120})`|((?:[A-Za-z]:)?[\w./\\-]+\.(?:mjs|cjs|js|ts|json|md|html|css|py|yml|yaml|pdf|png|jpg|jpeg|zip|cmd|txt|jsonl|svg|mp4|csv|xlsx|docx))\b/g;
|
|
60
|
+
|
|
61
|
+
// A negated claim word ("haven't written it to any file", "No bundle shipped", "never pushed") is not a claim of that
|
|
62
|
+
// kind; 0.10 has no matcher for negatives. "nothing changed" and "no longer" start their own match, so they pass.
|
|
63
|
+
const NEGATED_RE = /\b(?:not|never|haven't|hasn't|hadn't|didn't|wasn't|weren't|won't|isn't|aren't|don't|doesn't|nothing|no|without)\s+(?:\w+\s+){0,2}$/i;
|
|
64
|
+
function kindOf(s) {
|
|
65
|
+
for (const [kind, re] of KINDS) { const m = s.match(re); if (m && m.index != null && !NEGATED_RE.test(s.slice(Math.max(0, m.index - 40), m.index))) return kind; }
|
|
66
|
+
return null;
|
|
67
|
+
}
|
|
68
|
+
export function classify(s) {
|
|
69
|
+
if (/:\s*$/.test(s) || s.length < 8) return null; // a heading or a label, not a sentence
|
|
70
|
+
if (NOT_CLAIM_RE.test(s) || DESCRIBES_RE.test(s)) return null;
|
|
71
|
+
// Words inside short quotes are content the agent is showing, not asserting ("fixed quote, your deadline").
|
|
72
|
+
const kind = kindOf(s.replace(/["“][^"”\n]{1,80}["”]/g, '"…"'));
|
|
73
|
+
// "Not verified: …" / "I couldn't push" are declarations; a marker tacked on late ("…the audit I couldn't run here")
|
|
74
|
+
// does not turn a ship or test claim into one.
|
|
75
|
+
const dm = s.match(DECLARED_RE);
|
|
76
|
+
if (dm && (dm.index < s.length / 2 || !kind)) return 'declared';
|
|
77
|
+
// "Hook: a Stop hook installed as …" — an inventory line naming a thing, not an action. Only a write-shaped line
|
|
78
|
+
// with a noun label qualifies; "Fixed: the crash…" and "Verified: the README…" keep their kind.
|
|
79
|
+
const inv = kind === 'write' && s.match(/^([A-Z][\w' -]{0,24}):\s+(?:a|an|the)\b/);
|
|
80
|
+
if (inv && !/ed$|\b(?:done|live|ok)$/i.test(inv[1])) return null;
|
|
81
|
+
return kind;
|
|
82
|
+
}
|
|
83
|
+
// Small bare numbers are only worth checking when the sentence is a count ("58 of 58 pass", "27 pages").
|
|
84
|
+
const numbersIn = (s) => Array.from(new Set((s.match(NUM_RE) || []).map((n) => n.replace(/,/g, '')).filter((n) => !/^\d{1,2}$/.test(n) || /\b(?:pass|tests?|of|files?|pages?|commits?|sessions?|runs?)\b/i.test(s))));
|
|
85
|
+
|
|
86
|
+
// ---------------------------------------------------------------------------------------------
|
|
87
|
+
// Evidence
|
|
88
|
+
// ---------------------------------------------------------------------------------------------
|
|
89
|
+
const SHELL = (c) => c.name === 'Bash' || c.name === 'PowerShell' || /device_bash$/.test(c.name || '');
|
|
90
|
+
const cmdOf = (c) => { const i = c.input || {}; return typeof i.command === 'string' ? i.command : typeof i.script === 'string' ? i.script : ''; };
|
|
91
|
+
const inputText = (c) => { try { return JSON.stringify(c.input || {}); } catch (e) { return ''; } };
|
|
92
|
+
const TEST_CMD = /\b(?:npm|pnpm|yarn|bun)\s+(?:run\s+)?(?:test|check|e2e|audit|lint|build|typecheck)\b|\bnode\s+--test\b|\bnode\s+(?:[\w./-]*\/)?test\/[\w.-]+\.m?js\b|\bpytest\b|\bpython3?\s+-m\s+(?:pytest|unittest)\b|\bcargo\s+test\b|\bgo\s+test\b|\bvitest\b|\bjest\b|\bmocha\b|\bplaywright\s+test\b|\btsc\b|\beslint\b|\bruff\b|\bmake\s+(?:test|check)\b|\bclaude\s+plugin\s+validate\b|\bdotnet\s+test\b|\bphpunit\b|\brspec\b|\bphp\s+-l\b|\bnode\s+(?:--check|-c)\b|\bpython3?\s+-m\s+py_compile\b|\bflake8\b|\bpylint\b|\bmypy\b|\bshellcheck\b|\bprettier\s+--check\b|\bbiome\s+(?:check|lint)\b|\bpython3?\s+(?:[\w./-]*\/)?(?:test_[\w-]+|[\w-]+_test|tests?)\.py\b|\bnode\s+(?:[\w./-]*\/)?[\w-]*test[\w-]*\.[cm]?js\b/i;
|
|
93
|
+
const CI_RESULT = /completed[:\s]+success|conclusion["']?\s*[:=]\s*["']?success|completed\s+successfully|\bci\s+#?\d+[^.\n]{0,40}\b(?:green|success)|"status":\s*"completed"[^}]{0,80}"conclusion":\s*"success"|publish\s+run[^\n]{0,40}success/i;
|
|
94
|
+
const CI_FAIL = /conclusion["']?\s*[:=]\s*["']?(?:failure|cancelled)|completed[:\s]+failure|\bfailed\b[^\n]{0,20}\bci\b/i;
|
|
95
|
+
const FAIL_MARK = /\bnot ok\b|# fail [1-9]|\b[1-9]\d* fail(?:ed|ing|ures?)?\b|\bFAIL(?:ED)?\b|AssertionError|Error:|\bTypeError|\bReferenceError|\bSyntaxError|Tests:\s+\d+\s+failed|✗|✖|\bexit(?:ed|\s*code)?\s*[=:]?\s*[1-9]\d*\b|Cannot find module|MODULE_NOT_FOUND/;
|
|
96
|
+
// A test-like run's identity: the runner and file it names, so a failing e2e run does not contradict "58 of 58 pass"
|
|
97
|
+
// said about the unit run beside it.
|
|
98
|
+
const testKey = (c) => { const m = cmdOf(c).match(TEST_CMD); return m ? m[0].toLowerCase().replace(/\s+/g, ' ') : (c.name + ':' + (c.resultText || '').slice(0, 40)); };
|
|
99
|
+
function runFailed(c) {
|
|
100
|
+
const r = c.resultText || ''; const k = testCounts(r);
|
|
101
|
+
return !!(c.isError || (k ? k.fail > 0 : FAIL_MARK.test(r)) || (CI_FAIL.test(r) && !CI_RESULT.test(r)));
|
|
102
|
+
}
|
|
103
|
+
const SHELL_EDIT_RE = /\bsed\s+-i\b|python3?\s+-\s*<<|\bcat\s*>>?\s|\btee\b|\bmv\b|\bcp\b|>\s*[\w./-]+\s*(?:&&|;|$)/;
|
|
104
|
+
const SHIP_CMD = /\bgit\s+(?:push|commit)\b|\bnpm\s+publish\b|\bgh\s+(?:pr|release)\b/;
|
|
105
|
+
// The sentence itself says the push (publish, upload, deploy) did not happen.
|
|
106
|
+
const SHIP_DECLINED_RE = /\b(?:push|publish|upload|deploy|release)(?:es|ed|ing)?\s+(?:was\s+|were\s+|got\s+|is\s+|has\s+been\s+)?(?:refused|rejected|failed|blocked|denied|not\s+done)\b|\bnot\s+pushed\b|\bcouldn't\s+(?:push|publish|upload|deploy)\b|\bwithout\s+pushing\b|\bpush\s+(?:is\s+)?(?:pending|waiting|still\s+to\s+do)\b/i;
|
|
107
|
+
const WRITE_TOOLS = new Set(['Write', 'Edit', 'MultiEdit', 'NotebookEdit']);
|
|
108
|
+
// The agent says it did the writing ("I wrote", "Saved to", "Rebuilt dist/"), as opposed to reporting someone else's write.
|
|
109
|
+
const MY_WRITE_RE = /^(?:I|I've|I'd|I\s+have|Also|And|Then|Now|Done|Saved|Wrote|Written|Created|Updated|Added|Delivered|Copied|Rebuilt|Regenerated|Generated|Exported|Installed|Removed|Deleted|Renamed|Moved|Replaced|Bumped|Backed|Both|All|Everything)\b|\b(?:I|I've|we|we've)\s+(?:also\s+|just\s+|now\s+)?(?:wrote|saved|created|added|updated|copied|delivered|generated|rebuilt|regenerated|exported|installed|removed|deleted|renamed|moved|replaced|bumped|backed|normali[sz]ed)\b/i;
|
|
110
|
+
const SHIP_TOOLS = new Set(['mcp__remote-devices__device_commit_files', 'Artifact', 'SendUserFile', 'mcp__claude-in-chrome__file_upload']);
|
|
111
|
+
// An edit that can undo a test result: code or tests inside the project's code folders. A report written to a
|
|
112
|
+
// scratch folder, or a site page, does not make an earlier test run stale.
|
|
113
|
+
const SOURCE_FILE = /(?:^|[\\/])(?:src|lib|test|tests|spec|specs|app|pkg|cmd|internal|hooks|bin|scripts)[\\/][^\\/]*\.(?:mjs|cjs|js|jsx|ts|tsx|py|rb|go|rs|java|kt|cs|php|c|cc|cpp|h|hpp|swift|sh|ps1|yml|yaml|json|toml)$|\.(?:test|spec)\.[cm]?[jt]sx?$|_test\.(?:py|go|rs)$/i;
|
|
114
|
+
|
|
115
|
+
// A result that reads like a test run whatever the command was: a summary line, PASS/FAIL rows, a lint's verdict.
|
|
116
|
+
const looksLikeRun = (r) => { r = String(r || ''); return !!testCounts(r) || /audit passed|browser test passed|Validation passed|No syntax errors detected|^\s*(?:PASS|FAIL)\b.*\n\s*(?:PASS|FAIL)\b/m.test(r); };
|
|
117
|
+
function testCounts(text) {
|
|
118
|
+
const t = String(text || '');
|
|
119
|
+
let m;
|
|
120
|
+
if ((m = t.match(/# pass (\d+)[\s\S]*?# fail (\d+)/))) return { pass: +m[1], fail: +m[2], total: +m[1] + +m[2] };
|
|
121
|
+
if ((m = t.match(/# fail (\d+)[\s\S]*?# pass (\d+)/))) return { pass: +m[2], fail: +m[1], total: +m[1] + +m[2] };
|
|
122
|
+
if ((m = t.match(/(\d+) passed(?:,\s*(\d+) failed)?/))) return { pass: +m[1], fail: +(m[2] || 0), total: +m[1] + +(m[2] || 0) };
|
|
123
|
+
if ((m = t.match(/Tests:\s+(?:(\d+) failed,\s*)?(\d+) passed,\s*(\d+) total/))) return { pass: +m[2], fail: +(m[1] || 0), total: +m[3] };
|
|
124
|
+
if ((m = t.match(/(\d+) passing(?:\s*\n\s*(\d+) failing)?/))) return { pass: +m[1], fail: +(m[2] || 0), total: +m[1] + +(m[2] || 0) };
|
|
125
|
+
return null;
|
|
126
|
+
}
|
|
127
|
+
const claimedCount = (s) => { const m = s.match(/\b(?:all\s+)?(\d[\d,]*)\s*(?:of\s*\d[\d,]*\s*)?(?:tests?|specs?)?\s*(?:still\s+)?(?:pass|green)|\b(\d[\d,]*)\s*\/\s*(\d[\d,]*)/i); return m ? +(m[1] || m[3] || m[2]).replace(/,/g, '') : null; };
|
|
128
|
+
|
|
129
|
+
// The window: this agent's calls in the same turn before the sentence (the last `cap` of them), plus the rest
|
|
130
|
+
// of the turn after it (a claim sometimes precedes its own check). A claim that opens a turn refers to the
|
|
131
|
+
// previous turn's work, so it gets that turn's tail.
|
|
132
|
+
function windowFor(trace, item, byReq, cap = 80) {
|
|
133
|
+
const reqs = trace.requests;
|
|
134
|
+
const before = [], after = [];
|
|
135
|
+
let opensTurn = true;
|
|
136
|
+
for (let i = 0; i < item.index; i++) { const r = reqs[i]; if (r.agent === item.agent && r.turnIndex === item.request.turnIndex) { opensTurn = false; before.push(...(byReq.get(r.id) || [])); } }
|
|
137
|
+
if (opensTurn) { const prev = []; for (let i = item.index - 1; i >= 0 && prev.length < 20; i--) { const r = reqs[i]; if (r.agent !== item.agent || r.turnIndex === item.request.turnIndex) continue; prev.unshift(...(byReq.get(r.id) || [])); } before.unshift(...prev.slice(-20)); }
|
|
138
|
+
before.push(...(byReq.get(item.request.id) || []));
|
|
139
|
+
for (let i = item.index + 1; i < reqs.length && after.length < 12; i++) { const r = reqs[i]; if (r.agent !== item.agent || r.turnIndex !== item.request.turnIndex) break; after.push(...(byReq.get(r.id) || [])); }
|
|
140
|
+
return { before: before.slice(-cap), after };
|
|
141
|
+
}
|
|
142
|
+
// Calls that can never be a receipt: the agent talking to the human (the sentence itself often travels in one),
|
|
143
|
+
// bookkeeping, skill and tool lookups, memory reads, and subagent results (a subagent saying "pushed" is hearsay;
|
|
144
|
+
// its own transcript is judged on its own). Their results carry words like "delivered" that the matchers look for.
|
|
145
|
+
const NOISE = new Set(['SendUserMessage', 'AskUserQuestion', 'ToolSearch', 'Skill', 'ListSkills', 'SearchSkills', 'TaskList', 'TaskGet', 'TaskCreate', 'TaskUpdate', 'TodoWrite', 'ReadNotifications', 'EnterPlanMode', 'ExitPlanMode', 'ListAgents', 'SendMessage', 'ScheduleWakeup']);
|
|
146
|
+
const usable = (c) => !NOISE.has(c.name) && c.category !== 'agent' && !/^mcp__memory__/.test(c.name || '');
|
|
147
|
+
function lastWriteAt(calls) { let t = -Infinity; for (const c of calls) { const p = (c.input && (c.input.file_path || c.input.notebook_path)) || ''; if (WRITE_TOOLS.has(c.name) && SOURCE_FILE.test(String(p)) && c.start != null && c.start > t) t = c.start; } return t; }
|
|
148
|
+
function contains(text, needle) { return needle && String(text || '').toLowerCase().includes(String(needle).toLowerCase()); }
|
|
149
|
+
|
|
150
|
+
// Judge one claim. Returns { verdict, evidence: [toolCallIds], note }.
|
|
151
|
+
function judgeOne(trace, claim, win) {
|
|
152
|
+
const all = win.before.concat(win.after).filter(usable);
|
|
153
|
+
const s = claim.text;
|
|
154
|
+
const ev = (calls, note, verdict) => ({ verdict, evidence: calls.map((c) => c.id), note });
|
|
155
|
+
if (claim.kind === 'test') {
|
|
156
|
+
const runs = all.filter((c) => (SHELL(c) && (TEST_CMD.test(cmdOf(c)) || looksLikeRun(c.resultText))) || ((SHELL(c) || c.name === 'WebFetch' || /javascript|browser_batch|get_page_text|read_page/.test(c.name) || c.name === 'Read') && CI_RESULT.test(c.resultText || '')));
|
|
157
|
+
if (!runs.length) {
|
|
158
|
+
// A count quoted from a file or note the agent read is sourced, not measured — listed, not flagged.
|
|
159
|
+
const want = claimedCount(s);
|
|
160
|
+
const src = want != null ? all.filter((c) => c.category === 'read' && contains(c.resultText, String(want))) : [];
|
|
161
|
+
if (src.length) return ev(src.slice(-1), `the count comes from something the agent read, not from a run`, 'sourced');
|
|
162
|
+
// "All green." / "All checks pass." after a round of curls is a verify claim in test clothing.
|
|
163
|
+
if (!/\b(?:tests?|suites?|specs?|e2e|audit|ci|lint(?:er)?|build|typecheck)/i.test(s)) {
|
|
164
|
+
const insp = all.filter((c) => c.category !== 'write' && c.category !== 'user' && c.category !== 'agent' && !c.isError);
|
|
165
|
+
if (insp.length) return ev(insp.slice(-2), 'no test command; backed by an inspection in the window', 'verified');
|
|
166
|
+
}
|
|
167
|
+
return ev([], 'no test, build, validate or CI result in the window', 'unverified');
|
|
168
|
+
}
|
|
169
|
+
// Which run is the sentence about? One that names the e2e, audit, lint, build or CI is judged on those runs;
|
|
170
|
+
// one that names a count is judged on the latest run whose counts agree with it, unless the same command failed
|
|
171
|
+
// later. Otherwise the latest run in the window.
|
|
172
|
+
const want = claimedCount(s);
|
|
173
|
+
const vocab = (s.match(/\b(e2e|audit|lint|build|typecheck|type-check|validat|ci)\b/i) || [])[1];
|
|
174
|
+
let pool = runs;
|
|
175
|
+
if (vocab) { const re = new RegExp('\\b' + vocab, 'i'); const v = runs.filter((c) => re.test(cmdOf(c) + ' ' + (c.resultText || '').slice(0, 200)) || (/^ci$/i.test(vocab) && CI_RESULT.test(c.resultText || ''))); if (v.length) pool = v; }
|
|
176
|
+
let last = pool[pool.length - 1];
|
|
177
|
+
if (want != null) {
|
|
178
|
+
const match = pool.filter((c) => { const k = testCounts(c.resultText); return k && (k.pass === want || k.total === want); });
|
|
179
|
+
if (match.length) { const m = match[match.length - 1]; if (!pool.some((c) => c.start != null && m.start != null && c.start > m.start && testKey(c) === testKey(m) && runFailed(c))) last = m; }
|
|
180
|
+
}
|
|
181
|
+
const ci = runs.filter((c) => CI_RESULT.test(c.resultText || ''));
|
|
182
|
+
if (runFailed(last)) return ev([last], 'the latest run in the window did not pass', 'contradicted');
|
|
183
|
+
const counts = testCounts(last.resultText);
|
|
184
|
+
if (want != null && counts && counts.pass !== want && counts.total !== want) {
|
|
185
|
+
if (ci.length && ci[ci.length - 1] !== last) return ev([last, ci[ci.length - 1]], `the run says ${counts.pass} passed, the sentence says ${want}; CI reports success`, 'verified');
|
|
186
|
+
return ev([last], `the sentence says ${want}, the last run says ${counts.pass} of ${counts.total}`, 'partial');
|
|
187
|
+
}
|
|
188
|
+
const lw = lastWriteAt(win.before.filter((c) => c.start != null && last.start != null && c.start > last.start));
|
|
189
|
+
if (lw > -Infinity && !ci.some((c) => c.start != null && c.start > lw)) return ev([last], 'the run predates a later edit to source or tests, and nothing re-ran', 'stale');
|
|
190
|
+
return ev([last], counts ? `${counts.pass} passed, ${counts.fail} failed` : (CI_RESULT.test(last.resultText || '') ? 'CI success' : 'run ok'), 'verified');
|
|
191
|
+
}
|
|
192
|
+
if (claim.kind === 'ship') {
|
|
193
|
+
const shas = (s.match(SHA_RE) || []).filter((h) => /\d/.test(h) && /[a-f]/.test(h));
|
|
194
|
+
const versions = s.match(VERSION_RE) || [];
|
|
195
|
+
const hits = [];
|
|
196
|
+
for (const c of all) {
|
|
197
|
+
const r = c.resultText || '';
|
|
198
|
+
const cmd = cmdOf(c);
|
|
199
|
+
const isShip = (SHELL(c) && SHIP_CMD.test(cmd)) || SHIP_TOOLS.has(c.name) || /\bnpm\s+view\b|\bgit\s+(?:fetch|log|ls-remote)\b|\bcurl\b/.test(cmd) || /Published|Version \d+|written|delivered|status=1|"success"|completed|dist-tags|origin\/main|\bmain\b.*->|\bOK\b|\bsaved\b|done=true|uploaded|extracted/.test(r);
|
|
200
|
+
if (!isShip) continue;
|
|
201
|
+
if (shas.length && !shas.some((h) => contains(r, h))) continue;
|
|
202
|
+
if (versions.length && !versions.some((v) => contains(r, v) || contains(inputText(c), v))) continue;
|
|
203
|
+
hits.push(c);
|
|
204
|
+
}
|
|
205
|
+
if (!hits.length && versions.length) {
|
|
206
|
+
// The version is in the sentence but not in the result: a ship-tool result in the window still counts, with the gap noted.
|
|
207
|
+
const tool = all.filter((c) => SHIP_TOOLS.has(c.name) && !c.isError);
|
|
208
|
+
if (tool.length) return ev(tool.slice(-1), `ship result in the window (the result does not carry ${versions[0]})`, 'verified');
|
|
209
|
+
const src = all.filter((c) => c.category === 'read' && versions.some((v) => contains(c.resultText, v)));
|
|
210
|
+
if (src.length) return ev(src.slice(-1), 'the version comes from something the agent read, not from a ship step', 'sourced');
|
|
211
|
+
}
|
|
212
|
+
if (!hits.length) return ev([], shas.length ? `no result carries ${shas[0]}` : versions.length ? `no result carries ${versions[0]}` : 'no push, publish, upload or commit result in the window', 'unverified');
|
|
213
|
+
const ok = hits.filter((c) => !c.isError && !/\b(?:fatal|rejected|denied|Authentication failed|ERR!)\b/i.test(c.resultText || ''));
|
|
214
|
+
// "Committed as c9b4747, but the push was refused": the failure is in the sentence, so it is not a contradiction.
|
|
215
|
+
if (!ok.length) return SHIP_DECLINED_RE.test(s) ? ev(hits.slice(-1), 'the ship step failed, as the sentence says', 'verified') : ev(hits.slice(-1), 'the ship step in the window failed', 'contradicted');
|
|
216
|
+
return ev(ok.slice(-1), 'ship result in the window', 'verified');
|
|
217
|
+
}
|
|
218
|
+
if (claim.kind === 'write') {
|
|
219
|
+
const names = [];
|
|
220
|
+
let m; PATH_RE.lastIndex = 0; while ((m = PATH_RE.exec(s))) { const n = (m[1] || m[2]).replace(/[\\/]+$/, '').split(/[\\/]/).pop(); if (n) names.push(n); }
|
|
221
|
+
// A shell call counts as a write only when its command writes (cp, mv, sed -i, a redirect, a build); `tail x` naming the file does not.
|
|
222
|
+
const writes = (c) => WRITE_TOOLS.has(c.name) || SHIP_TOOLS.has(c.name) || (SHELL(c) && (SHELL_EDIT_RE.test(cmdOf(c)) || /\b(?:mkdir|git\s+(?:rm|add|tag|worktree|commit|update-ref)|printf|zip|tar|touch|rsync|scp|curl\s+-T|npm\s+run\s+build|node\s+scripts\/build)\b/.test(cmdOf(c))));
|
|
223
|
+
const hits = all.filter((c) => !c.isError && writes(c) && (!names.length || names.some((n) => contains(inputText(c), n) || contains(c.resultText, n))));
|
|
224
|
+
if (!hits.length && names.length && !MY_WRITE_RE.test(s)) {
|
|
225
|
+
// "The repair agent wrote 10 rows into logs/x.jsonl": someone else's write, seen by reading the file.
|
|
226
|
+
const seen = all.filter((c) => !c.isError && c.category !== 'write' && names.some((n) => contains(inputText(c), n) || contains(c.resultText, n)));
|
|
227
|
+
if (seen.length) return ev(seen.slice(-1), 'the file was read in the window (the write is attributed to something else)', 'verified');
|
|
228
|
+
}
|
|
229
|
+
if (!hits.length) return ev([], names.length ? `no write touching ${names[0]}` : 'no write in the window', 'unverified');
|
|
230
|
+
return ev(hits.slice(-1), 'write in the window', 'verified');
|
|
231
|
+
}
|
|
232
|
+
if (claim.kind === 'fix') {
|
|
233
|
+
// The receipt is a run after the last edit to code or tests, or failing that any inspection after the last edit
|
|
234
|
+
// of any kind: a fix to a crash is checked by running the thing again, a fix to a label by reading the page.
|
|
235
|
+
// Shell edits (sed -i, a python heredoc, cat >) count as edits, not checks.
|
|
236
|
+
const isEdit = (c) => !c.isError && c.start != null && ((c.category === 'write') || (SHELL(c) && SHELL_EDIT_RE.test(cmdOf(c))));
|
|
237
|
+
const writes = all.filter(isEdit);
|
|
238
|
+
const codeEdits = writes.filter((c) => c.category === 'write' ? SOURCE_FILE.test(String((c.input && (c.input.file_path || c.input.notebook_path)) || '')) : (cmdOf(c).match(/[\w.\/\\-]+\.[a-z]{1,5}\b/g) || []).some((x) => SOURCE_FILE.test(x)));
|
|
239
|
+
const lwCode = codeEdits.length ? Math.max(...codeEdits.map((c) => c.start)) : -Infinity;
|
|
240
|
+
const lwAny = writes.length ? Math.max(...writes.map((c) => c.start)) : -Infinity;
|
|
241
|
+
const runs = all.filter((c) => SHELL(c) && !isEdit(c) && (TEST_CMD.test(cmdOf(c)) || /node\s+-e|node\s+bin\/|python3?\s+|curl\b/.test(cmdOf(c))) && !c.isError && c.start != null && c.start >= lwCode);
|
|
242
|
+
if (runs.length) return ev(runs.slice(-1), 'a run after the last edit succeeded', 'verified');
|
|
243
|
+
const after = all.filter((c) => !isEdit(c) && c.category !== 'user' && !c.isError && c.start != null && c.start >= lwAny);
|
|
244
|
+
if (after.length) return ev(after.slice(-1), writes.length ? 'checked after the last edit (no test or re-run)' : 'inspection in the window', 'verified');
|
|
245
|
+
if (writes.length) return ev(writes.slice(-1), 'an edit, but nothing ran or was read after it', 'partial');
|
|
246
|
+
return ev([], 'no edit, run or check in the window', 'unverified');
|
|
247
|
+
}
|
|
248
|
+
// verify / state: an inspection by this agent, in the window, that did not error. (Staleness is a test-claim
|
|
249
|
+
// notion: a report written to a scratch file does not undo a check of the code.)
|
|
250
|
+
const inspections = all.filter((c) => c.category !== 'write' && c.category !== 'user' && c.category !== 'agent' && !c.isError);
|
|
251
|
+
if (inspections.length) return ev(inspections.slice(-2), 'inspection in the window', 'verified');
|
|
252
|
+
const failedOnly = all.filter((c) => c.category !== 'write' && c.isError);
|
|
253
|
+
if (failedOnly.length) return ev(failedOnly.slice(-1), 'the only check in the window failed', 'contradicted');
|
|
254
|
+
return ev([], 'no inspection in the window', 'unverified');
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
// ---------------------------------------------------------------------------------------------
|
|
258
|
+
// The ledger
|
|
259
|
+
// ---------------------------------------------------------------------------------------------
|
|
260
|
+
export function extractClaims(trace) {
|
|
261
|
+
const texts = userFacingTexts(trace);
|
|
262
|
+
const claims = [];
|
|
263
|
+
for (const t of texts) {
|
|
264
|
+
for (const sent of sentences(t.text)) {
|
|
265
|
+
const kind = classify(sent);
|
|
266
|
+
if (!kind) continue;
|
|
267
|
+
claims.push({ id: `${t.request.id}:${claims.length}`, request: t.request.id, requestNo: t.index + 1, turn: t.request.turnIndex + 1, turnIndex: t.request.turnIndex, agent: t.agent, via: t.via, kind, text: sent, numbers: kind === 'declared' ? [] : numbersIn(sent), _item: t });
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
return { texts, claims };
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
// What a claim is about: its kind, the two words before the claim words, the claim words, and its numbers.
|
|
274
|
+
function restateKey(cl) {
|
|
275
|
+
const re = (KINDS.find((k) => k[0] === cl.kind) || [])[1]; const t = cl.text.replace(/\s+/g, ' ');
|
|
276
|
+
const m = re && t.match(re); if (!m || m.index == null) return null;
|
|
277
|
+
const before = t.slice(0, m.index).toLowerCase().replace(/[^\w\s]/g, ' ').split(/\s+/).filter(Boolean).slice(-2).join(' ');
|
|
278
|
+
return `${cl.kind}|${before}|${m[0].toLowerCase()}|${(cl.numbers || []).join(',')}`;
|
|
279
|
+
}
|
|
280
|
+
export function judgeClaims(trace, opts = {}) {
|
|
281
|
+
const { texts, claims } = extractClaims(trace);
|
|
282
|
+
const allResults = opts.numbers === false ? null : trace.toolCalls.map((c) => c.resultText || '').join('\n');
|
|
283
|
+
const byReq = new Map();
|
|
284
|
+
for (const c of trace.toolCalls) { if (c.unmatched) continue; if (!byReq.has(c.requestId)) byReq.set(c.requestId, []); byReq.get(c.requestId).push(c); }
|
|
285
|
+
for (const cl of claims) {
|
|
286
|
+
if (cl.kind === 'declared') { cl.verdict = 'declared'; cl.evidence = []; cl.note = 'the agent said so'; continue; }
|
|
287
|
+
const win = windowFor(trace, cl._item, byReq, opts.window || 80);
|
|
288
|
+
const j = judgeOne(trace, cl, win);
|
|
289
|
+
cl.verdict = j.verdict; cl.evidence = j.evidence; cl.note = j.note;
|
|
290
|
+
// The same ship or write claim, verified earlier, said again in a summary: the earlier receipt stands. A commit
|
|
291
|
+
// exists and a file stays written; a test, a state or a fix claim repeated later is exactly the pattern to flag.
|
|
292
|
+
if (cl.verdict === 'unverified' && (cl.kind === 'ship' || cl.kind === 'write')) {
|
|
293
|
+
const key = restateKey(cl);
|
|
294
|
+
const earlier = key && claims.find((o) => o !== cl && o.verdict === 'verified' && o.turn <= cl.turn && restateKey(o) === key);
|
|
295
|
+
if (earlier) { cl.verdict = 'verified'; cl.evidence = earlier.evidence.slice(); cl.note = `restates turn ${earlier.turn}, verified there`; }
|
|
296
|
+
}
|
|
297
|
+
if (allResults && cl.numbers.length) { const missing = cl.numbers.filter((n) => !allResults.includes(n) && !allResults.includes(n.replace(/^\$/, ''))); if (missing.length) cl.numbersMissing = missing; }
|
|
298
|
+
// A contradicted claim the agent corrected later gets the note, and drops out of the error rule.
|
|
299
|
+
if (cl.verdict === 'contradicted') { const later = texts.find((t) => t.index > cl._item.index && t.agent === cl.agent && DECLARED_RE.test(t.text)); if (later) cl.correctedAt = later.request.turnIndex + 1; }
|
|
300
|
+
}
|
|
301
|
+
for (const cl of claims) delete cl._item;
|
|
302
|
+
const count = (v) => claims.filter((c) => c.verdict === v).length;
|
|
303
|
+
const summary = { claims: claims.length, verified: count('verified'), declared: count('declared'), sourced: count('sourced'), partial: count('partial'), stale: count('stale'), unverified: count('unverified'), contradicted: count('contradicted'), byKind: {} };
|
|
304
|
+
for (const cl of claims) summary.byKind[cl.kind] = (summary.byKind[cl.kind] || 0) + 1;
|
|
305
|
+
return { kind: 'claims', schema: 1, session: trace.meta.sessionId, generated: new Date().toISOString(), redacted: false, claims, summary };
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
// check rules: contradicted → error, unverified → warn, stale / partial → info.
|
|
309
|
+
export const CLAIM_RULES = { contradicted: ['contradicted-claim', 'error'], unverified: ['unverified-claim', 'warn'], stale: ['stale-claim', 'info'], partial: ['stale-claim', 'info'] };
|
|
310
|
+
const oneLine = (s, n) => { s = String(s == null ? '' : s).replace(/\s+/g, ' ').trim(); return s.length > n ? s.slice(0, n - 1) + '…' : s; };
|
|
311
|
+
// A long sentence, quoted around the words that made it a claim ("…and the tests still pass.").
|
|
312
|
+
export function focus(text, kind, n = 160) {
|
|
313
|
+
const t = String(text).replace(/\s+/g, ' ').trim();
|
|
314
|
+
if (t.length <= n) return t;
|
|
315
|
+
const re = (KINDS.find((k) => k[0] === kind) || [])[1];
|
|
316
|
+
const m = re && t.match(re);
|
|
317
|
+
if (!m || m.index == null) return t.slice(0, n - 1) + '…';
|
|
318
|
+
const start = Math.max(0, Math.min(m.index - 40, t.length - n));
|
|
319
|
+
return (start ? '…' : '') + t.slice(start, start + n) + (start + n < t.length ? '…' : '');
|
|
320
|
+
}
|
|
321
|
+
export function claimFindings(ledger) {
|
|
322
|
+
const out = [];
|
|
323
|
+
for (const cl of ledger.claims) {
|
|
324
|
+
const r = CLAIM_RULES[cl.verdict]; if (!r) continue;
|
|
325
|
+
const [id, sev] = cl.correctedAt ? ['contradicted-claim', 'info'] : r;
|
|
326
|
+
const what = cl.verdict === 'partial' ? 'only partly backed' : cl.verdict === 'stale' ? 'backed by a check that predates a later edit' : cl.verdict === 'contradicted' ? 'contradicted by the transcript' : 'with no receipt in the transcript';
|
|
327
|
+
// The sentence goes in `detail` (blanked by --redact); the title is names and numbers only.
|
|
328
|
+
out.push({ id, severity: sev, title: `Turn ${cl.turn}${cl.agent !== 'main' ? ' (subagent)' : ''}: a ${cl.kind} claim ${what}${cl.correctedAt ? ` (corrected at turn ${cl.correctedAt})` : ''}`, detail: `"${focus(cl.text, cl.kind)}" — ${cl.note}.${cl.numbersMissing ? ` Numbers not found in any tool result: ${cl.numbersMissing.join(', ')}.` : ''}`, evidence: { requestIds: [cl.request], toolCallIds: cl.evidence, turnIndex: cl.turnIndex }, metric: cl.verdict === 'contradicted' ? 3 : cl.verdict === 'unverified' ? 2 : 1, claim: { kind: cl.kind, verdict: cl.verdict, turn: cl.turn } });
|
|
329
|
+
}
|
|
330
|
+
return out;
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
// ---------------------------------------------------------------------------------------------
|
|
334
|
+
// Renderings
|
|
335
|
+
// ---------------------------------------------------------------------------------------------
|
|
336
|
+
const ORDER = { contradicted: 0, unverified: 1, partial: 2, stale: 3, sourced: 4, verified: 5, declared: 6 };
|
|
337
|
+
const blank = (s) => `«${String(s == null ? '' : s).length} chars»`;
|
|
338
|
+
// Under --redact a note must not carry pieces of the sentence either: shas, versions, the numbers it quoted.
|
|
339
|
+
const noteOf = (c, redact) => {
|
|
340
|
+
const n = redact ? String(c.note || '').replace(SHA_RE, '«sha»').replace(VERSION_RE, '«version»') : c.note;
|
|
341
|
+
const nums = c.numbersMissing ? (redact ? `«${c.numbersMissing.length} number${c.numbersMissing.length === 1 ? '' : 's'}»` : c.numbersMissing.join(', ')) : null;
|
|
342
|
+
return { note: n, nums };
|
|
343
|
+
};
|
|
344
|
+
export function headline(rep) { const s = rep.summary; return `${s.claims} claim${s.claims === 1 ? '' : 's'} · ${s.verified} verified · ${s.declared} declared · ${s.unverified} unverified · ${s.contradicted} contradicted${s.partial || s.stale || s.sourced ? ` (${s.partial} partial, ${s.stale} stale, ${s.sourced} sourced)` : ''}`; }
|
|
345
|
+
export function claimsText(rep, opts = {}) {
|
|
346
|
+
const R = (t) => opts.redact ? blank(t) : t;
|
|
347
|
+
const L = [`Glassbox claims · ${rep.session ? rep.session.slice(0, 8) : 'session'}`, ` ${headline(rep)}`, ''];
|
|
348
|
+
const rows = rep.claims.slice().sort((a, b) => ORDER[a.verdict] - ORDER[b.verdict] || a.turn - b.turn);
|
|
349
|
+
for (const c of rows) {
|
|
350
|
+
const { note, nums } = noteOf(c, opts.redact);
|
|
351
|
+
L.push(` ${c.verdict.toUpperCase().padEnd(12)} ${c.kind.padEnd(8)} t${String(c.turn).padEnd(3)} ${oneLine(R(c.text), 96)}`);
|
|
352
|
+
L.push(` ↳ ${note}${c.evidence.length ? ` (${c.evidence.length} call${c.evidence.length === 1 ? '' : 's'})` : ''}${c.correctedAt ? ` · corrected at t${c.correctedAt}` : ''}${nums ? ` · numbers not in any result: ${nums}` : ''}`);
|
|
353
|
+
}
|
|
354
|
+
if (!rows.length) L.push(' no claims found — nothing the agent said asserted a checkable result');
|
|
355
|
+
L.push('', ' A verdict is about the sentence, not the code: verified means a tool result in this transcript shows it; unverified means none does. Checks done outside the transcript are invisible here.');
|
|
356
|
+
return L.join('\n') + '\n';
|
|
357
|
+
}
|
|
358
|
+
export function claimsMarkdown(rep, trace, opts = {}) {
|
|
359
|
+
const R = (t) => opts.redact ? blank(t) : t;
|
|
360
|
+
const callLine = (id) => { const c = trace && trace.toolCalls.find((x) => x.id === id); if (!c) return `\`${id}\``; const inp = c.input || {}; const what = inp.command || inp.file_path || inp.url || inp.message || ''; return `${c.name}${c.isError ? ' (error)' : ''} · turn ${c.turnIndex + 1}${what ? ': `' + oneLine(R(what), 80).replace(/`/g, "'") + '`' : ''}`; };
|
|
361
|
+
const L = ['# Glassbox claims — said vs did', '', `Session \`${rep.session || '—'}\` · ${rep.generated.slice(0, 10)}`, '', `**${headline(rep)}**`, '', '| turn | kind | claim | evidence | verdict |', '|---|---|---|---|---|'];
|
|
362
|
+
const rows = rep.claims.slice().sort((a, b) => ORDER[a.verdict] - ORDER[b.verdict] || a.turn - b.turn);
|
|
363
|
+
for (const c of rows) L.push(`| ${c.turn}${c.agent !== 'main' ? ' (sub)' : ''} | \`${c.kind}\` | ${oneLine(R(c.text), 110).replace(/\|/g, '\\|')} | ${c.evidence.length ? c.evidence.map(callLine).join('<br>').replace(/\|/g, '\\|') : '—'} | **${c.verdict}**${c.correctedAt ? ` (corrected at turn ${c.correctedAt})` : ''} |`);
|
|
364
|
+
const bad = rows.filter((c) => c.verdict === 'contradicted' || c.verdict === 'unverified');
|
|
365
|
+
if (bad.length) { L.push('', '## Without a receipt', ''); for (const c of bad) { const { note, nums } = noteOf(c, opts.redact); L.push(`- **${c.verdict}** · turn ${c.turn} · ${oneLine(R(c.text), 160)} — ${note}${nums ? `; numbers not in any tool result: ${nums}` : ''}`); } }
|
|
366
|
+
L.push('', '## How to read this', '', 'Every claim is one sentence the agent addressed to the human. *Verified* means a tool result in this transcript shows it, after the last change that could have undone it; *declared* means the agent itself said it had not checked; *partial* means the evidence covers part of the sentence; *stale* means the check ran before a later edit; *unverified* means no result backs it; *contradicted* means a result says the opposite. Nothing here judges the code — only whether the sentence has a receipt. `glassbox claims --format json` has every row.', '');
|
|
367
|
+
return L.join('\n');
|
|
368
|
+
}
|
package/src/cli.mjs
CHANGED
|
@@ -7,6 +7,8 @@ import crypto from 'node:crypto';
|
|
|
7
7
|
import { createRequire } from 'node:module';
|
|
8
8
|
import { fileURLToPath } from 'node:url';
|
|
9
9
|
import { readTextFile } from './textfile.mjs';
|
|
10
|
+
import { guardResponse } from './guard.mjs';
|
|
11
|
+
import { judgeClaims, claimFindings, claimsText, claimsMarkdown } from './claims.mjs';
|
|
10
12
|
const require = createRequire(import.meta.url);
|
|
11
13
|
const core = require('./trace-core.js');
|
|
12
14
|
// fileURLToPath, not URL.pathname: on Windows the latter gives "/C:/…" which path.resolve turns into "\C:\…".
|
|
@@ -83,7 +85,18 @@ export function loadSessionFiles(s) {
|
|
|
83
85
|
return files;
|
|
84
86
|
}
|
|
85
87
|
|
|
86
|
-
|
|
88
|
+
// Findings are the mechanical rules from trace-core plus the claims ledger's three (contradicted / unverified /
|
|
89
|
+
// stale), in one list, so check, the hook, the Action and compare all see them.
|
|
90
|
+
const SEV_ORDER = { error: 0, warn: 1, info: 2 };
|
|
91
|
+
export function analyse(files, rates) {
|
|
92
|
+
const trace = core.parseTrace(files);
|
|
93
|
+
const findings = core.diagnose(trace, rates ? { rates } : undefined);
|
|
94
|
+
const cost = core.estimateCost(trace, rates);
|
|
95
|
+
const claims = judgeClaims(trace);
|
|
96
|
+
findings.push(...claimFindings(claims));
|
|
97
|
+
findings.sort((a, b) => SEV_ORDER[a.severity] - SEV_ORDER[b.severity] || (b.metric || 0) - (a.metric || 0));
|
|
98
|
+
return { trace, findings, cost, claims };
|
|
99
|
+
}
|
|
87
100
|
|
|
88
101
|
// A team rate card: --rates FILE or GLASSBOX_RATES. JSON keyed by model-id prefix, USD per million tokens:
|
|
89
102
|
// { "claude-opus-5": { "in": 4, "out": 20, "read": 0.4, "w5m": 5, "w1h": 8 } }
|
|
@@ -164,7 +177,7 @@ export function collapseFindings(findings) {
|
|
|
164
177
|
}
|
|
165
178
|
const findingLine = ({ f, n }) => `${f.severity.toUpperCase().padEnd(5)} ${f.id.padEnd(16)} ${f.title}${n > 1 ? ` ×${n}` : ''}`;
|
|
166
179
|
|
|
167
|
-
export function checkReport({ trace, findings, cost }, opts = {}) {
|
|
180
|
+
export function checkReport({ trace, findings, cost, claims }, opts = {}) {
|
|
168
181
|
const failOn = opts.failOn || 'error';
|
|
169
182
|
const failing = findings.filter((f) => SEV[f.severity] <= SEV[failOn]);
|
|
170
183
|
const T = trace.totals, m = trace.meta;
|
|
@@ -176,6 +189,7 @@ export function checkReport({ trace, findings, cost }, opts = {}) {
|
|
|
176
189
|
const summary = { session: m.sessionId, title: m.title ? (redact ? '«' + m.title.length + ' chars»' : m.title) : null, model: m.models, wallMs: T.wallMs, activeMs: T.activeMs, turns: T.turns, requests: T.requests, toolCalls: T.toolCalls, toolErrors: T.toolErrors, orphans: T.orphans, usage: T.usage, cacheHitRatio: T.cacheHitRatio, cost: cost.reported != null ? cost.reported : cost.total, costSource: cost.source, start: m.start != null ? new Date(m.start).toISOString() : null, end: m.end != null ? new Date(m.end).toISOString() : null };
|
|
177
190
|
// Per-file use, keyed when a legend is given; with --redact and no legend the paths would leak, so it is omitted.
|
|
178
191
|
if (legend || !redact) summary.files = core.fileStats(trace).map((r) => ({ key: legend ? legend.keyFor(r.path) : r.path, reads: r.reads, writes: r.writes, errors: r.errors, agents: r.agents, chars: r.chars }));
|
|
192
|
+
if (claims) summary.claims = claims.summary; // said vs did, in one line of numbers (additive; schema stays 2)
|
|
179
193
|
const findingJson = (f) => {
|
|
180
194
|
const ev = f.evidence || {};
|
|
181
195
|
const paths = Array.from(new Set((ev.toolCallIds || []).map((id) => { const c = trace.toolCalls.find((x) => x.id === id); return c ? core.filePathOf(c) : null; }).filter(Boolean)));
|
|
@@ -248,6 +262,8 @@ export function hookResponse(input, opts = {}) {
|
|
|
248
262
|
const failOn = opts.failOn || 'warn';
|
|
249
263
|
const feedback = !!opts.feedback;
|
|
250
264
|
if (input && input.hook_event_name === 'SessionStart') return opts.context ? sessionStartResponse(input) : { suppressOutput: true };
|
|
265
|
+
// PreToolUse: null means "say nothing" (normal permission flow), never an approval.
|
|
266
|
+
if (input && input.hook_event_name === 'PreToolUse') return opts.guard ? guardResponse(input) : null;
|
|
251
267
|
// Second Stop after a feedback block: record what the agent said it would do differently, then let it stop.
|
|
252
268
|
if (opts.context && input && input.hook_event_name === 'Stop' && input.stop_hook_active) {
|
|
253
269
|
const file = notesPath(input.cwd);
|
|
@@ -289,23 +305,33 @@ export function readStdinJson() {
|
|
|
289
305
|
return new Promise((resolve) => { let buf = ''; process.stdin.setEncoding('utf8'); process.stdin.on('data', (d) => { buf += d; }); process.stdin.on('end', () => { try { resolve(JSON.parse(buf || '{}')); } catch (e) { resolve({}); } }); if (process.stdin.isTTY) resolve({}); });
|
|
290
306
|
}
|
|
291
307
|
|
|
292
|
-
|
|
308
|
+
// A Glassbox hook command in settings.json: `npx -y glassbox-trace hook …`, `glassbox hook …`, or the direct
|
|
309
|
+
// form `node "…/bin/glassbox.mjs" hook …` that --guard installs.
|
|
310
|
+
export const HOOK_RE = /\bglassbox(?:-trace)?(?:\.mjs)?["']?\s+hook\b/;
|
|
293
311
|
export function settingsPath(home) { return path.join(home || claudeHome(), 'settings.json'); }
|
|
294
312
|
export function installHook(opts = {}) {
|
|
295
313
|
const file = settingsPath(opts.home);
|
|
296
314
|
let settings = {};
|
|
297
315
|
if (fs.existsSync(file)) { try { settings = JSON.parse(readTextFile(file)); } catch (e) { throw new Error(`${file} is not valid JSON — fix it first (nothing was changed)`); } fs.copyFileSync(file, file + '.glassbox-backup'); }
|
|
298
316
|
settings.hooks = settings.hooks || {};
|
|
299
|
-
const cmd = `${opts.command || 'npx -y glassbox-trace'} hook${opts.feedback ? ' --feedback' : ''}${opts.context ? ' --context' : ''}${opts.failOn ? ' --fail-on ' + opts.failOn : ''}`;
|
|
300
|
-
|
|
317
|
+
const cmd = `${opts.command || 'npx -y glassbox-trace'} hook${opts.feedback ? ' --feedback' : ''}${opts.context ? ' --context' : ''}${opts.guard ? ' --guard' : ''}${opts.failOn ? ' --fail-on ' + opts.failOn : ''}`;
|
|
318
|
+
const events = opts.events || ['Stop', ...(opts.context ? ['SessionStart'] : []), ...(opts.guard ? ['PreToolUse'] : [])];
|
|
319
|
+
for (const ev of events) {
|
|
301
320
|
const list = Array.isArray(settings.hooks[ev]) ? settings.hooks[ev] : [];
|
|
302
321
|
const kept = list.filter((entry) => !(entry && Array.isArray(entry.hooks) && entry.hooks.some((h) => h && typeof h.command === 'string' && HOOK_RE.test(h.command))));
|
|
303
|
-
|
|
322
|
+
// PreToolUse runs before every tool call, so it gets a short timeout; no matcher means all tools.
|
|
323
|
+
kept.push({ hooks: [{ type: 'command', command: cmd, timeout: ev === 'PreToolUse' ? 10 : 60 }] });
|
|
304
324
|
settings.hooks[ev] = kept;
|
|
305
325
|
}
|
|
306
326
|
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
307
327
|
fs.writeFileSync(file, JSON.stringify(settings, null, 2) + '\n');
|
|
308
|
-
return { file, command: cmd };
|
|
328
|
+
return { file, command: cmd, events };
|
|
329
|
+
}
|
|
330
|
+
// The guard runs before every tool call, so it can't go through npx (seconds per call). Point the hook at
|
|
331
|
+
// this install's own entry point instead — unless this *is* a throwaway npx cache copy.
|
|
332
|
+
export function guardCommand(root) {
|
|
333
|
+
if (/[\\/]_npx[\\/]/.test(root)) throw new Error('--guard runs before every tool call, and through npx that takes seconds each time. Install Glassbox once (npm i -g glassbox-trace) and run `glassbox hook install --guard` again, or pass --command with a direct path.');
|
|
334
|
+
return `node "${path.join(root, 'bin', 'glassbox.mjs')}"`;
|
|
309
335
|
}
|
|
310
336
|
export function uninstallHook(opts = {}) {
|
|
311
337
|
const file = settingsPath(opts.home);
|
|
@@ -325,7 +351,7 @@ export function parseArgs(argv) {
|
|
|
325
351
|
const args = { _: [], flags: {} };
|
|
326
352
|
for (let i = 0; i < argv.length; i++) {
|
|
327
353
|
const a = argv[i];
|
|
328
|
-
if (a.startsWith('--')) { const [k, v] = a.slice(2).split('='); if (v !== undefined) args.flags[k] = v; else if (i + 1 < argv.length && !argv[i + 1].startsWith('-') && ['last', 'grep', 'project', 'out', 'fail-on', 'home', 'command', 'events', 'format', 'label-a', 'label-b', 'port', 'rates', 'since', 'legend', 'claude-md', 'fail-under'].includes(k)) args.flags[k] = argv[++i]; else args.flags[k] = true; }
|
|
354
|
+
if (a.startsWith('--')) { const [k, v] = a.slice(2).split('='); if (v !== undefined) args.flags[k] = v; else if (i + 1 < argv.length && !argv[i + 1].startsWith('-') && ['last', 'grep', 'project', 'out', 'fail-on', 'home', 'command', 'events', 'format', 'label-a', 'label-b', 'port', 'rates', 'since', 'legend', 'claude-md', 'fail-under', 'key'].includes(k)) args.flags[k] = argv[++i]; else args.flags[k] = true; }
|
|
329
355
|
else args._.push(a);
|
|
330
356
|
}
|
|
331
357
|
return args;
|
|
@@ -353,10 +379,11 @@ export const HELP = `glassbox — other tools show you what happened in a Claude
|
|
|
353
379
|
same task, two sessions: what changed in time, tokens, cost, tools and findings
|
|
354
380
|
glassbox watch [ID|FILE] [--port N] [--no-open]
|
|
355
381
|
live tail: serves the viewer on 127.0.0.1 and pushes the transcript as it grows
|
|
356
|
-
glassbox hook install [--feedback] [--context] [--fail-on warn]
|
|
382
|
+
glassbox hook install [--feedback] [--context] [--guard] [--fail-on warn]
|
|
357
383
|
add a Claude Code Stop hook so every session ends with a Glassbox summary;
|
|
358
384
|
--feedback also hands the findings back to the agent once, so it can learn from them;
|
|
359
|
-
--context keeps them in <project>/.glassbox/last-session.md and gives them to the next session
|
|
385
|
+
--context keeps them in <project>/.glassbox/last-session.md and gives them to the next session;
|
|
386
|
+
--guard blocks a tool call that already failed twice in a row, unchanged, and says why
|
|
360
387
|
glassbox hook uninstall remove it (a .glassbox-backup of settings.json is kept)
|
|
361
388
|
glassbox collect DIR [--since 30d] [--format text|md|json] [--out FILE] [--top N]
|
|
362
389
|
fleet view from a folder of check reports: each machine writes one with
|
|
@@ -367,7 +394,11 @@ export const HELP = `glassbox — other tools show you what happened in a Claude
|
|
|
367
394
|
is my CLAUDE.md doing anything? every rule in the project's instruction files judged
|
|
368
395
|
against every session of that project: obeyed / broken per occasion, with evidence;
|
|
369
396
|
shapes it cannot check are listed as such. --fail-under 80 exits 1 below that rate
|
|
370
|
-
glassbox
|
|
397
|
+
glassbox claims [ID|FILE] [--format text|md|json] [--out FILE] [--redact] [--fail-on contradicted|unverified|none]
|
|
398
|
+
said vs did: every claim the agent made about its own work (tests pass, committed,
|
|
399
|
+
live, verified, nothing changed) matched to the tool result behind it — or the gap.
|
|
400
|
+
exit 1 when a claim at/above --fail-on has no receipt (default: contradicted)
|
|
401
|
+
glassbox fence [ID|FILE|DIR] [--since 30d] [--format text|md|json] [--out FILE] [--fail-on error|warn|info] [--shred] [--key FILE] [--sessions-only]
|
|
371
402
|
secrets that reached a transcript: known key formats, secrets named by context,
|
|
372
403
|
credential-file reads — with a masked preview and a fingerprint, never the value.
|
|
373
404
|
no target = every session under the home; exit 1 when anything at/above --fail-on was found
|
|
@@ -520,22 +551,39 @@ export async function main(argv, io = {}) {
|
|
|
520
551
|
}
|
|
521
552
|
if (cmd === 'hook') {
|
|
522
553
|
const sub = args._[1];
|
|
523
|
-
if (sub === 'install') { const r = installHook({ home, feedback: !!args.flags.feedback, context: !!args.flags.context, failOn: args.flags['fail-on'], command: args.flags.command, events: args.flags.events ? String(args.flags.events).split(',') : undefined }); out(`Installed ${
|
|
554
|
+
if (sub === 'install') { const r = installHook({ home, feedback: !!args.flags.feedback, context: !!args.flags.context, guard: !!args.flags.guard, failOn: args.flags['fail-on'], command: args.flags.command || (args.flags.guard ? guardCommand(ROOT) : undefined), events: args.flags.events ? String(args.flags.events).split(',') : undefined }); out(`Installed ${r.events.join(', ')} hook${r.events.length > 1 ? 's' : ''} in ${r.file}\n ${r.command}\nEvery session now ends with a Glassbox summary${args.flags.feedback ? ', and findings are handed back to the agent once' : ''}${args.flags.context ? `; findings are kept in <project>/${NOTES_FILE.replace(/\\/g, '/')} and handed to the next session there` : ''}${args.flags.guard ? '; a call that already failed twice in a row, unchanged, is blocked with the reason' : ''}. Restart Claude Code to pick it up.`); return 0; }
|
|
524
555
|
if (sub === 'uninstall') { const r = uninstallHook({ home }); out(`Removed ${r.removed} Glassbox hook${r.removed === 1 ? '' : 's'} from ${r.file}`); return 0; }
|
|
525
556
|
// A hook must never fail loudly: exit 2 from a Stop hook blocks Claude with the error as the reason.
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
557
|
+
// A null reply prints nothing, which for PreToolUse means "no decision".
|
|
558
|
+
let reply = null, event = null;
|
|
559
|
+
try { const input = io.stdin !== undefined ? io.stdin : await readStdinJson(); event = input && input.hook_event_name; reply = hookResponse(input, { failOn: args.flags['fail-on'], feedback: !!args.flags.feedback, context: !!args.flags.context, guard: !!args.flags.guard, rates, stateDir: io.stateDir }); }
|
|
560
|
+
catch (e) { reply = event === 'PreToolUse' ? null : { systemMessage: 'Glassbox: ' + e.message, suppressOutput: true }; }
|
|
561
|
+
if (reply != null) out(JSON.stringify(reply));
|
|
530
562
|
return 0;
|
|
531
563
|
}
|
|
564
|
+
if (cmd === 'claims') {
|
|
565
|
+
// Said vs did: the claims ledger for one session (subagents included).
|
|
566
|
+
const s = resolveTarget(args._[1], { home });
|
|
567
|
+
const { trace } = analyse(loadSessionFiles(s), rates);
|
|
568
|
+
const rep = judgeClaims(trace);
|
|
569
|
+
const failOn = String(args.flags['fail-on'] || 'contradicted');
|
|
570
|
+
if (!['contradicted', 'unverified', 'none'].includes(failOn)) throw new Error(`--fail-on must be contradicted, unverified or none (got "${failOn}")`);
|
|
571
|
+
const redact = !!args.flags.redact;
|
|
572
|
+
if (redact) { rep.redacted = true; for (const c of rep.claims) c.text = `«${c.text.length} chars»`; }
|
|
573
|
+
const fmtOut = outputFormat(args.flags);
|
|
574
|
+
const text = fmtOut === 'json' ? JSON.stringify(rep, null, 2) : fmtOut === 'md' ? claimsMarkdown(rep, trace, { redact }) : claimsText(rep, { redact });
|
|
575
|
+
if (args.flags.out) { const f = path.resolve(args.flags.out); fs.writeFileSync(f, text); out(`${f} (${rep.summary.claims} claims, ${rep.summary.unverified} unverified, ${rep.summary.contradicted} contradicted)`); }
|
|
576
|
+
else out(text.replace(/\n$/, ''));
|
|
577
|
+
const failed = failOn === 'none' ? false : failOn === 'unverified' ? (rep.summary.unverified + rep.summary.contradicted) > 0 : rep.summary.contradicted > 0;
|
|
578
|
+
return failed ? 1 : 0;
|
|
579
|
+
}
|
|
532
580
|
if (cmd === 'fence') {
|
|
533
581
|
// Secrets that reached a transcript. No target = every session under the home; an id, a file, or a folder of .jsonl.
|
|
534
582
|
const { fence, fenceText, fenceMarkdown, failedAt } = await import('./fence.mjs');
|
|
535
583
|
const since = args.flags.since ? Date.now() - parseSince(args.flags.since) : null;
|
|
536
584
|
const failOn = args.flags['fail-on'] || 'error';
|
|
537
585
|
const fmtOut = outputFormat(args.flags);
|
|
538
|
-
const rep = fence({ home, target: args._[1], since, project: args.flags.project, shred: !!args.flags.shred });
|
|
586
|
+
const rep = fence({ home, target: args._[1], since, project: args.flags.project, shred: !!args.flags.shred, keyFile: args.flags.key, sessionsOnly: !!args.flags['sessions-only'] });
|
|
539
587
|
const failed = failedAt(rep, failOn);
|
|
540
588
|
const text = fmtOut === 'json' ? JSON.stringify(rep, null, 2) : fmtOut === 'md' ? fenceMarkdown(rep) : fenceText(rep);
|
|
541
589
|
if (args.flags.out) { const f = path.resolve(args.flags.out); fs.writeFileSync(f, text); out(`${f} (${rep.scanned.files} files, ${rep.findings.length} findings${rep.shredded ? `, ${rep.shredded.values} values shredded` : ''})`); }
|