crewforth 3.1.0-rc.3 → 3.1.0-rc.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +66 -9
- package/README.md +1 -1
- package/kit/AGENT_TEMPLATE.md +26 -17
- package/kit/CLAUDE.md +11 -0
- package/kit/agents/crew-backend-expert.md +5 -0
- package/kit/agents/crew-commit-agent.md +8 -3
- package/kit/agents/crew-database-expert.md +5 -0
- package/kit/agents/crew-devops-expert.md +5 -0
- package/kit/agents/crew-frontend-expert.md +5 -0
- package/kit/agents/crew-performance-expert.md +5 -0
- package/kit/agents/crew-planner.md +5 -0
- package/kit/agents/crew-privacy-agent.md +7 -2
- package/kit/agents/crew-review-agent.md +5 -0
- package/kit/agents/crew-security-expert.md +8 -4
- package/kit/agents/crew-session-manager.md +7 -3
- package/kit/agents/crew-test-expert.md +5 -0
- package/kit/eval/doctor.sh +23 -7
- package/kit/eval/smoke-test.sh +377 -27
- package/kit/hooks/context-usage.sh +1 -1
- package/kit/hooks/guard-agent-model.sh +392 -0
- package/kit/hooks/guard-bash.sh +61 -5
- package/kit/hooks/guard-schedule.sh +92 -0
- package/kit/hooks/prompt-approval.sh +68 -21
- package/kit/settings.json +22 -0
- package/kit/skills/crew-approve/SKILL.md +26 -0
- package/kit/studio/server/index.js +1 -1
- package/kit/studio/server/lib/graph.js +127 -10
- package/kit/studio/server/lib/usage.js +96 -25
- package/kit/studio/web/app.js +49 -6
- package/kit/studio/web/canvas.js +195 -30
- package/kit/studio/web/chat.js +8 -1
- package/kit/studio/web/convo.js +5 -1
- package/kit/studio/web/graph-plan.js +139 -6
- package/kit/studio/web/inspect.js +33 -1
- package/kit/studio/web/list.js +8 -2
- package/kit/studio/web/names.js +65 -0
- package/kit/studio/web/style.css +65 -6
- package/kit/studio/web/timeline-plan.js +8 -0
- package/kit/studio/web/timeline.js +10 -3
- package/kit/studio/web/usage.js +53 -12
- package/package.json +1 -1
|
@@ -8,8 +8,8 @@
|
|
|
8
8
|
# yes, and this hook is the only thing that turns it into a record.
|
|
9
9
|
#
|
|
10
10
|
# WHAT COUNTS. The WHOLE message, trimmed, is one of (letter case does not matter):
|
|
11
|
-
# approve
|
|
12
|
-
#
|
|
11
|
+
# /crew-approve commit /crew-approve push /crew-approve commit+push
|
|
12
|
+
# and, as the release candidates of 3.1.0 had it, `approve: <what>` or `onay: <what>`.
|
|
13
13
|
# A sentence that contains those words is not an approval, and neither is a message with a second line: text
|
|
14
14
|
# that was pasted, quoted or relayed from somewhere else never has that shape by accident.
|
|
15
15
|
#
|
|
@@ -51,8 +51,13 @@ _CREW_LOCALE="${LC_ALL:-${LC_CTYPE:-${LANG:-}}}"
|
|
|
51
51
|
case "$_CREW_LOCALE" in C|POSIX) _CREW_LOCALE="" ;; esac
|
|
52
52
|
export LC_ALL=C
|
|
53
53
|
# ---- /CREW-LOCALE
|
|
54
|
-
|
|
55
|
-
|
|
54
|
+
# guard-schedule.sh SOURCES this file for one thing, _crew_appr_op: which text is an approval is decided here and
|
|
55
|
+
# nowhere else. Sourced, the file defines its functions and returns before anything is read or written.
|
|
56
|
+
_PA_LIB=0; [ "${BASH_SOURCE[0]}" != "$0" ] && _PA_LIB=1
|
|
57
|
+
if [ "$_PA_LIB" = 0 ]; then
|
|
58
|
+
IFS= read -r -d '' INPUT || true
|
|
59
|
+
case "$INPUT" in *'"hook_event_name"'*UserPromptSubmit*) ;; *) exit 0 ;; esac
|
|
60
|
+
fi
|
|
56
61
|
# ---- CREW-JSON-PARSE ------------------------------------------------------------------------------------
|
|
57
62
|
_json_find(){ # $1 = text, $2 = a literal -> _JF = the offset of its first occurrence, -1 when there is none
|
|
58
63
|
# `${text%%"$literal"*}` gives the same offset and costs the DISTANCE to the occurrence times the length of the
|
|
@@ -351,6 +356,38 @@ _crew_appr_path(){ # $1 = a directory -> _AP: the record's path in that worktre
|
|
|
351
356
|
}
|
|
352
357
|
# ---- /CREW-APPROVAL-PATH ---------------------------------------------------------------------------------
|
|
353
358
|
|
|
359
|
+
# ---- WHICH TEXT IS AN APPROVAL: one function, asked here of the user's prompt and by guard-schedule.sh of a
|
|
360
|
+
# prompt a tool is about to schedule ---------------------------------------------------------------------------
|
|
361
|
+
_crew_appr_op(){ # $1 = a text, decoded -> OP: commit | push | commit+push, or "" when the text is not an approval
|
|
362
|
+
local P="$1" _k _v LC_ALL=C
|
|
363
|
+
OP=""
|
|
364
|
+
P="${P//$'\r'/}"
|
|
365
|
+
while :; do case "$P" in [$' \t\n']*) P="${P:1}" ;; *) break ;; esac; done
|
|
366
|
+
while :; do case "$P" in *[$' \t\n']) P="${P%?}" ;; *) break ;; esac; done
|
|
367
|
+
[ "${#P}" -le 40 ] || return 0
|
|
368
|
+
case "$P" in
|
|
369
|
+
*$'\n'*) return 0 ;;
|
|
370
|
+
# THE COMMAND: `/crew-approve <what>`, or `/<plugin>:crew-approve <what>` as a plugin names its commands. Measured
|
|
371
|
+
# on Claude Code 2.1.284: a command the user types reaches UserPromptSubmit with `prompt` as typed, and a skill
|
|
372
|
+
# the model starts with the Skill tool raises PreToolUse and PostToolUse only, never that event.
|
|
373
|
+
/*) _k="${P%%[$' \t']*}"; [ "$_k" != "$P" ] || return 0
|
|
374
|
+
_v="${P:${#_k}}"; _k="${_k#/}"; _k="@${_k##*:}" ;; # `@`: a command, so `crew-approve: commit` as text is not one
|
|
375
|
+
# The text form of 3.1.0's release candidates, kept: `approve: <what>` and `onay: <what>`.
|
|
376
|
+
*:*) _k="${P%%:*}"; _v="${P#*:}" ;;
|
|
377
|
+
*) return 0 ;;
|
|
378
|
+
esac
|
|
379
|
+
_k="${_k//[$' \t']/}"; _v="${_v//[$' \t']/}"
|
|
380
|
+
# In the C locale: under tr_TR an upper-case I does not fold to i, and `ONAY: COMMIT` was not an approval (review).
|
|
381
|
+
shopt -s nocasematch
|
|
382
|
+
case "$_k" in
|
|
383
|
+
@crew-approve|approve|onay)
|
|
384
|
+
case "$_v" in commit) OP=commit ;; push) OP=push ;; commit+push) OP=commit+push ;; esac ;;
|
|
385
|
+
esac
|
|
386
|
+
shopt -u nocasematch
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
[ "$_PA_LIB" = 1 ] && return 0
|
|
390
|
+
|
|
354
391
|
# A second "prompt" key cannot come from the text of a prompt (inside a JSON string every quote is escaped), so
|
|
355
392
|
# more than one means the payload is not the shape this hook knows, and nothing is read from it.
|
|
356
393
|
_json_keycount "$INPUT" prompt; _n_prompt=$_KC
|
|
@@ -384,23 +421,8 @@ if [ -n "$REC" ] && [ -s "$REC" ]; then : > "$REC" 2>/dev/null || true; fi
|
|
|
384
421
|
# ---- is this message an approval? ------------------------------------------------------------------------
|
|
385
422
|
[ "$_n_prompt" = 1 ] || exit 0
|
|
386
423
|
[ "${#RAW}" -le 120 ] || exit 0 # an approval is a few words; a pasted page is not decoded to find that out
|
|
387
|
-
_json_unescape "$RAW" >/dev/null
|
|
388
|
-
|
|
389
|
-
while :; do case "$P" in [$' \t\n']*) P="${P:1}" ;; *) break ;; esac; done
|
|
390
|
-
while :; do case "$P" in *[$' \t\n']) P="${P%?}" ;; *) break ;; esac; done
|
|
391
|
-
[ "${#P}" -le 40 ] || exit 0
|
|
392
|
-
case "$P" in *$'\n'*) exit 0 ;; *:*) ;; *) exit 0 ;; esac
|
|
393
|
-
_k="${P%%:*}"; _v="${P#*:}"
|
|
394
|
-
_k="${_k//[$' \t']/}"; _v="${_v//[$' \t']/}"
|
|
395
|
-
OP=""
|
|
396
|
-
# In the C locale: under tr_TR an upper-case I does not fold to i, and `ONAY: COMMIT` was not an approval (review).
|
|
397
|
-
_pa_match(){ local LC_ALL=C; shopt -s nocasematch
|
|
398
|
-
case "$_k" in
|
|
399
|
-
approve|onay)
|
|
400
|
-
case "$_v" in commit) OP=commit ;; push) OP=push ;; commit+push) OP=commit+push ;; esac ;;
|
|
401
|
-
esac
|
|
402
|
-
shopt -u nocasematch; }
|
|
403
|
-
_pa_match
|
|
424
|
+
_json_unescape "$RAW" >/dev/null
|
|
425
|
+
_crew_appr_op "$_JU"
|
|
404
426
|
[ -n "$OP" ] || exit 0
|
|
405
427
|
|
|
406
428
|
# From here on the message IS an approval, so every way of not recording it says why - to the user and to the
|
|
@@ -420,6 +442,31 @@ case "$PM" in
|
|
|
420
442
|
esac
|
|
421
443
|
[ -n "$REC" ] || no "this directory is not inside a git repository."
|
|
422
444
|
|
|
445
|
+
# A PERSON, IN A SESSION THAT HAS SHOWN THEM SOMETHING. A session can start another one and hand it the approval as
|
|
446
|
+
# its prompt: `claude -p "/crew-approve commit"`, or an interactive one driven through a pseudo-terminal. That
|
|
447
|
+
# session's only message is the approval, this hook would record it, and the session would then commit: the first
|
|
448
|
+
# session has approved for the user. Measured on Claude Code 2.1.294 (macOS), from a hook that printed what it saw:
|
|
449
|
+
# started how CLAUDE_CODE_SESSION_ATTENDED the transcript file when this hook runs
|
|
450
|
+
# claude -p, from a session's Bash 0 (set to 1 by the caller: still 0) not there (first message)
|
|
451
|
+
# claude -p, from an emptied environment 0 not there (first message)
|
|
452
|
+
# interactive, in a pty, from a session 1 not there, and never written: a
|
|
453
|
+
# session started inside another keeps none
|
|
454
|
+
# interactive, in a pty, emptied environment 1 not there at the first message; there after
|
|
455
|
+
# the desktop app, a conversation under way 1 there
|
|
456
|
+
# Nothing else told them apart: the payload has the same fields, CLAUDECODE and CLAUDE_CODE_CHILD_SESSION are 1 in
|
|
457
|
+
# all of them, CLAUDE_CODE_ENTRYPOINT is inherited from the outer session, and no hook has a terminal on stdin.
|
|
458
|
+
# So two things are asked. Neither is a documented contract, and both fail towards NOT recording:
|
|
459
|
+
# 1. the session is not a headless one;
|
|
460
|
+
# 2. the session's transcript is already on disk when the approval arrives, so the approval is not the session's
|
|
461
|
+
# first message and the session is not one that keeps no transcript.
|
|
462
|
+
# HONEST SCOPE: a session started with an emptied environment and driven through a pseudo-terminal for a second
|
|
463
|
+
# message passes both. guard-bash.sh refuses the forms of that it can read; a program that types is past it.
|
|
464
|
+
case "${CLAUDE_CODE_SESSION_ATTENDED-}" in
|
|
465
|
+
0) no "this session has nobody in front of it (it was started with -p, or by a program), and an approval is what a person types into their own session." ;;
|
|
466
|
+
esac
|
|
467
|
+
_json_slice "$INPUT" transcript_path >/dev/null; _json_unescape "$_JS" >/dev/null; TP="${_JU//\\//}"
|
|
468
|
+
{ [ -n "$TP" ] && [ -s "$TP" ]; } || no "this is the first message of this session, or the session keeps no transcript (one started from inside another session keeps none). An approval answers a commit message the session has shown: have it shown, then approve."
|
|
469
|
+
|
|
423
470
|
# The session the user is writing in. guard-bash.sh accepts the record only from that one: a session started by a
|
|
424
471
|
# command can be handed any prompt.
|
|
425
472
|
_json_slice "$INPUT" session_id >/dev/null; SID="$_JS"
|
package/kit/settings.json
CHANGED
|
@@ -86,6 +86,28 @@
|
|
|
86
86
|
"timeout": 600
|
|
87
87
|
}
|
|
88
88
|
]
|
|
89
|
+
},
|
|
90
|
+
{
|
|
91
|
+
"matcher": "CronCreate|ScheduleWakeup|RemoteTrigger|mcp__.*([Tt]rigger|[Ss]chedul|[Cc]ron|[Ll]ater).*",
|
|
92
|
+
"hooks": [
|
|
93
|
+
{
|
|
94
|
+
"type": "command",
|
|
95
|
+
"shell": "bash",
|
|
96
|
+
"command": "cd \"$CLAUDE_PROJECT_DIR\" 2>/dev/null; bash .claude/hooks/guard-schedule.sh",
|
|
97
|
+
"timeout": 600
|
|
98
|
+
}
|
|
99
|
+
]
|
|
100
|
+
},
|
|
101
|
+
{
|
|
102
|
+
"matcher": "Agent|Task",
|
|
103
|
+
"hooks": [
|
|
104
|
+
{
|
|
105
|
+
"type": "command",
|
|
106
|
+
"shell": "bash",
|
|
107
|
+
"command": "cd \"$CLAUDE_PROJECT_DIR\" 2>/dev/null; bash .claude/hooks/guard-agent-model.sh",
|
|
108
|
+
"timeout": 600
|
|
109
|
+
}
|
|
110
|
+
]
|
|
89
111
|
}
|
|
90
112
|
],
|
|
91
113
|
"UserPromptSubmit": [
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: crew-approve
|
|
3
|
+
description: Approve the commit or push Claude just showed you (auto and dontAsk modes).
|
|
4
|
+
argument-hint: "[commit | push | commit+push]"
|
|
5
|
+
disable-model-invocation: true
|
|
6
|
+
metadata:
|
|
7
|
+
kind: command
|
|
8
|
+
---
|
|
9
|
+
# /crew-approve
|
|
10
|
+
Argument: $ARGUMENTS
|
|
11
|
+
|
|
12
|
+
The user typed this command. Typing it is the approval: `hooks/prompt-approval.sh` read the message itself and
|
|
13
|
+
recorded it, tied to what is staged, to `HEAD` and to this session, for 30 minutes or until the user's next
|
|
14
|
+
message. Nothing in this text records anything, and you cannot give this approval for the user.
|
|
15
|
+
|
|
16
|
+
Read the line Crewforth added to this turn and do what it says:
|
|
17
|
+
|
|
18
|
+
- **"approval recorded"**: run exactly what it names, each command alone in its call. A commit is
|
|
19
|
+
`git commit -m '…'` with the message you showed (single-quoted, or from a here-document with a quoted delimiter);
|
|
20
|
+
a push is `git push <remote> <branch>`. The review record of §4.6 is still required.
|
|
21
|
+
- **"approval NOT recorded"**: tell the user the reason it gives, and run nothing.
|
|
22
|
+
- **No such line**: the session is in a mode where the gate asks the user itself (`default`, `acceptEdits`), or
|
|
23
|
+
the argument was not `commit`, `push` or `commit+push`. Say which, and in the first case run the command and let
|
|
24
|
+
the gate's own prompt ask.
|
|
25
|
+
|
|
26
|
+
Force-push and the rest of §4.5 are not opened by this.
|
|
@@ -205,7 +205,7 @@ async function handle(req, res) {
|
|
|
205
205
|
const session = await findSession(id);
|
|
206
206
|
if (!session) continue; // a session on another machine: not read here, and not guessed
|
|
207
207
|
const u = await cachedUsage(session);
|
|
208
|
-
usage[id] = { durationMs: u.durationMs, workedMs: u.workedMs, tokens: u.tokens.
|
|
208
|
+
usage[id] = { durationMs: u.durationMs, workedMs: u.workedMs, startedAt: u.startedAt, tokens: u.tokens.fresh, cost: u.cost, unpriced: u.unpriced };
|
|
209
209
|
}
|
|
210
210
|
return sendJson(res, 200, { measured: true, usage });
|
|
211
211
|
}
|
|
@@ -19,7 +19,7 @@ import path from 'node:path';
|
|
|
19
19
|
|
|
20
20
|
import { readAll, contextFill } from './transcript.js';
|
|
21
21
|
import { agentMetaFiles } from './projects.js';
|
|
22
|
-
import { Usage,
|
|
22
|
+
import { Usage, withTimes } from './usage.js';
|
|
23
23
|
import { detailOf } from './permissions.js';
|
|
24
24
|
|
|
25
25
|
const SESSION_NODE = 'session';
|
|
@@ -60,6 +60,74 @@ function harvestCompletions(text, into, at = null) {
|
|
|
60
60
|
}
|
|
61
61
|
}
|
|
62
62
|
|
|
63
|
+
// The line the model gate (kit/hooks/guard-agent-model.sh) refuses a call with starts with this, whatever follows.
|
|
64
|
+
const MODEL_GATE = 'GUARD (agent model):';
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* What the model gate said when it refused a call, or null when the text is not its refusal.
|
|
68
|
+
* `floor` is the model it says the agent runs on "or above", when it says one; nothing is read into a sentence
|
|
69
|
+
* that does not say it.
|
|
70
|
+
*/
|
|
71
|
+
export function refusalOf(text) {
|
|
72
|
+
const at = typeof text === 'string' ? text.indexOf(MODEL_GATE) : -1;
|
|
73
|
+
if (at === -1) return null;
|
|
74
|
+
const line = text.slice(at).split('\n')[0].trim();
|
|
75
|
+
const floor = line.match(/ runs on ([a-z0-9.-]+) or above/);
|
|
76
|
+
return { text: line.slice(0, 400), floor: floor ? floor[1] : null };
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
// Which of two models is the larger, for the two questions asked of it: was a call repeated on a higher model,
|
|
80
|
+
// and which is "one model up". A name this does not know has no rank, and nothing is concluded from it.
|
|
81
|
+
const FAMILY = ['haiku', 'sonnet', 'opus', 'fable'];
|
|
82
|
+
export function modelRank(model) {
|
|
83
|
+
if (typeof model !== 'string') return null;
|
|
84
|
+
const i = FAMILY.findIndex((f) => model === f || model.includes(`-${f}-`) || model.startsWith(`${f}-`) || model.endsWith(`-${f}`));
|
|
85
|
+
return i === -1 ? null : i;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** How a report says it ended: the `confidence: high|low` line a crew agent closes with, or null. */
|
|
89
|
+
export function confidenceOf(text) {
|
|
90
|
+
const m = typeof text === 'string' ? text.trimEnd().match(/(?:^|\n)\W*confidence:\s*(high|low)\W*$/i) : null;
|
|
91
|
+
return m ? m[1].toLowerCase() : null;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Tie each agent call to what came before it, where the transcript says so.
|
|
96
|
+
*
|
|
97
|
+
* `firstTry` — the gate refused a call and the session made the same call again (same agent, same task): the
|
|
98
|
+
* call that ran carries what was asked first and why it was refused.
|
|
99
|
+
* Returns nothing; it writes onto the calls.
|
|
100
|
+
*/
|
|
101
|
+
export function linkRetries(calls) {
|
|
102
|
+
const list = [...calls.values()].sort((a, b) => a.order - b.order);
|
|
103
|
+
list.forEach((c, i) => {
|
|
104
|
+
if (!c.refused) return;
|
|
105
|
+
const again = list.slice(i + 1).find((n) => n.subagentType === c.subagentType && n.description === c.description);
|
|
106
|
+
if (!again) return;
|
|
107
|
+
// A call refused twice keeps the first thing that was asked.
|
|
108
|
+
again.firstTry = c.firstTry ?? { asked: c.model, floor: c.refused.floor, text: c.refused.text };
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* `escalatedFrom` / `escalatedTo` — an agent's report closed with `confidence: low`, and the next call to the same
|
|
114
|
+
* agent after it reported asked for a higher model: the same work, done again one model up. Read from the order
|
|
115
|
+
* of the calls, the two reports and the two models; where any of the three is missing nothing is linked.
|
|
116
|
+
*/
|
|
117
|
+
export function linkEscalations(nodes) {
|
|
118
|
+
const agents = nodes.filter((n) => n.kind === 'agent' && n.order != null).sort((a, b) => a.order - b.order);
|
|
119
|
+
agents.forEach((low, i) => {
|
|
120
|
+
if (low.confidence !== 'low' || low.reportedAt == null) return;
|
|
121
|
+
const from = modelRank(low.model ?? low.modelAsked);
|
|
122
|
+
if (from === null) return;
|
|
123
|
+
const again = agents.slice(i + 1).find((n) => n.agentType === low.agentType && n.calledAt != null && n.calledAt >= low.reportedAt);
|
|
124
|
+
const to = again ? modelRank(again.modelAsked ?? again.model) : null;
|
|
125
|
+
if (to === null || to <= from || again.escalatedFrom) return;
|
|
126
|
+
again.escalatedFrom = { id: low.id, from: low.model ?? low.modelAsked, to: again.modelAsked ?? again.model };
|
|
127
|
+
low.escalatedTo = again.id;
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
|
|
63
131
|
function scanMain(records) {
|
|
64
132
|
const out = {
|
|
65
133
|
cwd: null, gitBranch: null, version: null, model: null,
|
|
@@ -67,6 +135,7 @@ function scanMain(records) {
|
|
|
67
135
|
agentCalls: new Map(), // toolUseId -> { subagentType, description }
|
|
68
136
|
links: new Map(), // toolUseId -> { agentId, status, model }
|
|
69
137
|
completions: new Map(), // task id (an agent's, or a background command's) -> { status, at }
|
|
138
|
+
waves: new Map(), // assistant message id -> wave number, in the order the messages were written
|
|
70
139
|
commands: new Map(), // toolUseId -> the command a Bash call ran
|
|
71
140
|
background: new Map(), // background task id -> { toolUseId, startedAt }
|
|
72
141
|
owners: new Map(), // toolUseId -> owner node id
|
|
@@ -97,10 +166,20 @@ function scanMain(records) {
|
|
|
97
166
|
if (c?.type === 'tool_use' && c.id) {
|
|
98
167
|
out.owners.set(c.id, SESSION_NODE);
|
|
99
168
|
if (typeof c.input?.command === 'string') out.commands.set(c.id, c.input.command);
|
|
100
|
-
if (c.name === 'Agent') {
|
|
169
|
+
if (c.name === 'Agent' || c.name === 'Task') {
|
|
170
|
+
// The order the session called its agents in, and which calls were made together: the calls of one
|
|
171
|
+
// assistant message are one wave. A message is written as several records, one per block, and they
|
|
172
|
+
// share its id — so the wave is the message id, not the record.
|
|
173
|
+
const mid = r.message?.id ?? `record:${out.agentCalls.size}`;
|
|
174
|
+
if (!out.waves.has(mid)) out.waves.set(mid, out.waves.size + 1);
|
|
101
175
|
out.agentCalls.set(c.id, {
|
|
102
176
|
subagentType: c.input?.subagent_type ?? null,
|
|
103
177
|
description: c.input?.description ?? null,
|
|
178
|
+
order: out.agentCalls.size + 1,
|
|
179
|
+
wave: out.waves.get(mid),
|
|
180
|
+
calledAt: at,
|
|
181
|
+
// The model the call asked for: its own `model` field. Absent, the agent runs on its default.
|
|
182
|
+
model: typeof c.input?.model === 'string' ? c.input.model : null,
|
|
104
183
|
});
|
|
105
184
|
}
|
|
106
185
|
}
|
|
@@ -110,6 +189,12 @@ function scanMain(records) {
|
|
|
110
189
|
}
|
|
111
190
|
|
|
112
191
|
if (r?.type === 'user') {
|
|
192
|
+
// A call the model gate refused: the answer is an error that carries the gate's own line. No agent started.
|
|
193
|
+
for (const c of Array.isArray(r.message?.content) ? r.message.content : []) {
|
|
194
|
+
if (c?.type !== 'tool_result' || c.is_error !== true || !out.agentCalls.has(c.tool_use_id)) continue;
|
|
195
|
+
const said = refusalOf(textOf(typeof c.content === 'string' ? [{ type: 'text', text: c.content }] : c.content));
|
|
196
|
+
if (said) out.agentCalls.get(c.tool_use_id).refused = said;
|
|
197
|
+
}
|
|
113
198
|
out.userTurns += 1;
|
|
114
199
|
harvestCompletions(textOf(r.message?.content), out.completions, at);
|
|
115
200
|
}
|
|
@@ -119,7 +204,8 @@ function scanMain(records) {
|
|
|
119
204
|
if (tur && typeof tur === 'object' && tur.agentId) {
|
|
120
205
|
const id = r.message?.content?.find?.((c) => c?.type === 'tool_result')?.tool_use_id
|
|
121
206
|
?? r.toolUseID ?? null;
|
|
122
|
-
|
|
207
|
+
// `at` is when the answer was written: for a call the session waited on, that is when the agent reported.
|
|
208
|
+
const entry = { agentId: tur.agentId, status: tur.status ?? null, model: tur.resolvedModel ?? null, at };
|
|
123
209
|
if (id) out.links.set(id, entry);
|
|
124
210
|
else out.links.set(`agent:${tur.agentId}`, entry);
|
|
125
211
|
}
|
|
@@ -195,11 +281,13 @@ export function openCalls(records) {
|
|
|
195
281
|
}
|
|
196
282
|
|
|
197
283
|
/** Roll one agent's own transcript into the numbers its node shows. */
|
|
198
|
-
async function scanAgent(file, usage = null) {
|
|
284
|
+
async function scanAgent(file, usage = null, agentId = null) {
|
|
199
285
|
const { records } = await readAll(file);
|
|
200
|
-
usage?.add(records);
|
|
286
|
+
usage?.add(records, agentId ?? 'agents:unnamed');
|
|
201
287
|
const stats = {
|
|
202
288
|
tools: {}, toolCount: 0, lastTool: null, errors: 0,
|
|
289
|
+
model: null, // the model its own transcript names: what it actually ran on
|
|
290
|
+
lastText: null, // the end of the last thing it said, for the line a report closes with
|
|
203
291
|
tokens: null, startedAt: null, endedAt: null, turns: 0,
|
|
204
292
|
owns: [], // tool_use ids this agent emitted — how nesting is resolved
|
|
205
293
|
};
|
|
@@ -214,6 +302,10 @@ async function scanAgent(file, usage = null) {
|
|
|
214
302
|
}
|
|
215
303
|
if (r?.type === 'assistant') {
|
|
216
304
|
stats.turns += 1;
|
|
305
|
+
const mdl = r.message?.model;
|
|
306
|
+
if (typeof mdl === 'string' && mdl && mdl !== '<synthetic>') stats.model = mdl;
|
|
307
|
+
const said = textOf(r.message?.content);
|
|
308
|
+
if (said.trim()) stats.lastText = said.slice(-400);
|
|
217
309
|
const u = r.message?.usage;
|
|
218
310
|
if (u && u.cache_read_input_tokens != null) {
|
|
219
311
|
stats.tokens = (u.input_tokens ?? 0) + (u.cache_read_input_tokens ?? 0) + (u.cache_creation_input_tokens ?? 0);
|
|
@@ -264,7 +356,7 @@ async function readAgentDir(subagentsDir, usage = null) {
|
|
|
264
356
|
// `stats` needs the transcript read; a missing file is a normal state (the
|
|
265
357
|
// agent has not written yet), so absence is a null rather than a throw.
|
|
266
358
|
let stats = null;
|
|
267
|
-
if (mtime !== null) stats = await scanAgent(jsonl, usage);
|
|
359
|
+
if (mtime !== null) stats = await scanAgent(jsonl, usage, agentId);
|
|
268
360
|
|
|
269
361
|
out.push({
|
|
270
362
|
agentId,
|
|
@@ -300,6 +392,7 @@ export async function buildGraph(session, { staleMs = STALE_MS } = {}) {
|
|
|
300
392
|
const { records, malformed } = await readAll(session.file);
|
|
301
393
|
const main = scanMain(records);
|
|
302
394
|
// What the session spent, its agents' tokens included and each response counted once.
|
|
395
|
+
linkRetries(main.agentCalls);
|
|
303
396
|
const usage = new Usage().add(records);
|
|
304
397
|
const agents = await readAgentDir(session.subagentsDir, usage);
|
|
305
398
|
const now = Date.now();
|
|
@@ -314,6 +407,8 @@ export async function buildGraph(session, { staleMs = STALE_MS } = {}) {
|
|
|
314
407
|
// toolUseId -> agentId, from the returning result record.
|
|
315
408
|
const byToolUse = new Map();
|
|
316
409
|
for (const [k, v] of main.links) if (!k.startsWith('agent:')) byToolUse.set(k, v);
|
|
410
|
+
const callOf = new Map();
|
|
411
|
+
for (const [k, v] of byToolUse) callOf.set(v.agentId, k);
|
|
317
412
|
|
|
318
413
|
const nodes = [{
|
|
319
414
|
id: SESSION_NODE,
|
|
@@ -329,7 +424,7 @@ export async function buildGraph(session, { staleMs = STALE_MS } = {}) {
|
|
|
329
424
|
tokens: contextFill(records),
|
|
330
425
|
startedAt: main.startedAt,
|
|
331
426
|
endedAt: main.updatedAt,
|
|
332
|
-
usage:
|
|
427
|
+
usage: withTimes(usage, records),
|
|
333
428
|
// Commands sent to the background that no notice has ended, oldest first. A session that is over has none
|
|
334
429
|
// running whatever this says: the page shows them only while the session is live.
|
|
335
430
|
backgroundCommands: [...main.background]
|
|
@@ -345,8 +440,10 @@ export async function buildGraph(session, { staleMs = STALE_MS } = {}) {
|
|
|
345
440
|
const edges = [];
|
|
346
441
|
|
|
347
442
|
for (const a of agents) {
|
|
348
|
-
|
|
349
|
-
const
|
|
443
|
+
// The call that started it: named by its own meta file, or found from the answer that named the agent.
|
|
444
|
+
const callId = a.toolUseId ?? callOf.get(a.agentId) ?? null;
|
|
445
|
+
const call = callId ? main.agentCalls.get(callId) : null;
|
|
446
|
+
const link = callId ? byToolUse.get(callId) : null;
|
|
350
447
|
const notice = main.completions.get(a.agentId) ?? null;
|
|
351
448
|
const completion = noticeStands(notice, a.stats?.endedAt ?? null) ? notice.status : null;
|
|
352
449
|
|
|
@@ -380,7 +477,16 @@ export async function buildGraph(session, { staleMs = STALE_MS } = {}) {
|
|
|
380
477
|
spawnDepth: a.spawnDepth,
|
|
381
478
|
workflow: a.workflow,
|
|
382
479
|
toolUseId: a.toolUseId,
|
|
383
|
-
model:
|
|
480
|
+
// The model it ran on: what its own transcript names, and failing that what the launch answer resolved.
|
|
481
|
+
model: a.stats?.model ?? link?.model ?? null,
|
|
482
|
+
// The model the call asked for; null when the call named none and the agent ran on its default.
|
|
483
|
+
modelAsked: call?.model ?? null,
|
|
484
|
+
// Set when the model gate refused this call the first time it was made: what was asked then, and why not.
|
|
485
|
+
firstTry: call?.firstTry ?? null,
|
|
486
|
+
// `confidence: high|low`, the line a crew agent's report closes with; null when it has none.
|
|
487
|
+
confidence: confidenceOf(a.stats?.lastText ?? null),
|
|
488
|
+
// Its own share of the session's tokens, and what they cost at list price.
|
|
489
|
+
usage: usage.of(a.agentId),
|
|
384
490
|
launchStatus: link?.status ?? null,
|
|
385
491
|
// Started in the background, and its transcript stops in the middle of a turn. Such an agent can be quiet
|
|
386
492
|
// for a long time inside one tool call; whether it is still working is the live session's to say, so the
|
|
@@ -398,11 +504,22 @@ export async function buildGraph(session, { staleMs = STALE_MS } = {}) {
|
|
|
398
504
|
endedAt: a.stats?.endedAt ?? null,
|
|
399
505
|
durationMs: a.stats?.durationMs ?? null,
|
|
400
506
|
updatedAt: a.mtime,
|
|
507
|
+
// Where it stands among the agents the session called: its place in the order of the calls, the wave it
|
|
508
|
+
// was called in, and when. Null for an agent the session's own transcript did not call (one a workflow
|
|
509
|
+
// started): nothing is known about its place from here.
|
|
510
|
+
order: call?.order ?? null,
|
|
511
|
+
wave: call?.wave ?? null,
|
|
512
|
+
calledAt: call?.calledAt ?? null,
|
|
513
|
+
// When it handed its result back: the answer to a call the session waited on, or the notice that ended a
|
|
514
|
+
// background agent. Null while it has not, and when no time was recorded.
|
|
515
|
+
reportedAt: link?.status === 'completed' ? (link.at ?? null) : (completion ? (notice.at ?? null) : null),
|
|
401
516
|
});
|
|
402
517
|
|
|
403
518
|
edges.push({ id: `${parentId}->${a.agentId}`, source: parentId, target: a.agentId, kind: 'spawn' });
|
|
404
519
|
}
|
|
405
520
|
|
|
521
|
+
linkEscalations(nodes);
|
|
522
|
+
|
|
406
523
|
// A workflow run is a real container, not a rendering trick: its agents were
|
|
407
524
|
// spawned by one orchestration script, not by the session directly. Hanging
|
|
408
525
|
// 243 of them straight off the root produced a graph nobody could read, and
|
|
@@ -36,55 +36,115 @@ export function usageOfRecord(r) {
|
|
|
36
36
|
};
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
-
|
|
39
|
+
const KINDS = ['input', 'output', 'cacheWrite', 'cacheRead'];
|
|
40
|
+
|
|
41
|
+
/** One running sum: tokens by kind, what they cost, and which models had no price. */
|
|
42
|
+
class Tally {
|
|
40
43
|
constructor() {
|
|
41
|
-
this.seen = new Set();
|
|
42
44
|
this.tokens = { input: 0, output: 0, cacheWrite: 0, cacheRead: 0 };
|
|
43
45
|
this.usd = 0;
|
|
44
46
|
this.responses = 0;
|
|
45
|
-
this.priced = new Set();
|
|
46
47
|
this.unpriced = new Set();
|
|
47
48
|
}
|
|
48
49
|
|
|
49
|
-
|
|
50
|
-
|
|
50
|
+
take(u, usd) {
|
|
51
|
+
this.responses += 1;
|
|
52
|
+
this.tokens.input += u.input;
|
|
53
|
+
this.tokens.output += u.output;
|
|
54
|
+
this.tokens.cacheWrite += u.cacheWrite5m + u.cacheWrite1h;
|
|
55
|
+
this.tokens.cacheRead += u.cacheRead;
|
|
56
|
+
if (usd === null) this.unpriced.add(u.model ?? 'unknown model');
|
|
57
|
+
else this.usd += usd;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* `fresh` is what was sent and written for the first time: input, output and cache writes. `cacheRead` is the
|
|
62
|
+
* same context read back from the cache on every later response, which is why it dwarfs the rest in a long
|
|
63
|
+
* session and is kept out of the headline. `cost` is null when any of the tokens came from a model the price
|
|
64
|
+
* table has no row for — a sum that leaves some of them out would read as the whole.
|
|
65
|
+
*/
|
|
66
|
+
result() {
|
|
67
|
+
const t = this.tokens;
|
|
68
|
+
return {
|
|
69
|
+
tokens: { ...t, fresh: t.input + t.output + t.cacheWrite, total: KINDS.reduce((a, k) => a + t[k], 0) },
|
|
70
|
+
responses: this.responses,
|
|
71
|
+
cost: this.unpriced.size ? null : this.usd,
|
|
72
|
+
unpriced: [...this.unpriced].sort(),
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export class Usage {
|
|
78
|
+
constructor() {
|
|
79
|
+
this.seen = new Set();
|
|
80
|
+
this.all = new Tally();
|
|
81
|
+
this.who = new Map(); // 'session', or an agent's id -> Tally
|
|
82
|
+
this.model = new Map(); // model id -> Tally
|
|
83
|
+
this.first = null; // the earliest and the latest record of any file added
|
|
84
|
+
this.last = null;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
#tally(map, key) {
|
|
88
|
+
if (!map.has(key)) map.set(key, new Tally());
|
|
89
|
+
return map.get(key);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Add a transcript's records. A response already counted, from this file or another, is not counted again.
|
|
94
|
+
* @param who 'session' for the session's own transcript, or the id of the agent whose transcript this is.
|
|
95
|
+
* A record the session's transcript marks as an agent's (`isSidechain`) is counted as an agent's.
|
|
96
|
+
*/
|
|
97
|
+
add(records, who = 'session') {
|
|
51
98
|
for (const r of records ?? []) {
|
|
99
|
+
const t = r?.timestamp ? Date.parse(r.timestamp) : NaN;
|
|
100
|
+
if (Number.isFinite(t)) {
|
|
101
|
+
if (this.first === null || t < this.first) this.first = t;
|
|
102
|
+
if (this.last === null || t > this.last) this.last = t;
|
|
103
|
+
}
|
|
52
104
|
const u = usageOfRecord(r);
|
|
53
105
|
if (!u) continue;
|
|
54
106
|
if (u.id) { if (this.seen.has(u.id)) continue; this.seen.add(u.id); }
|
|
55
|
-
|
|
56
|
-
if (!total) continue; // a record the harness made up for itself carries no tokens
|
|
57
|
-
this.responses += 1;
|
|
58
|
-
this.tokens.input += u.input;
|
|
59
|
-
this.tokens.output += u.output;
|
|
60
|
-
this.tokens.cacheWrite += u.cacheWrite5m + u.cacheWrite1h;
|
|
61
|
-
this.tokens.cacheRead += u.cacheRead;
|
|
107
|
+
if (!(u.input + u.output + u.cacheWrite5m + u.cacheWrite1h + u.cacheRead)) continue; // a record the harness made up carries no tokens
|
|
62
108
|
const usd = costOf(u);
|
|
63
|
-
|
|
64
|
-
|
|
109
|
+
this.all.take(u, usd);
|
|
110
|
+
this.#tally(this.who, who === 'session' && r.isSidechain === true ? 'agents:inline' : who).take(u, usd);
|
|
111
|
+
this.#tally(this.model, u.model ?? 'unknown model').take(u, usd);
|
|
65
112
|
}
|
|
66
113
|
return this;
|
|
67
114
|
}
|
|
68
115
|
|
|
116
|
+
/** One agent's own share, or null when its transcript carried no usage. */
|
|
117
|
+
of(who) { return this.who.get(who)?.result() ?? null; }
|
|
118
|
+
|
|
69
119
|
/**
|
|
70
|
-
* @returns
|
|
71
|
-
*
|
|
120
|
+
* @returns the whole session's tally, and the same split two ways: `parts.session` is what the session's own
|
|
121
|
+
* transcript spent and `parts.agents` everything its agents did; `byModel` is one tally per model.
|
|
72
122
|
*/
|
|
73
123
|
result() {
|
|
74
|
-
const
|
|
124
|
+
const agents = new Tally();
|
|
125
|
+
for (const [who, t] of this.who) {
|
|
126
|
+
if (who === 'session') continue;
|
|
127
|
+
for (const k of KINDS) agents.tokens[k] += t.tokens[k];
|
|
128
|
+
agents.usd += t.usd;
|
|
129
|
+
agents.responses += t.responses;
|
|
130
|
+
for (const m of t.unpriced) agents.unpriced.add(m);
|
|
131
|
+
}
|
|
132
|
+
const all = this.all.result();
|
|
75
133
|
return {
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
134
|
+
...all,
|
|
135
|
+
models: [...this.model.keys()].filter((m) => !all.unpriced.includes(m)).sort(),
|
|
136
|
+
parts: { session: (this.who.get('session') ?? new Tally()).result(), agents: agents.result() },
|
|
137
|
+
byModel: Object.fromEntries([...this.model].map(([m, t]) => [m, t.result()]).sort((a, b) => b[1].tokens.total - a[1].tokens.total)),
|
|
138
|
+
startedAt: this.first,
|
|
139
|
+
lastAt: this.last,
|
|
81
140
|
priceSource: SOURCE,
|
|
82
141
|
};
|
|
83
142
|
}
|
|
84
143
|
}
|
|
85
144
|
|
|
86
145
|
/**
|
|
87
|
-
* How long a session
|
|
146
|
+
* How long a session's own transcript says it took. `durationMs` is from its first record to its last, pauses
|
|
147
|
+
* included; with its agents' transcripts counted too, that span is `Usage`'s `startedAt` to `lastAt`.
|
|
88
148
|
* `workedMs` is the turns Claude Code timed itself (`system` / `turn_duration` records), added up; zero when it
|
|
89
149
|
* recorded none, and a turn that is still running is not in it.
|
|
90
150
|
*/
|
|
@@ -107,6 +167,16 @@ export function timesOf(records) {
|
|
|
107
167
|
return { durationMs: first !== null && last !== null ? last - first : null, workedMs, turns };
|
|
108
168
|
}
|
|
109
169
|
|
|
170
|
+
/**
|
|
171
|
+
* The usage with the times beside it. The span runs to the last record of ANY of the session's files: an agent
|
|
172
|
+
* can be writing long after the session's own transcript last moved.
|
|
173
|
+
*/
|
|
174
|
+
export function withTimes(usage, mainRecords) {
|
|
175
|
+
const u = usage.result();
|
|
176
|
+
const t = timesOf(mainRecords);
|
|
177
|
+
return { ...u, workedMs: t.workedMs, turns: t.turns, durationMs: u.startedAt !== null && u.lastAt !== null ? u.lastAt - u.startedAt : null };
|
|
178
|
+
}
|
|
179
|
+
|
|
110
180
|
/** A session's usage with its agents', and how long it took. */
|
|
111
181
|
export async function sessionUsage(session) {
|
|
112
182
|
const usage = new Usage();
|
|
@@ -114,9 +184,10 @@ export async function sessionUsage(session) {
|
|
|
114
184
|
usage.add(records);
|
|
115
185
|
for (const meta of await agentMetaFiles(session.subagentsDir)) {
|
|
116
186
|
const jsonl = meta.replace(/\.meta\.json$/, '.jsonl');
|
|
117
|
-
|
|
187
|
+
const agentId = path.basename(jsonl).slice('agent-'.length, -'.jsonl'.length);
|
|
188
|
+
try { usage.add((await readAll(jsonl)).records, agentId); } catch { /* an agent that has not written yet */ }
|
|
118
189
|
}
|
|
119
|
-
return
|
|
190
|
+
return withTimes(usage, records);
|
|
120
191
|
}
|
|
121
192
|
|
|
122
193
|
// One answer per session, kept until one of its files changes. The key is what the stream's own signature
|