@monster-orange/den 0.4.7 → 0.4.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +94 -0
- package/README.md +7 -7
- package/docs/distributions.md +120 -1
- package/package.json +1 -1
- package/schemas/den.plugin.v1.json +3 -2
- package/src/shared/agents/den-claude.ts +19 -0
- package/src/shared/agents/personas.ts +11 -1
- package/src/shared/agents/records.ts +114 -0
- package/src/shared/agents/runtime.ts +12 -0
- package/src/shared/den-config.ts +30 -0
- package/src/shared/plugins/manifest.ts +3 -0
- package/src/shared/usage/paths.ts +16 -0
- package/src/shared/usage/report.ts +283 -0
- package/src/shared/usage/sources.ts +127 -0
- package/src/shared/usage/transcript.ts +262 -0
- package/src/ui/cli/agent-ops.ts +488 -50
- package/src/ui/cli/agents-command.ts +120 -14
- package/src/ui/cli/index.ts +13 -1
- package/src/ui/cli/usage-command.ts +256 -0
- package/src/ui/mcp/api.ts +10 -1
- package/src/ui/mcp/server.ts +34 -0
- package/src/ui/mcp/tools.ts +50 -0
- package/src/ui/server/agent-resume-grant.ts +53 -0
- package/src/ui/server/agent-routes.ts +230 -9
- package/src/ui/server/audit.ts +68 -1
- package/src/ui/server/index.ts +16 -5
- package/src/ui/server/public/agents-page.js +333 -11
- package/src/ui/server/public/agents-section.js +17 -3
- package/src/ui/server/public/agents.html +44 -0
- package/src/ui/server/public/i18n/es.js +28 -0
- package/src/ui/server/public/i18n/ja.js +25 -0
- package/src/ui/server/public/shell.js +66 -0
- package/src/ui/server/request-guard.ts +21 -1
- package/src/ui/server/route-classes.ts +23 -0
- package/src/ui/server/session-routes.ts +52 -4
- package/src/ui/server/sudo-store.ts +36 -7
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The usage report (den#6, ported from the agent-usage report): per agent,
|
|
3
|
+
* its tokens and its outcome, joined on the agent record's resumeToken; and
|
|
4
|
+
* per week, the numbers to move, diffed against the last report.
|
|
5
|
+
*
|
|
6
|
+
* An agent session is "attributed" when its transcripts were found and
|
|
7
|
+
* counted (tokens) and its outcome is known (at least its wall-clock). The
|
|
8
|
+
* share of attributed sessions is the scorecard metric: before this, no
|
|
9
|
+
* agent's tokens or outcome were joined to it (~0).
|
|
10
|
+
*
|
|
11
|
+
* Everything is read and written on this machine; nothing is sent.
|
|
12
|
+
*/
|
|
13
|
+
import type {
|
|
14
|
+
AgentOutcome,
|
|
15
|
+
AgentRecord,
|
|
16
|
+
AgentUsage,
|
|
17
|
+
} from "../agents/records.js";
|
|
18
|
+
import { sourceFor, type UsageSource } from "./sources.js";
|
|
19
|
+
import type { TranscriptStats } from "./transcript.js";
|
|
20
|
+
|
|
21
|
+
export interface AgentUsageRow {
|
|
22
|
+
agent: string;
|
|
23
|
+
workspace: string;
|
|
24
|
+
persona: string;
|
|
25
|
+
runtime: string;
|
|
26
|
+
/** No source reads its runtime, or it has no resume token. */
|
|
27
|
+
why?: string;
|
|
28
|
+
/** How many transcripts were found (its own and its subagents'). */
|
|
29
|
+
transcripts: number;
|
|
30
|
+
usage?: AgentUsage;
|
|
31
|
+
outcome?: AgentOutcome;
|
|
32
|
+
attributed: boolean;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** The sum of transcripts' counts, as the record keeps them. */
|
|
36
|
+
export function sumUsage(stats: readonly TranscriptStats[]): AgentUsage {
|
|
37
|
+
const u: AgentUsage = {
|
|
38
|
+
calls: 0,
|
|
39
|
+
input: 0,
|
|
40
|
+
output: 0,
|
|
41
|
+
cacheRead: 0,
|
|
42
|
+
cacheWrite: 0,
|
|
43
|
+
compactions: 0,
|
|
44
|
+
};
|
|
45
|
+
for (const s of stats) {
|
|
46
|
+
u.calls += s.calls;
|
|
47
|
+
u.input += s.tokens.input;
|
|
48
|
+
u.output += s.tokens.output;
|
|
49
|
+
u.cacheRead += s.tokens.cacheRead;
|
|
50
|
+
u.cacheWrite += s.tokens.cacheWrite;
|
|
51
|
+
u.compactions += s.compactions;
|
|
52
|
+
}
|
|
53
|
+
return u;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** An agent's outcome from its transcripts and its record. */
|
|
57
|
+
export function outcomeOf(
|
|
58
|
+
rec: AgentRecord,
|
|
59
|
+
stats: readonly TranscriptStats[],
|
|
60
|
+
): AgentOutcome {
|
|
61
|
+
const own = stats.filter((s) => !s.subagent);
|
|
62
|
+
const all = own.length ? own : stats;
|
|
63
|
+
const firsts = all
|
|
64
|
+
.map((s) => s.first)
|
|
65
|
+
.filter((x): x is number => x !== undefined);
|
|
66
|
+
const lasts = all
|
|
67
|
+
.map((s) => s.last)
|
|
68
|
+
.filter((x): x is number => x !== undefined);
|
|
69
|
+
const out: AgentOutcome = {};
|
|
70
|
+
if (firsts.length && lasts.length)
|
|
71
|
+
out.wallClockMs = Math.max(...lasts) - Math.min(...firsts);
|
|
72
|
+
else {
|
|
73
|
+
const start = Date.parse(rec.startedAt);
|
|
74
|
+
const end = Date.parse(
|
|
75
|
+
rec.stoppedAt ?? rec.endedAt ?? rec.interruptedAt ?? "",
|
|
76
|
+
);
|
|
77
|
+
if (Number.isFinite(start) && Number.isFinite(end) && end >= start)
|
|
78
|
+
out.wallClockMs = end - start;
|
|
79
|
+
}
|
|
80
|
+
const pr = [...all].reverse().find((s) => s.prOpened)?.prOpened;
|
|
81
|
+
if (pr) out.prOpened = pr;
|
|
82
|
+
if (all.some((s) => s.prMerged)) out.prMerged = true;
|
|
83
|
+
// The last test run: of the transcript that ended last and ran tests.
|
|
84
|
+
const tested = all
|
|
85
|
+
.filter((s) => s.testsGreen !== undefined)
|
|
86
|
+
.sort((a, b) => (a.last ?? 0) - (b.last ?? 0));
|
|
87
|
+
if (tested.length) out.testsGreen = tested[tested.length - 1].testsGreen;
|
|
88
|
+
return out;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Each record's usage and outcome, from its runtime's transcripts (by its
|
|
93
|
+
* resumeToken). `source` picks the reader (default: the one for its runtime).
|
|
94
|
+
*/
|
|
95
|
+
export async function agentUsage(
|
|
96
|
+
records: readonly AgentRecord[],
|
|
97
|
+
opts: {
|
|
98
|
+
env?: NodeJS.ProcessEnv;
|
|
99
|
+
source?: (runtime: string) => UsageSource | undefined;
|
|
100
|
+
} = {},
|
|
101
|
+
): Promise<AgentUsageRow[]> {
|
|
102
|
+
const pick = opts.source ?? sourceFor;
|
|
103
|
+
const rows: AgentUsageRow[] = [];
|
|
104
|
+
for (const rec of records) {
|
|
105
|
+
const base = {
|
|
106
|
+
agent: rec.id,
|
|
107
|
+
workspace: rec.workspace,
|
|
108
|
+
persona: rec.persona,
|
|
109
|
+
runtime: rec.runtime,
|
|
110
|
+
};
|
|
111
|
+
const src = pick(rec.runtime);
|
|
112
|
+
if (!src) {
|
|
113
|
+
rows.push({
|
|
114
|
+
...base,
|
|
115
|
+
why: `no usage source reads ${rec.runtime}`,
|
|
116
|
+
transcripts: 0,
|
|
117
|
+
attributed: false,
|
|
118
|
+
});
|
|
119
|
+
continue;
|
|
120
|
+
}
|
|
121
|
+
if (!rec.resumeToken) {
|
|
122
|
+
rows.push({
|
|
123
|
+
...base,
|
|
124
|
+
why: "no conversation id to join on",
|
|
125
|
+
transcripts: 0,
|
|
126
|
+
attributed: false,
|
|
127
|
+
});
|
|
128
|
+
continue;
|
|
129
|
+
}
|
|
130
|
+
const files = src.agentFiles(rec.resumeToken, opts.env);
|
|
131
|
+
const stats = await Promise.all(files.map((f) => src.mine(f)));
|
|
132
|
+
if (!stats.length) {
|
|
133
|
+
rows.push({
|
|
134
|
+
...base,
|
|
135
|
+
why: "no transcript found",
|
|
136
|
+
transcripts: 0,
|
|
137
|
+
attributed: false,
|
|
138
|
+
});
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
141
|
+
const usage = sumUsage(stats);
|
|
142
|
+
const outcome = outcomeOf(rec, stats);
|
|
143
|
+
rows.push({
|
|
144
|
+
...base,
|
|
145
|
+
transcripts: files.length,
|
|
146
|
+
usage,
|
|
147
|
+
outcome,
|
|
148
|
+
attributed: usage.calls > 0 && outcome.wallClockMs !== undefined,
|
|
149
|
+
});
|
|
150
|
+
}
|
|
151
|
+
return rows;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/** The share of agent sessions attributed, in percent (0 with none). */
|
|
155
|
+
export function attributedPct(rows: readonly AgentUsageRow[]): number {
|
|
156
|
+
if (!rows.length) return 0;
|
|
157
|
+
return Math.round(
|
|
158
|
+
(100 * rows.filter((r) => r.attributed).length) / rows.length,
|
|
159
|
+
);
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/** The week's numbers (lower is better for all but calls, output, prompts). */
|
|
163
|
+
export interface UsageMetrics {
|
|
164
|
+
calls: number;
|
|
165
|
+
cache_read_B: number;
|
|
166
|
+
output_M: number;
|
|
167
|
+
avg_context_k_main: number;
|
|
168
|
+
top15_share_pct: number;
|
|
169
|
+
reread_share_pct: number;
|
|
170
|
+
compactions: number;
|
|
171
|
+
polls: number;
|
|
172
|
+
denials: number;
|
|
173
|
+
tool_error_pct: number;
|
|
174
|
+
prompts: number;
|
|
175
|
+
correction_pct: number;
|
|
176
|
+
subagent_opus_pct: number;
|
|
177
|
+
/** den#6: the share of agent sessions with attributed tokens and outcome. */
|
|
178
|
+
attributed_pct?: number;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
const round = (n: number, d = 0) => {
|
|
182
|
+
const f = 10 ** d;
|
|
183
|
+
return Math.round(n * f) / f;
|
|
184
|
+
};
|
|
185
|
+
|
|
186
|
+
export function weeklyMetrics(
|
|
187
|
+
stats: readonly TranscriptStats[],
|
|
188
|
+
agents?: readonly AgentUsageRow[],
|
|
189
|
+
): UsageMetrics {
|
|
190
|
+
const sum = (
|
|
191
|
+
xs: readonly TranscriptStats[],
|
|
192
|
+
f: (s: TranscriptStats) => number,
|
|
193
|
+
) => xs.reduce((a, s) => a + f(s), 0);
|
|
194
|
+
const calls = sum(stats, (s) => s.calls);
|
|
195
|
+
const reads = sum(stats, (s) => s.tokens.cacheRead);
|
|
196
|
+
const writes = sum(stats, (s) => s.tokens.cacheWrite);
|
|
197
|
+
const out = sum(stats, (s) => s.tokens.output);
|
|
198
|
+
const main = stats.filter((s) => !s.subagent);
|
|
199
|
+
const mainCalls = sum(main, (s) => s.calls);
|
|
200
|
+
const top15 = [...main]
|
|
201
|
+
.sort((a, b) => b.tokens.cacheRead - a.tokens.cacheRead)
|
|
202
|
+
.slice(0, 15);
|
|
203
|
+
const tools = sum(stats, (s) =>
|
|
204
|
+
Object.values(s.tools).reduce((a, b) => a + b, 0),
|
|
205
|
+
);
|
|
206
|
+
const errors = sum(stats, (s) =>
|
|
207
|
+
Object.values(s.toolErrors).reduce((a, b) => a + b, 0),
|
|
208
|
+
);
|
|
209
|
+
let subTotal = 0;
|
|
210
|
+
let subOpus = 0;
|
|
211
|
+
for (const s of stats)
|
|
212
|
+
if (s.subagent)
|
|
213
|
+
for (const [m, n] of Object.entries(s.models)) {
|
|
214
|
+
subTotal += n;
|
|
215
|
+
if (m.includes("opus")) subOpus += n;
|
|
216
|
+
}
|
|
217
|
+
const prompts = sum(stats, (s) => s.prompts);
|
|
218
|
+
const m: UsageMetrics = {
|
|
219
|
+
calls,
|
|
220
|
+
cache_read_B: round(reads / 1e9, 2),
|
|
221
|
+
output_M: round(out / 1e6, 1),
|
|
222
|
+
avg_context_k_main: round(
|
|
223
|
+
sum(main, (s) => s.tokens.cacheRead) / Math.max(1, mainCalls) / 1e3,
|
|
224
|
+
),
|
|
225
|
+
top15_share_pct: round(
|
|
226
|
+
(100 * sum(top15, (s) => s.tokens.cacheRead)) / Math.max(1, reads),
|
|
227
|
+
),
|
|
228
|
+
// The usual price ratios: a cache read a tenth of input, a write 1.25x, output 5x.
|
|
229
|
+
reread_share_pct: round(
|
|
230
|
+
(100 * reads * 0.1) / Math.max(1, reads * 0.1 + writes * 1.25 + out * 5),
|
|
231
|
+
),
|
|
232
|
+
compactions: sum(stats, (s) => s.compactions),
|
|
233
|
+
polls: sum(stats, (s) => s.polls),
|
|
234
|
+
denials: sum(stats, (s) => s.denials),
|
|
235
|
+
tool_error_pct: round((100 * errors) / Math.max(1, tools), 1),
|
|
236
|
+
prompts,
|
|
237
|
+
correction_pct: round(
|
|
238
|
+
(100 * sum(stats, (s) => s.corrections)) / Math.max(1, prompts),
|
|
239
|
+
1,
|
|
240
|
+
),
|
|
241
|
+
subagent_opus_pct: round((100 * subOpus) / Math.max(1, subTotal)),
|
|
242
|
+
};
|
|
243
|
+
if (agents) m.attributed_pct = attributedPct(agents);
|
|
244
|
+
return m;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
/** Higher is better for these; context only for calls, output and prompts. */
|
|
248
|
+
const BETTER_HIGH = new Set(["attributed_pct"]);
|
|
249
|
+
const CONTEXT = new Set(["calls", "output_M", "prompts"]);
|
|
250
|
+
|
|
251
|
+
/** The report as markdown: this week, last week, and the change. */
|
|
252
|
+
export function renderReport(
|
|
253
|
+
m: UsageMetrics,
|
|
254
|
+
date: string,
|
|
255
|
+
prev?: { date: string; metrics: Partial<UsageMetrics> },
|
|
256
|
+
): string {
|
|
257
|
+
const lines = [`# Agent usage, week to ${date}`, ""];
|
|
258
|
+
if (prev) lines.push(`Compared with ${prev.date}.`, "");
|
|
259
|
+
lines.push(
|
|
260
|
+
"| Metric | This week | Last | Change |",
|
|
261
|
+
"| --- | --- | --- | --- |",
|
|
262
|
+
);
|
|
263
|
+
for (const [k, v] of Object.entries(m) as [keyof UsageMetrics, number][]) {
|
|
264
|
+
const p = prev?.metrics[k];
|
|
265
|
+
let change = "";
|
|
266
|
+
if (typeof p === "number" && p) {
|
|
267
|
+
const d = round(v - p, 2);
|
|
268
|
+
const mark =
|
|
269
|
+
!d || CONTEXT.has(k)
|
|
270
|
+
? ""
|
|
271
|
+
: (BETTER_HIGH.has(k) ? d > 0 : d < 0)
|
|
272
|
+
? "better"
|
|
273
|
+
: "worse";
|
|
274
|
+
change = `${d > 0 ? "+" : ""}${d} ${mark}`.trim();
|
|
275
|
+
}
|
|
276
|
+
lines.push(`| ${k} | ${v} | ${p ?? ""} | ${change} |`);
|
|
277
|
+
}
|
|
278
|
+
lines.push(
|
|
279
|
+
"",
|
|
280
|
+
"Read from this machine's transcripts; nothing was sent anywhere.",
|
|
281
|
+
);
|
|
282
|
+
return lines.join("\n") + "\n";
|
|
283
|
+
}
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Usage sources (den#6): plugin kind `usage`, strategy kind `usage`. A
|
|
3
|
+
* source reads one runtime's own transcripts on this machine: which files
|
|
4
|
+
* belong to an agent (by the agent record's resumeToken, the runtime's
|
|
5
|
+
* conversation id), and every file touched in a window (the weekly
|
|
6
|
+
* report). Den's built-in reads Claude Code's (den-claude); a plugin adds
|
|
7
|
+
* one for another runtime the same way, by registering here.
|
|
8
|
+
*
|
|
9
|
+
* A source reads locally and never sends what it reads anywhere.
|
|
10
|
+
*/
|
|
11
|
+
import * as fs from "fs";
|
|
12
|
+
import * as path from "path";
|
|
13
|
+
import {
|
|
14
|
+
getStrategy,
|
|
15
|
+
registerStrategy,
|
|
16
|
+
strategyIds,
|
|
17
|
+
type StrategyEntry,
|
|
18
|
+
} from "../strategies/registry.js";
|
|
19
|
+
import { mineTranscript, type TranscriptStats } from "./transcript.js";
|
|
20
|
+
|
|
21
|
+
export const USAGE_KIND = "usage";
|
|
22
|
+
|
|
23
|
+
export interface UsageSource {
|
|
24
|
+
id: string;
|
|
25
|
+
/** The runtimes whose agents it reads (den-claude). */
|
|
26
|
+
runtimes: readonly string[];
|
|
27
|
+
/** Where it reads (for doctor and the report's header). */
|
|
28
|
+
root(env?: NodeJS.ProcessEnv): string;
|
|
29
|
+
/** An agent's transcripts: its conversation's file, then its subagents'. */
|
|
30
|
+
agentFiles(resumeToken: string, env?: NodeJS.ProcessEnv): string[];
|
|
31
|
+
/** Every transcript changed since `since` (ms since the epoch). */
|
|
32
|
+
files(since: number, env?: NodeJS.ProcessEnv): string[];
|
|
33
|
+
mine(file: string, opts?: { since?: number }): Promise<TranscriptStats>;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** A conversation id as Claude Code names its files: never a path. */
|
|
37
|
+
const TOKEN_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/;
|
|
38
|
+
|
|
39
|
+
import { claudeProjectsDir } from "./paths.js";
|
|
40
|
+
export { claudeProjectsDir };
|
|
41
|
+
|
|
42
|
+
function dirs(p: string): string[] {
|
|
43
|
+
try {
|
|
44
|
+
return fs
|
|
45
|
+
.readdirSync(p, { withFileTypes: true })
|
|
46
|
+
.filter((e) => e.isDirectory())
|
|
47
|
+
.map((e) => path.join(p, e.name));
|
|
48
|
+
} catch {
|
|
49
|
+
return [];
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function jsonl(p: string): string[] {
|
|
54
|
+
try {
|
|
55
|
+
return fs
|
|
56
|
+
.readdirSync(p, { withFileTypes: true })
|
|
57
|
+
.filter((e) => e.isFile() && e.name.endsWith(".jsonl"))
|
|
58
|
+
.map((e) => path.join(p, e.name));
|
|
59
|
+
} catch {
|
|
60
|
+
return [];
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** Claude Code: <projects>/<cwd slug>/<session id>.jsonl, and <session id>/subagents/*.jsonl. */
|
|
65
|
+
export const claudeCodeSource: UsageSource = {
|
|
66
|
+
id: "claude-code",
|
|
67
|
+
runtimes: ["den-claude"],
|
|
68
|
+
root: claudeProjectsDir,
|
|
69
|
+
agentFiles(token, env) {
|
|
70
|
+
if (!TOKEN_RE.test(token)) return [];
|
|
71
|
+
const out: string[] = [];
|
|
72
|
+
for (const proj of dirs(claudeProjectsDir(env))) {
|
|
73
|
+
const f = path.join(proj, `${token}.jsonl`);
|
|
74
|
+
if (fs.existsSync(f)) out.push(f);
|
|
75
|
+
out.push(...jsonl(path.join(proj, token, "subagents")));
|
|
76
|
+
}
|
|
77
|
+
return out;
|
|
78
|
+
},
|
|
79
|
+
files(since, env) {
|
|
80
|
+
const out: string[] = [];
|
|
81
|
+
const walk = (p: string, depth: number) => {
|
|
82
|
+
for (const f of jsonl(p)) {
|
|
83
|
+
try {
|
|
84
|
+
if (fs.statSync(f).mtimeMs >= since) out.push(f);
|
|
85
|
+
} catch {
|
|
86
|
+
/* gone */
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
if (depth < 3) for (const d of dirs(p)) walk(d, depth + 1);
|
|
90
|
+
};
|
|
91
|
+
walk(claudeProjectsDir(env), 0);
|
|
92
|
+
return out.sort();
|
|
93
|
+
},
|
|
94
|
+
mine: (file, opts) => mineTranscript(file, opts),
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
/** Register a usage source (Den's built-in, or a `usage` plugin's). */
|
|
98
|
+
export function registerUsageSource(src: UsageSource): void {
|
|
99
|
+
const entry: StrategyEntry<typeof USAGE_KIND, UsageSource> = {
|
|
100
|
+
id: src.id,
|
|
101
|
+
kind: USAGE_KIND,
|
|
102
|
+
detect(env) {
|
|
103
|
+
return fs.existsSync(src.root(env))
|
|
104
|
+
? { available: true }
|
|
105
|
+
: { available: false, reason: `no transcripts at ${src.root(env)}` };
|
|
106
|
+
},
|
|
107
|
+
capabilities: src,
|
|
108
|
+
};
|
|
109
|
+
registerStrategy(entry);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export function usageSources(): UsageSource[] {
|
|
113
|
+
return strategyIds(USAGE_KIND)
|
|
114
|
+
.map(
|
|
115
|
+
(id) =>
|
|
116
|
+
getStrategy<typeof USAGE_KIND, UsageSource>(USAGE_KIND, id)
|
|
117
|
+
?.capabilities,
|
|
118
|
+
)
|
|
119
|
+
.filter((s): s is UsageSource => Boolean(s));
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** The source that reads `runtime`'s agents, if any. */
|
|
123
|
+
export function sourceFor(runtime: string): UsageSource | undefined {
|
|
124
|
+
return usageSources().find((s) => s.runtimes.includes(runtime));
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
registerUsageSource(claudeCodeSource);
|
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One Claude Code transcript, read for what it cost and what it came to
|
|
3
|
+
* (den#6). Ported from the agent-usage miner: per file, inside a time
|
|
4
|
+
* window, it counts tokens by kind (once per message id), model calls by
|
|
5
|
+
* model, tool calls and tool errors, permission denials, polling loops and
|
|
6
|
+
* their sleeps, compactions, people's prompts and the ones that read as
|
|
7
|
+
* corrections, and interrupts. It also finds the outcome: a pull request
|
|
8
|
+
* opened or merged, whether the last test run passed, and the wall-clock
|
|
9
|
+
* from the first message to the last.
|
|
10
|
+
*
|
|
11
|
+
* It reads the file on this machine and keeps counts only: never a
|
|
12
|
+
* prompt's text, a command's output or a file's contents, and it sends
|
|
13
|
+
* nothing anywhere (this module imports no network code).
|
|
14
|
+
*/
|
|
15
|
+
import * as fs from "fs";
|
|
16
|
+
import * as readline from "readline";
|
|
17
|
+
|
|
18
|
+
export interface TokenCounts {
|
|
19
|
+
input: number;
|
|
20
|
+
output: number;
|
|
21
|
+
cacheRead: number;
|
|
22
|
+
cacheWrite: number;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export interface TranscriptStats {
|
|
26
|
+
file: string;
|
|
27
|
+
/** Under a session's subagents/ directory. */
|
|
28
|
+
subagent: boolean;
|
|
29
|
+
/** Its title (custom, agent name or the runtime's own), when it has one. */
|
|
30
|
+
title?: string;
|
|
31
|
+
/** First and last message inside the window (ms since the epoch). */
|
|
32
|
+
first?: number;
|
|
33
|
+
last?: number;
|
|
34
|
+
/** Model calls: assistant messages, once per message id. */
|
|
35
|
+
calls: number;
|
|
36
|
+
tokens: TokenCounts;
|
|
37
|
+
models: Record<string, number>;
|
|
38
|
+
tools: Record<string, number>;
|
|
39
|
+
toolErrors: Record<string, number>;
|
|
40
|
+
denials: number;
|
|
41
|
+
polls: number;
|
|
42
|
+
sleepSeconds: number;
|
|
43
|
+
compactions: number;
|
|
44
|
+
prompts: number;
|
|
45
|
+
corrections: number;
|
|
46
|
+
interrupts: number;
|
|
47
|
+
agentSpawns: number;
|
|
48
|
+
/** Time between messages less than 5 minutes apart (seconds). */
|
|
49
|
+
activeSeconds: number;
|
|
50
|
+
/** The last pull request URL it opened. */
|
|
51
|
+
prOpened?: string;
|
|
52
|
+
prMerged: boolean;
|
|
53
|
+
/** The last test run's result; undefined when it ran none. */
|
|
54
|
+
testsGreen?: boolean;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** A correction, as the miner reads one (emphatic ones; polite ones slip by). */
|
|
58
|
+
const CORRECTION =
|
|
59
|
+
/\b(wtf|fuck|shit|damn|not what i (said|asked)|that('s| is) wrong|you (missed|forgot|broke)|again\??$|still (not|broken|failing|404)|where (is|the)|why (did|didn't|are|is)|i told you|stop|no,|nope|ugh|boo)\b/i;
|
|
60
|
+
/** A polling loop in a shell command. */
|
|
61
|
+
const POLL = /\bsleep\s+\d|for i in \$\(seq|until |while .*sleep|watch /;
|
|
62
|
+
/** A test run in a shell command. */
|
|
63
|
+
const TEST_RUN =
|
|
64
|
+
/\b(jest|vitest|pytest|playwright test|go test|cargo test|(npm|pnpm|yarn)( run)? test\w*|make (test|check|gate)\w*)\b/;
|
|
65
|
+
const PR_URL = /https:\/\/github\.com\/[\w.-]+\/[\w.-]+\/pull\/\d+/;
|
|
66
|
+
/** Entries with nothing to count (checked on the line's head, before JSON). */
|
|
67
|
+
const SKIP = [
|
|
68
|
+
"api-",
|
|
69
|
+
"file-history",
|
|
70
|
+
"queue-",
|
|
71
|
+
"bridge",
|
|
72
|
+
"last-prompt",
|
|
73
|
+
"mode",
|
|
74
|
+
"permission-mode",
|
|
75
|
+
"tools-used",
|
|
76
|
+
];
|
|
77
|
+
|
|
78
|
+
function ts(s: unknown): number | undefined {
|
|
79
|
+
if (typeof s !== "string") return undefined;
|
|
80
|
+
const t = Date.parse(s);
|
|
81
|
+
return Number.isFinite(t) ? t : undefined;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function textOf(content: unknown): string {
|
|
85
|
+
if (typeof content === "string") return content;
|
|
86
|
+
if (Array.isArray(content))
|
|
87
|
+
return content
|
|
88
|
+
.map((c) =>
|
|
89
|
+
c && typeof c === "object" && (c as { type?: unknown }).type === "text"
|
|
90
|
+
? String((c as { text?: unknown }).text ?? "")
|
|
91
|
+
: "",
|
|
92
|
+
)
|
|
93
|
+
.join("\n");
|
|
94
|
+
return "";
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
const bump = (m: Record<string, number>, k: string, n = 1) => {
|
|
98
|
+
m[k] = (m[k] ?? 0) + n;
|
|
99
|
+
};
|
|
100
|
+
|
|
101
|
+
/** The line's `"type"`, read from its head without parsing it. */
|
|
102
|
+
function headType(line: string): string | null {
|
|
103
|
+
const m = /^\{\s*"type"\s*:\s*"([^"]+)"/.exec(line.slice(0, 120));
|
|
104
|
+
return m ? m[1] : null;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Mine `file`. `since`: only messages at or after it count (a session that
|
|
109
|
+
* runs for weeks contributes its recent calls). Lines it can't parse are
|
|
110
|
+
* skipped.
|
|
111
|
+
*/
|
|
112
|
+
export async function mineTranscript(
|
|
113
|
+
file: string,
|
|
114
|
+
opts: { since?: number } = {},
|
|
115
|
+
): Promise<TranscriptStats> {
|
|
116
|
+
const r: TranscriptStats = {
|
|
117
|
+
file,
|
|
118
|
+
subagent: /[\\/]subagents[\\/]/.test(file),
|
|
119
|
+
calls: 0,
|
|
120
|
+
tokens: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
|
121
|
+
models: {},
|
|
122
|
+
tools: {},
|
|
123
|
+
toolErrors: {},
|
|
124
|
+
denials: 0,
|
|
125
|
+
polls: 0,
|
|
126
|
+
sleepSeconds: 0,
|
|
127
|
+
compactions: 0,
|
|
128
|
+
prompts: 0,
|
|
129
|
+
corrections: 0,
|
|
130
|
+
interrupts: 0,
|
|
131
|
+
agentSpawns: 0,
|
|
132
|
+
activeSeconds: 0,
|
|
133
|
+
prMerged: false,
|
|
134
|
+
};
|
|
135
|
+
const seen = new Set<string>();
|
|
136
|
+
const toolNames = new Map<string, string>();
|
|
137
|
+
/** Tool-use ids of the commands whose result decides an outcome. */
|
|
138
|
+
const tests = new Map<string, number>();
|
|
139
|
+
const merges = new Set<string>();
|
|
140
|
+
const creates = new Set<string>();
|
|
141
|
+
let order = 0;
|
|
142
|
+
let lastTest: { at: number; green: boolean } | undefined;
|
|
143
|
+
let lastAt: number | undefined;
|
|
144
|
+
const since = opts.since ?? -Infinity;
|
|
145
|
+
const lines = readline.createInterface({
|
|
146
|
+
input: fs.createReadStream(file, { encoding: "utf-8" }),
|
|
147
|
+
crlfDelay: Infinity,
|
|
148
|
+
});
|
|
149
|
+
for await (const line of lines) {
|
|
150
|
+
const head = headType(line);
|
|
151
|
+
if (head && SKIP.some((p) => head.startsWith(p))) continue;
|
|
152
|
+
if (head === "pr-link") {
|
|
153
|
+
const m = PR_URL.exec(line);
|
|
154
|
+
if (m) r.prOpened = m[0];
|
|
155
|
+
continue;
|
|
156
|
+
}
|
|
157
|
+
let d: Record<string, unknown>;
|
|
158
|
+
try {
|
|
159
|
+
const v: unknown = JSON.parse(line);
|
|
160
|
+
if (!v || typeof v !== "object" || Array.isArray(v)) continue;
|
|
161
|
+
d = v as Record<string, unknown>;
|
|
162
|
+
} catch {
|
|
163
|
+
continue;
|
|
164
|
+
}
|
|
165
|
+
const type = d.type;
|
|
166
|
+
if (
|
|
167
|
+
type === "ai-title" ||
|
|
168
|
+
type === "custom-title" ||
|
|
169
|
+
type === "agent-name"
|
|
170
|
+
) {
|
|
171
|
+
const t = d.customTitle ?? d.agentName ?? d.aiTitle;
|
|
172
|
+
if (typeof t === "string" && t) r.title = t.slice(0, 200);
|
|
173
|
+
continue;
|
|
174
|
+
}
|
|
175
|
+
const at = ts(d.timestamp);
|
|
176
|
+
if (at !== undefined && at < since) continue;
|
|
177
|
+
if (at !== undefined) {
|
|
178
|
+
r.first ??= at;
|
|
179
|
+
r.last = at;
|
|
180
|
+
if (lastAt !== undefined && at - lastAt > 0 && at - lastAt < 300_000)
|
|
181
|
+
r.activeSeconds += (at - lastAt) / 1000;
|
|
182
|
+
lastAt = at;
|
|
183
|
+
}
|
|
184
|
+
if (type === "system") {
|
|
185
|
+
if (String(d.subtype ?? "").includes("compact")) r.compactions++;
|
|
186
|
+
continue;
|
|
187
|
+
}
|
|
188
|
+
const m = (d.message ?? {}) as Record<string, unknown>;
|
|
189
|
+
if (type === "assistant") {
|
|
190
|
+
const id = typeof m.id === "string" ? m.id : undefined;
|
|
191
|
+
if (id && !seen.has(id)) {
|
|
192
|
+
seen.add(id);
|
|
193
|
+
r.calls++;
|
|
194
|
+
const u = (m.usage ?? {}) as Record<string, unknown>;
|
|
195
|
+
const n = (k: string) =>
|
|
196
|
+
typeof u[k] === "number" ? (u[k] as number) : 0;
|
|
197
|
+
r.tokens.input += n("input_tokens");
|
|
198
|
+
r.tokens.output += n("output_tokens");
|
|
199
|
+
r.tokens.cacheRead += n("cache_read_input_tokens");
|
|
200
|
+
r.tokens.cacheWrite += n("cache_creation_input_tokens");
|
|
201
|
+
bump(r.models, typeof m.model === "string" ? m.model : "?");
|
|
202
|
+
}
|
|
203
|
+
for (const c of Array.isArray(m.content) ? m.content : []) {
|
|
204
|
+
if (!c || typeof c !== "object") continue;
|
|
205
|
+
const tu = c as Record<string, unknown>;
|
|
206
|
+
if (tu.type !== "tool_use") continue;
|
|
207
|
+
const name = typeof tu.name === "string" ? tu.name : "?";
|
|
208
|
+
const useId = typeof tu.id === "string" ? tu.id : "";
|
|
209
|
+
toolNames.set(useId, name);
|
|
210
|
+
bump(r.tools, name);
|
|
211
|
+
if (name === "Agent" || name === "Task") r.agentSpawns++;
|
|
212
|
+
if (name !== "Bash") continue;
|
|
213
|
+
const cmd = String(
|
|
214
|
+
((tu.input ?? {}) as Record<string, unknown>).command ?? "",
|
|
215
|
+
);
|
|
216
|
+
if (POLL.test(cmd)) r.polls++;
|
|
217
|
+
for (const s of cmd.matchAll(/\bsleep\s+(\d+)/g))
|
|
218
|
+
r.sleepSeconds += Number(s[1]);
|
|
219
|
+
if (TEST_RUN.test(cmd)) tests.set(useId, order++);
|
|
220
|
+
if (/\bgh pr merge\b/.test(cmd)) merges.add(useId);
|
|
221
|
+
if (/\bgh pr create\b/.test(cmd)) creates.add(useId);
|
|
222
|
+
}
|
|
223
|
+
continue;
|
|
224
|
+
}
|
|
225
|
+
if (type !== "user") continue;
|
|
226
|
+
const content = m.content;
|
|
227
|
+
if (Array.isArray(content)) {
|
|
228
|
+
for (const c of content) {
|
|
229
|
+
if (!c || typeof c !== "object") continue;
|
|
230
|
+
const tr = c as Record<string, unknown>;
|
|
231
|
+
if (tr.type !== "tool_result") continue;
|
|
232
|
+
const useId = typeof tr.tool_use_id === "string" ? tr.tool_use_id : "";
|
|
233
|
+
const err = tr.is_error === true;
|
|
234
|
+
if (err) {
|
|
235
|
+
bump(r.toolErrors, toolNames.get(useId) ?? "?");
|
|
236
|
+
const low = textOf(tr.content).toLowerCase();
|
|
237
|
+
if (/doesn't want to proceed|denied|rejected|blocked/.test(low))
|
|
238
|
+
r.denials++;
|
|
239
|
+
}
|
|
240
|
+
const t = tests.get(useId);
|
|
241
|
+
if (t !== undefined && (!lastTest || t >= lastTest.at))
|
|
242
|
+
lastTest = { at: t, green: !err };
|
|
243
|
+
if (merges.has(useId) && !err) r.prMerged = true;
|
|
244
|
+
if (creates.has(useId) && !err) {
|
|
245
|
+
const u = PR_URL.exec(textOf(tr.content));
|
|
246
|
+
if (u) r.prOpened = u[0];
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
if (d.isMeta || d.isSidechain || r.subagent) continue;
|
|
251
|
+
const text = textOf(content).trim();
|
|
252
|
+
if (text.includes("[Request interrupted")) {
|
|
253
|
+
r.interrupts++;
|
|
254
|
+
continue;
|
|
255
|
+
}
|
|
256
|
+
if (!text || text.startsWith("<") || text.startsWith("Caveat:")) continue;
|
|
257
|
+
r.prompts++;
|
|
258
|
+
if (CORRECTION.test(text.slice(0, 600))) r.corrections++;
|
|
259
|
+
}
|
|
260
|
+
if (lastTest) r.testsGreen = lastTest.green;
|
|
261
|
+
return r;
|
|
262
|
+
}
|