claude-cli-analytics 0.0.1 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +148 -60
- package/bin/cli.cjs +168 -0
- package/bin/sanity.mjs +3 -0
- package/dist/client/assets/{index-CXwfzzf8.js → index-B3X-FepG.js} +29 -22
- package/dist/client/index.html +1 -1
- package/dist/server/analyzer.js +305 -85
- package/dist/server/index.js +7 -5
- package/package.json +2 -2
- package/bin/cli.js +0 -44
package/dist/server/analyzer.js
CHANGED
|
@@ -2,8 +2,28 @@
|
|
|
2
2
|
import fs from 'fs';
|
|
3
3
|
import path from 'path';
|
|
4
4
|
import { parseJsonlFile, getProjectsList } from './parser.js';
|
|
5
|
+
// ── Model cost multiplier (Sonnet = 1.0 baseline) ──
|
|
6
|
+
const MODEL_COST_MULTIPLIER = {
|
|
7
|
+
haiku: 0.25,
|
|
8
|
+
sonnet: 1.0,
|
|
9
|
+
opus: 5.0,
|
|
10
|
+
};
|
|
11
|
+
function getModelCostMultiplier(modelName) {
|
|
12
|
+
const lower = modelName.toLowerCase();
|
|
13
|
+
for (const [key, value] of Object.entries(MODEL_COST_MULTIPLIER)) {
|
|
14
|
+
if (lower.includes(key))
|
|
15
|
+
return value;
|
|
16
|
+
}
|
|
17
|
+
return 1.0; // default to sonnet
|
|
18
|
+
}
|
|
19
|
+
function extractModelName(record) {
|
|
20
|
+
const model = record.message?.model;
|
|
21
|
+
if (model && typeof model === 'string')
|
|
22
|
+
return model;
|
|
23
|
+
return null;
|
|
24
|
+
}
|
|
5
25
|
// ── Helper: compute efficiency score ──
|
|
6
|
-
function computeEfficiencyScore(cacheHitRate, toolErrorRate, toolErrorCount, duplicateReadRate, readEditRatio, repeatedEditRate, sessionExit, tokensPerEdit) {
|
|
26
|
+
function computeEfficiencyScore(cacheHitRate, toolErrorRate, toolErrorCount, duplicateReadRate, readEditRatio, repeatedEditRate, sessionExit, tokensPerEdit, modelName = 'sonnet') {
|
|
7
27
|
let score = 0;
|
|
8
28
|
// 1. Cache efficiency (30 pts)
|
|
9
29
|
score += cacheHitRate >= 95 ? 30 : cacheHitRate >= 90 ? 25 : cacheHitRate >= 80 ? 20 : cacheHitRate >= 60 ? 10 : 0;
|
|
@@ -15,10 +35,11 @@ function computeEfficiencyScore(cacheHitRate, toolErrorRate, toolErrorCount, dup
|
|
|
15
35
|
+ (repeatedEditRate < 10 ? 5 : repeatedEditRate < 20 ? 3 : 0);
|
|
16
36
|
// 4. Session termination (10 pts)
|
|
17
37
|
score += sessionExit === 'clean' ? 10 : sessionExit === 'unknown' ? 5 : 0;
|
|
18
|
-
// 5. Cost efficiency (15 pts)
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
38
|
+
// 5. Cost efficiency (15 pts) — normalized by model cost multiplier
|
|
39
|
+
const costPerEdit = tokensPerEdit * getModelCostMultiplier(modelName);
|
|
40
|
+
score += costPerEdit > 0 && costPerEdit < 30000 ? 15
|
|
41
|
+
: costPerEdit < 50000 ? 10
|
|
42
|
+
: costPerEdit < 80000 ? 5 : 0;
|
|
22
43
|
// 6. Error penalty
|
|
23
44
|
score = Math.max(0, score - (toolErrorCount * 5));
|
|
24
45
|
return score;
|
|
@@ -62,12 +83,13 @@ function computeGrade(cacheHitRate, toolErrorCount, requests, readEditRatio) {
|
|
|
62
83
|
};
|
|
63
84
|
}
|
|
64
85
|
// ── Helper: compute danger level ──
|
|
65
|
-
function getDangerLevel(avgContextSize) {
|
|
66
|
-
|
|
86
|
+
function getDangerLevel(avgContextSize, modelName = 'sonnet') {
|
|
87
|
+
const normalizedCost = avgContextSize * getModelCostMultiplier(modelName);
|
|
88
|
+
if (normalizedCost < 10000)
|
|
67
89
|
return 'optimal';
|
|
68
|
-
if (
|
|
90
|
+
if (normalizedCost < 20000)
|
|
69
91
|
return 'safe';
|
|
70
|
-
if (
|
|
92
|
+
if (normalizedCost > 50000)
|
|
71
93
|
return 'critical';
|
|
72
94
|
return 'caution';
|
|
73
95
|
}
|
|
@@ -125,32 +147,39 @@ export function getSessionsList(projectsDir, projectPath, startDate, endDate) {
|
|
|
125
147
|
let postSpecReadTotal = 0, postSpecReadErrors = 0;
|
|
126
148
|
let recordIndex = 0;
|
|
127
149
|
let toolResultCount = 0, toolErrorCount = 0;
|
|
128
|
-
let humanTurns = 0, autoTurns = 0;
|
|
150
|
+
let humanTurns = 0, autoTurns = 0, commandTurns = 0;
|
|
151
|
+
let sessionModel = 'sonnet';
|
|
129
152
|
for (const record of records) {
|
|
130
|
-
if (record.type === 'assistant'
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
153
|
+
if (record.type === 'assistant') {
|
|
154
|
+
// Extract model name from first assistant message
|
|
155
|
+
const model = extractModelName(record);
|
|
156
|
+
if (model && sessionModel === 'sonnet')
|
|
157
|
+
sessionModel = model;
|
|
158
|
+
if (record.message?.usage) {
|
|
159
|
+
const usage = record.message.usage;
|
|
160
|
+
inputTokens += usage.input_tokens || 0;
|
|
161
|
+
// outputTokens += usage.output_tokens || 0;
|
|
162
|
+
cacheRead += usage.cache_read_input_tokens || 0;
|
|
163
|
+
requests++;
|
|
164
|
+
if (record.message?.content && Array.isArray(record.message.content)) {
|
|
165
|
+
for (const block of record.message.content) {
|
|
166
|
+
if (block.type === 'tool_use') {
|
|
167
|
+
const toolName = block.name?.toLowerCase() || '';
|
|
168
|
+
if (toolName.includes('read') || toolName.includes('view') || toolName.includes('grep') || toolName.includes('glob') || toolName.includes('list')) {
|
|
169
|
+
readCount++;
|
|
170
|
+
const input = block.input || {};
|
|
171
|
+
const fp = input.file_path || input.path || '';
|
|
172
|
+
if (firstSpecReadIndex === -1 && (fp.includes('.claude/') || fp.includes('CLAUDE.md'))) {
|
|
173
|
+
firstSpecReadIndex = recordIndex;
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
else if (toolName.includes('edit') || toolName.includes('write') || toolName.includes('replace')) {
|
|
177
|
+
editCount++;
|
|
178
|
+
const input = block.input;
|
|
179
|
+
const filePath = input?.file_path || input?.filePath;
|
|
180
|
+
if (filePath)
|
|
181
|
+
filesEdited.push(filePath);
|
|
146
182
|
}
|
|
147
|
-
}
|
|
148
|
-
else if (toolName.includes('edit') || toolName.includes('write') || toolName.includes('replace')) {
|
|
149
|
-
editCount++;
|
|
150
|
-
const input = block.input;
|
|
151
|
-
const filePath = input?.file_path || input?.filePath;
|
|
152
|
-
if (filePath)
|
|
153
|
-
filesEdited.push(filePath);
|
|
154
183
|
}
|
|
155
184
|
}
|
|
156
185
|
}
|
|
@@ -179,8 +208,7 @@ export function getSessionsList(projectsDir, projectPath, startDate, endDate) {
|
|
|
179
208
|
if (fp.includes('.claude/') || fp.includes('CLAUDE.md'))
|
|
180
209
|
specFilesRead.push(fp);
|
|
181
210
|
}
|
|
182
|
-
// Human vs auto turn detection
|
|
183
|
-
// Human vs auto turn detection
|
|
211
|
+
// Human vs auto vs command turn detection
|
|
184
212
|
if (record.type === 'user') {
|
|
185
213
|
const hasTUR = !!record.toolUseResult;
|
|
186
214
|
const isMeta = !!record.isMeta;
|
|
@@ -192,15 +220,29 @@ export function getSessionsList(projectsDir, projectPath, startDate, endDate) {
|
|
|
192
220
|
// skip
|
|
193
221
|
}
|
|
194
222
|
else if (typeof msgContent === 'string' && msgContent.trim().length > 0) {
|
|
195
|
-
|
|
223
|
+
const raw = msgContent.trim();
|
|
224
|
+
// Slash commands (/feature, /init, etc.) are skill triggers, not user interventions
|
|
225
|
+
if (raw.includes('<command-name>') || raw.startsWith('This session is being continued')) {
|
|
226
|
+
commandTurns++;
|
|
227
|
+
}
|
|
228
|
+
else if (raw.startsWith('<local-command-stdout>') || raw.startsWith('<local-command-caveat>')) {
|
|
229
|
+
autoTurns++;
|
|
230
|
+
}
|
|
231
|
+
else {
|
|
232
|
+
humanTurns++;
|
|
233
|
+
}
|
|
196
234
|
}
|
|
197
235
|
else if (Array.isArray(msgContent)) {
|
|
198
|
-
const
|
|
199
|
-
|
|
200
|
-
if (
|
|
236
|
+
const hasToolResults = msgContent.some(b => b.type === 'tool_result');
|
|
237
|
+
// Tool results (even with contextual text) are automatic, not user interventions
|
|
238
|
+
if (hasToolResults) {
|
|
201
239
|
autoTurns++;
|
|
202
|
-
|
|
203
|
-
|
|
240
|
+
}
|
|
241
|
+
else {
|
|
242
|
+
const hasText = msgContent.some(b => b.type === 'text' && b.text?.trim().length);
|
|
243
|
+
if (hasText)
|
|
244
|
+
humanTurns++;
|
|
245
|
+
}
|
|
204
246
|
}
|
|
205
247
|
}
|
|
206
248
|
recordIndex++;
|
|
@@ -211,8 +253,7 @@ export function getSessionsList(projectsDir, projectPath, startDate, endDate) {
|
|
|
211
253
|
lastRecordRaw?.type === 'system' ? 'forced' : 'unknown';
|
|
212
254
|
const totalContextSent = inputTokens + cacheRead;
|
|
213
255
|
const avgContextSize = requests > 0 ? Math.round(totalContextSent / requests) : 0;
|
|
214
|
-
const dangerLevel = getDangerLevel(avgContextSize);
|
|
215
|
-
const limitImpact = Math.round((totalContextSent / 44000) * 100);
|
|
256
|
+
const dangerLevel = getDangerLevel(avgContextSize, sessionModel);
|
|
216
257
|
const uniqueFilesRead = new Set(filesReadList).size;
|
|
217
258
|
const duplicateReads = filesReadList.length - uniqueFilesRead;
|
|
218
259
|
const duplicateReadRate = filesReadList.length > 0 ? Math.round((duplicateReads / filesReadList.length) * 100) : 0;
|
|
@@ -223,7 +264,7 @@ export function getSessionsList(projectsDir, projectPath, startDate, endDate) {
|
|
|
223
264
|
const tokensPerEdit = editCount > 0 ? Math.round(totalContextSent / editCount) : 0;
|
|
224
265
|
const toolErrorRate = toolResultCount > 0 ? Math.round((toolErrorCount / toolResultCount) * 1000) / 10 : 0;
|
|
225
266
|
const cacheHitRate = (inputTokens + cacheRead) > 0 ? (cacheRead / (inputTokens + cacheRead)) * 100 : 0;
|
|
226
|
-
const efficiencyScore = computeEfficiencyScore(cacheHitRate, toolErrorRate, toolErrorCount, duplicateReadRate, readEditRatio, repeatedEditRate, sessionExit, tokensPerEdit);
|
|
267
|
+
const efficiencyScore = computeEfficiencyScore(cacheHitRate, toolErrorRate, toolErrorCount, duplicateReadRate, readEditRatio, repeatedEditRate, sessionExit, tokensPerEdit, sessionModel);
|
|
227
268
|
const sei = computeSEI(postSpecReadTotal, postSpecReadErrors, cacheRead, new Set(specFilesRead).size);
|
|
228
269
|
const grade = computeGrade(cacheHitRate, toolErrorCount, requests, readEditRatio);
|
|
229
270
|
// Timestamps & duration
|
|
@@ -238,18 +279,55 @@ export function getSessionsList(projectsDir, projectPath, startDate, endDate) {
|
|
|
238
279
|
durationMinutes = 0;
|
|
239
280
|
}
|
|
240
281
|
const specCount = new Set(specFilesRead).size;
|
|
282
|
+
const specTriggerRate = specCount > 0 ? 100 : 0;
|
|
283
|
+
// Command turns are skill triggers (automatic), include them in auto side for autonomy calculation
|
|
284
|
+
const effectiveAutoTurns = autoTurns + commandTurns;
|
|
285
|
+
const autonomyRate = (effectiveAutoTurns + humanTurns) > 0
|
|
286
|
+
? Math.round((effectiveAutoTurns / (effectiveAutoTurns + humanTurns)) * 1000) / 10 : 0;
|
|
287
|
+
const totalToolCalls = readCount + editCount;
|
|
288
|
+
const anthropicMetrics = {
|
|
289
|
+
skill_trigger: {
|
|
290
|
+
cache_hit_rate: Math.round(cacheHitRate * 10) / 10,
|
|
291
|
+
spec_trigger_rate: specTriggerRate,
|
|
292
|
+
danger_level: dangerLevel,
|
|
293
|
+
},
|
|
294
|
+
tool_efficiency: {
|
|
295
|
+
read_edit_ratio: readEditRatio,
|
|
296
|
+
tokens_per_edit: tokensPerEdit,
|
|
297
|
+
duplicate_read_rate: duplicateReadRate,
|
|
298
|
+
repeated_edit_rate: repeatedEditRate,
|
|
299
|
+
total_tool_calls: totalToolCalls,
|
|
300
|
+
},
|
|
301
|
+
api_reliability: {
|
|
302
|
+
tool_error_rate: toolErrorRate,
|
|
303
|
+
tool_error_count: toolErrorCount,
|
|
304
|
+
session_exit: sessionExit,
|
|
305
|
+
},
|
|
306
|
+
user_intervention: {
|
|
307
|
+
human_turns: humanTurns,
|
|
308
|
+
auto_turns: effectiveAutoTurns,
|
|
309
|
+
autonomy_rate: autonomyRate,
|
|
310
|
+
},
|
|
311
|
+
workflow_autonomy: {
|
|
312
|
+
efficiency_score: efficiencyScore,
|
|
313
|
+
sei,
|
|
314
|
+
},
|
|
315
|
+
session_consistency: {
|
|
316
|
+
grade: grade.letter,
|
|
317
|
+
grade_breakdown: grade.breakdown,
|
|
318
|
+
},
|
|
319
|
+
};
|
|
241
320
|
sessions.push({
|
|
242
321
|
session_id: file.replace('.jsonl', ''),
|
|
243
322
|
project: project.name,
|
|
323
|
+
model: sessionModel,
|
|
244
324
|
start_time: startTs,
|
|
245
325
|
end_time: endTs,
|
|
246
326
|
git_branch: firstTimestampRecord?.gitBranch || '',
|
|
247
327
|
total_requests: requests,
|
|
248
328
|
avg_context_size: avgContextSize,
|
|
249
329
|
danger_level: dangerLevel,
|
|
250
|
-
limit_impact: limitImpact,
|
|
251
330
|
total_context_tokens: totalContextSent,
|
|
252
|
-
total_output_tokens: 0,
|
|
253
331
|
duplicate_read_rate: duplicateReadRate,
|
|
254
332
|
read_edit_ratio: readEditRatio,
|
|
255
333
|
repeated_edit_rate: repeatedEditRate,
|
|
@@ -262,10 +340,12 @@ export function getSessionsList(projectsDir, projectPath, startDate, endDate) {
|
|
|
262
340
|
spec_files_count: specCount,
|
|
263
341
|
has_spec_context: specCount > 0,
|
|
264
342
|
human_turns: humanTurns,
|
|
265
|
-
auto_turns:
|
|
266
|
-
|
|
343
|
+
auto_turns: effectiveAutoTurns,
|
|
344
|
+
command_turns: commandTurns,
|
|
345
|
+
edit_count: editCount,
|
|
267
346
|
spec_efficiency: sei,
|
|
268
|
-
grade_breakdown: grade.breakdown
|
|
347
|
+
grade_breakdown: grade.breakdown,
|
|
348
|
+
anthropic_metrics: anthropicMetrics,
|
|
269
349
|
});
|
|
270
350
|
}
|
|
271
351
|
}
|
|
@@ -321,11 +401,16 @@ export function getSessionDetail(projectsDir, sessionId, projectPath) {
|
|
|
321
401
|
let firstSpecReadIndex = -1;
|
|
322
402
|
let postSpecReadTotal = 0, postSpecReadErrors = 0;
|
|
323
403
|
let recordIndex = 0;
|
|
404
|
+
let sessionModel = 'sonnet';
|
|
324
405
|
const toolIdMap = new Map();
|
|
325
406
|
const errorMap = new Map();
|
|
407
|
+
// Track pending questions from interactive tools (AskFollowupQuestion etc.)
|
|
408
|
+
const pendingQuestions = new Map();
|
|
326
409
|
// Track which files were read by the assistant via tool_use
|
|
327
410
|
// so we can deduplicate them from user's "context load" display
|
|
328
411
|
let lastAssistantReadFiles = new Set();
|
|
412
|
+
// ── Skill tracking: detect slash commands and associate subsequent work ──
|
|
413
|
+
let activeSkill = null;
|
|
329
414
|
let messageId = 0;
|
|
330
415
|
for (const record of records) {
|
|
331
416
|
if (record.type === 'user') {
|
|
@@ -347,6 +432,8 @@ export function getSessionDetail(projectsDir, sessionId, projectPath) {
|
|
|
347
432
|
const cmdArgs = argsMatch ? argsMatch[1].trim() : '';
|
|
348
433
|
content = cmdArgs ? `${cmdName} ${cmdArgs}` : cmdName;
|
|
349
434
|
messageSubtype = 'command';
|
|
435
|
+
// Set active skill from slash command
|
|
436
|
+
activeSkill = cmdMatch ? cmdMatch[1].replace(/^\//, '') : null;
|
|
350
437
|
}
|
|
351
438
|
else if (raw.startsWith('This session is being continued')) {
|
|
352
439
|
content = '(세션 이어하기 — 이전 대화 요약 자동 삽입)';
|
|
@@ -355,6 +442,8 @@ export function getSessionDetail(projectsDir, sessionId, projectPath) {
|
|
|
355
442
|
else {
|
|
356
443
|
content = raw;
|
|
357
444
|
messageSubtype = 'human';
|
|
445
|
+
// User typed a message → reset active skill
|
|
446
|
+
activeSkill = null;
|
|
358
447
|
}
|
|
359
448
|
}
|
|
360
449
|
else if (Array.isArray(msgContent)) {
|
|
@@ -391,6 +480,18 @@ export function getSessionDetail(projectsDir, sessionId, projectPath) {
|
|
|
391
480
|
for (const block of msgContent) {
|
|
392
481
|
if (block?.type === 'tool_result') {
|
|
393
482
|
toolResultCount++;
|
|
483
|
+
// Link answer back to pending interactive tool question
|
|
484
|
+
const resultToolId = block.tool_use_id;
|
|
485
|
+
if (resultToolId && pendingQuestions.has(resultToolId)) {
|
|
486
|
+
let answerText = '';
|
|
487
|
+
if (typeof block.content === 'string')
|
|
488
|
+
answerText = block.content;
|
|
489
|
+
else if (Array.isArray(block.content))
|
|
490
|
+
answerText = block.content.map(c => c.text || '').join(' ');
|
|
491
|
+
const pending = pendingQuestions.get(resultToolId);
|
|
492
|
+
pending.answer = answerText || undefined;
|
|
493
|
+
pendingQuestions.delete(resultToolId);
|
|
494
|
+
}
|
|
394
495
|
if (block.is_error) {
|
|
395
496
|
toolErrorCount++;
|
|
396
497
|
if (firstSpecReadIndex !== -1 && recordIndex > firstSpecReadIndex)
|
|
@@ -425,11 +526,16 @@ export function getSessionDetail(projectsDir, sessionId, projectPath) {
|
|
|
425
526
|
timestamp: record.timestamp || '',
|
|
426
527
|
content,
|
|
427
528
|
files_read: fileList,
|
|
428
|
-
tool_uses: toolUses
|
|
529
|
+
tool_uses: toolUses,
|
|
530
|
+
skill_name: messageSubtype === 'command' ? activeSkill || undefined : undefined,
|
|
429
531
|
});
|
|
430
532
|
}
|
|
431
533
|
}
|
|
432
534
|
else if (record.type === 'assistant') {
|
|
535
|
+
// Extract model name from first assistant message
|
|
536
|
+
const model = extractModelName(record);
|
|
537
|
+
if (model && sessionModel === 'sonnet')
|
|
538
|
+
sessionModel = model;
|
|
433
539
|
const usage = record.message?.usage;
|
|
434
540
|
const content = [];
|
|
435
541
|
const toolUses = [];
|
|
@@ -450,6 +556,7 @@ export function getSessionDetail(projectsDir, sessionId, projectPath) {
|
|
|
450
556
|
const nameLower = toolName.toLowerCase();
|
|
451
557
|
if (nameLower === 'read' || nameLower === 'view') {
|
|
452
558
|
const fp = input.file_path || input.filePath || '';
|
|
559
|
+
const isSpec = fp.includes('.claude/') || fp.includes('CLAUDE.md');
|
|
453
560
|
if (fp) {
|
|
454
561
|
detail = fp;
|
|
455
562
|
// Track that this file was read by assistant tool
|
|
@@ -473,18 +580,24 @@ export function getSessionDetail(projectsDir, sessionId, projectPath) {
|
|
|
473
580
|
firstSpecReadIndex = recordIndex;
|
|
474
581
|
}
|
|
475
582
|
}
|
|
583
|
+
toolUses.push({ name: toolName, detail: detail || undefined, category: isSpec ? 'read_spec' : 'read_code' });
|
|
584
|
+
continue;
|
|
476
585
|
}
|
|
477
586
|
else if (nameLower === 'edit' || nameLower === 'replace') {
|
|
478
587
|
detail = input.file_path || input.filePath || '';
|
|
479
588
|
editCount++;
|
|
480
589
|
if (detail)
|
|
481
590
|
filesEdited.push(detail);
|
|
591
|
+
toolUses.push({ name: toolName, detail: detail || undefined, category: 'edit' });
|
|
592
|
+
continue;
|
|
482
593
|
}
|
|
483
594
|
else if (nameLower === 'write') {
|
|
484
595
|
detail = input.file_path || input.filePath || '';
|
|
485
596
|
editCount++;
|
|
486
597
|
if (detail)
|
|
487
598
|
filesEdited.push(detail);
|
|
599
|
+
toolUses.push({ name: toolName, detail: detail || undefined, category: 'edit' });
|
|
600
|
+
continue;
|
|
488
601
|
}
|
|
489
602
|
else if (nameLower === 'bash') {
|
|
490
603
|
detail = input.command || '';
|
|
@@ -507,6 +620,36 @@ export function getSessionDetail(projectsDir, sessionId, projectPath) {
|
|
|
507
620
|
else if (nameLower === 'tasklist') {
|
|
508
621
|
detail = '';
|
|
509
622
|
}
|
|
623
|
+
else if (nameLower.includes('ask') || nameLower.includes('question') || nameLower.includes('followup') || nameLower === 'attempt_completion') {
|
|
624
|
+
// Interactive tools: capture the question/message
|
|
625
|
+
// AskUserQuestion format: input.questions[{question, header, options[{label, description}]}]
|
|
626
|
+
let question = '';
|
|
627
|
+
const questions = input.questions;
|
|
628
|
+
if (Array.isArray(questions) && questions.length > 0) {
|
|
629
|
+
const parts = [];
|
|
630
|
+
for (const q of questions) {
|
|
631
|
+
let qText = '';
|
|
632
|
+
if (q.header)
|
|
633
|
+
qText += `[${q.header}] `;
|
|
634
|
+
if (q.question)
|
|
635
|
+
qText += q.question;
|
|
636
|
+
if (q.options && q.options.length > 0) {
|
|
637
|
+
qText += '\n' + q.options.map((o, idx) => ` ${idx + 1}. ${o.label || ''}${o.description ? ` - ${o.description}` : ''}`).join('\n');
|
|
638
|
+
}
|
|
639
|
+
parts.push(qText);
|
|
640
|
+
}
|
|
641
|
+
question = parts.join('\n\n');
|
|
642
|
+
}
|
|
643
|
+
else {
|
|
644
|
+
question = input.question || input.message || input.result || input.text || '';
|
|
645
|
+
}
|
|
646
|
+
detail = question.length > 100 ? question.substring(0, 100) + '...' : question;
|
|
647
|
+
const toolEntry = { name: toolName, detail: detail || undefined, question: question || undefined };
|
|
648
|
+
toolUses.push(toolEntry);
|
|
649
|
+
if (toolId)
|
|
650
|
+
pendingQuestions.set(toolId, toolEntry);
|
|
651
|
+
continue; // skip the generic push below
|
|
652
|
+
}
|
|
510
653
|
else {
|
|
511
654
|
detail = input.file_path || input.filePath || input.command || input.description || '';
|
|
512
655
|
if (nameLower.includes('read') || nameLower.includes('view') || nameLower.includes('grep') || nameLower.includes('glob') || nameLower.includes('list')) {
|
|
@@ -525,7 +668,15 @@ export function getSessionDetail(projectsDir, sessionId, projectPath) {
|
|
|
525
668
|
filesEdited.push(detail);
|
|
526
669
|
}
|
|
527
670
|
}
|
|
528
|
-
toolUses.push({
|
|
671
|
+
toolUses.push({
|
|
672
|
+
name: toolName, detail: detail || undefined,
|
|
673
|
+
category: nameLower.includes('read') || nameLower.includes('view') || nameLower.includes('list')
|
|
674
|
+
? (detail && (detail.includes('.claude/') || detail.includes('CLAUDE.md')) ? 'read_spec' : 'read_code')
|
|
675
|
+
: nameLower.includes('grep') || nameLower.includes('glob') || nameLower.includes('search')
|
|
676
|
+
? 'search'
|
|
677
|
+
: nameLower.includes('edit') || nameLower.includes('write') || nameLower.includes('replace')
|
|
678
|
+
? 'edit' : 'other'
|
|
679
|
+
});
|
|
529
680
|
}
|
|
530
681
|
}
|
|
531
682
|
}
|
|
@@ -542,7 +693,8 @@ export function getSessionDetail(projectsDir, sessionId, projectPath) {
|
|
|
542
693
|
content: content.join('\n'),
|
|
543
694
|
usage,
|
|
544
695
|
files_read: [],
|
|
545
|
-
tool_uses: toolUses
|
|
696
|
+
tool_uses: toolUses,
|
|
697
|
+
skill_name: activeSkill || undefined,
|
|
546
698
|
});
|
|
547
699
|
}
|
|
548
700
|
recordIndex++;
|
|
@@ -555,8 +707,7 @@ export function getSessionDetail(projectsDir, sessionId, projectPath) {
|
|
|
555
707
|
messages.length > 0 && messages[messages.length - 1].type === 'assistant' ? 'clean' : 'unknown';
|
|
556
708
|
const totalContextSent = inputTokens + cacheRead;
|
|
557
709
|
const avgContextSize = requests > 0 ? Math.round(totalContextSent / requests) : 0;
|
|
558
|
-
const dangerLevel = getDangerLevel(avgContextSize);
|
|
559
|
-
const limitImpact = Math.round((totalContextSent / 44000) * 100);
|
|
710
|
+
const dangerLevel = getDangerLevel(avgContextSize, sessionModel);
|
|
560
711
|
const readEditRatio = editCount > 0 ? Math.round((readCount / editCount) * 10) / 10 : readCount;
|
|
561
712
|
const uniqueFilesRead = new Set(allReadFiles).size;
|
|
562
713
|
const duplicateReads = allReadFiles.length - uniqueFilesRead;
|
|
@@ -575,13 +726,14 @@ export function getSessionDetail(projectsDir, sessionId, projectPath) {
|
|
|
575
726
|
const repeatedEditRate = filesEdited.length > 0 ? Math.round((repeatedEdits / filesEdited.length) * 100) : 0;
|
|
576
727
|
const tokensPerEdit = editCount > 0 ? Math.round(totalContextSent / editCount) : 0;
|
|
577
728
|
const cacheHitRate = (inputTokens + cacheRead) > 0 ? (cacheRead / (inputTokens + cacheRead)) * 100 : 0;
|
|
578
|
-
const efficiencyScore = computeEfficiencyScore(cacheHitRate, toolErrorRate, toolErrorCount, duplicateReadRate, readEditRatio, repeatedEditRate, sessionExit, tokensPerEdit);
|
|
729
|
+
const efficiencyScore = computeEfficiencyScore(cacheHitRate, toolErrorRate, toolErrorCount, duplicateReadRate, readEditRatio, repeatedEditRate, sessionExit, tokensPerEdit, sessionModel);
|
|
579
730
|
const specFilesReadList = allReadFiles.filter(f => f.includes('.claude/') || f.includes('CLAUDE.md'));
|
|
580
731
|
const sei = computeSEI(postSpecReadTotal, postSpecReadErrors, cacheRead, specFilesReadList.length);
|
|
581
732
|
const grade = computeGrade(cacheHitRate, toolErrorCount, requests, readEditRatio);
|
|
582
733
|
return {
|
|
583
734
|
session_id: sessionId,
|
|
584
735
|
project: foundProject,
|
|
736
|
+
model: sessionModel,
|
|
585
737
|
start_time: firstRecord.timestamp || '',
|
|
586
738
|
end_time: lastRecord.timestamp || '',
|
|
587
739
|
git_branch: firstRecord?.gitBranch || '',
|
|
@@ -590,7 +742,6 @@ export function getSessionDetail(projectsDir, sessionId, projectPath) {
|
|
|
590
742
|
total_requests: requests,
|
|
591
743
|
avg_context_size: avgContextSize,
|
|
592
744
|
danger_level: dangerLevel,
|
|
593
|
-
limit_impact: limitImpact,
|
|
594
745
|
total_context_tokens: totalContextSent,
|
|
595
746
|
files_read: [...new Set(filesRead)],
|
|
596
747
|
spec_files_read: [...new Set(specFilesRead)]
|
|
@@ -616,11 +767,57 @@ export function getSessionDetail(projectsDir, sessionId, projectPath) {
|
|
|
616
767
|
+ (readEditRatio >= 5 && readEditRatio <= 20 ? 10 : readEditRatio >= 3 ? 5 : 0)
|
|
617
768
|
+ (repeatedEditRate < 10 ? 5 : repeatedEditRate < 20 ? 3 : 0),
|
|
618
769
|
termination: sessionExit === 'clean' ? 10 : sessionExit === 'unknown' ? 5 : 0,
|
|
619
|
-
cost_efficiency:
|
|
770
|
+
cost_efficiency: (() => {
|
|
771
|
+
const costPerEdit = tokensPerEdit * getModelCostMultiplier(sessionModel);
|
|
772
|
+
return costPerEdit > 0 && costPerEdit < 30000 ? 15 : costPerEdit < 50000 ? 10 : costPerEdit < 80000 ? 5 : 0;
|
|
773
|
+
})()
|
|
620
774
|
},
|
|
621
775
|
spec_efficiency: sei,
|
|
622
776
|
grade_breakdown: grade.breakdown
|
|
623
|
-
}
|
|
777
|
+
},
|
|
778
|
+
anthropic_metrics: (() => {
|
|
779
|
+
const humanTurns = messages.filter(m => m.type === 'user' && m.subtype === 'human').length;
|
|
780
|
+
const commandTurnsDetail = messages.filter(m => m.type === 'user' && (m.subtype === 'command' || m.subtype === 'continuation')).length;
|
|
781
|
+
const autoTurns = messages.filter(m => m.type === 'user' && (m.subtype === 'tool_result' || m.subtype === '')).length + commandTurnsDetail;
|
|
782
|
+
const autonomyRate = (autoTurns + humanTurns) > 0
|
|
783
|
+
? Math.round((autoTurns / (autoTurns + humanTurns)) * 1000) / 10 : 0;
|
|
784
|
+
const specFilesInSession = allReadFiles.filter(f => f.includes('.claude/') || f.includes('CLAUDE.md'));
|
|
785
|
+
const specTriggerRate = specFilesInSession.length > 0 ? 100 : 0;
|
|
786
|
+
const totalToolCalls = readCount + editCount;
|
|
787
|
+
return {
|
|
788
|
+
skill_trigger: {
|
|
789
|
+
cache_hit_rate: Math.round(cacheHitRate * 10) / 10,
|
|
790
|
+
spec_trigger_rate: specTriggerRate,
|
|
791
|
+
danger_level: dangerLevel,
|
|
792
|
+
},
|
|
793
|
+
tool_efficiency: {
|
|
794
|
+
read_edit_ratio: readEditRatio,
|
|
795
|
+
tokens_per_edit: tokensPerEdit,
|
|
796
|
+
duplicate_read_rate: duplicateReadRate,
|
|
797
|
+
repeated_edit_rate: repeatedEditRate,
|
|
798
|
+
total_tool_calls: totalToolCalls,
|
|
799
|
+
},
|
|
800
|
+
api_reliability: {
|
|
801
|
+
tool_error_rate: Math.round(toolErrorRate * 10) / 10,
|
|
802
|
+
tool_error_count: toolErrorCount,
|
|
803
|
+
session_exit: sessionExit,
|
|
804
|
+
error_details: errorDetails,
|
|
805
|
+
},
|
|
806
|
+
user_intervention: {
|
|
807
|
+
human_turns: humanTurns,
|
|
808
|
+
auto_turns: autoTurns,
|
|
809
|
+
autonomy_rate: autonomyRate,
|
|
810
|
+
},
|
|
811
|
+
workflow_autonomy: {
|
|
812
|
+
efficiency_score: Math.round(efficiencyScore),
|
|
813
|
+
sei,
|
|
814
|
+
},
|
|
815
|
+
session_consistency: {
|
|
816
|
+
grade: grade.letter,
|
|
817
|
+
grade_breakdown: grade.breakdown,
|
|
818
|
+
},
|
|
819
|
+
};
|
|
820
|
+
})(),
|
|
624
821
|
};
|
|
625
822
|
}
|
|
626
823
|
/**
|
|
@@ -629,7 +826,7 @@ export function getSessionDetail(projectsDir, sessionId, projectPath) {
|
|
|
629
826
|
export function getAnalytics(projectsDir, projectPath, startDate, endDate) {
|
|
630
827
|
const sessions = getSessionsList(projectsDir, projectPath, startDate, endDate);
|
|
631
828
|
const projects = getProjectsList(projectsDir);
|
|
632
|
-
let totalInput = 0,
|
|
829
|
+
let totalInput = 0, totalCacheRead = 0;
|
|
633
830
|
const projectsToScan = getProjectsToScan(projectsDir, projectPath);
|
|
634
831
|
for (const projectDir of projectsToScan) {
|
|
635
832
|
if (!fs.existsSync(projectDir.path))
|
|
@@ -642,7 +839,6 @@ export function getAnalytics(projectsDir, projectPath, startDate, endDate) {
|
|
|
642
839
|
if (record.type === 'assistant' && record.message?.usage) {
|
|
643
840
|
const usage = record.message.usage;
|
|
644
841
|
totalInput += usage.input_tokens || 0;
|
|
645
|
-
totalOutput += usage.output_tokens || 0;
|
|
646
842
|
totalCacheRead += usage.cache_read_input_tokens || 0;
|
|
647
843
|
}
|
|
648
844
|
}
|
|
@@ -652,32 +848,44 @@ export function getAnalytics(projectsDir, projectPath, startDate, endDate) {
|
|
|
652
848
|
const totalRequests = sessions.reduce((sum, s) => sum + s.total_requests, 0);
|
|
653
849
|
const avgContextSize = totalRequests > 0 ? Math.round(totalContextSent / totalRequests) : 0;
|
|
654
850
|
const dangerLevel = getDangerLevel(avgContextSize);
|
|
655
|
-
const limitImpact = Math.round((totalContextSent / 44000) * 100);
|
|
656
851
|
const avgEfficiency = sessions.length > 0 ? Math.round(sessions.reduce((sum, s) => sum + s.efficiency_score, 0) / sessions.length) : 0;
|
|
657
852
|
const avgToolErrorRate = sessions.length > 0 ? Math.round(sessions.reduce((sum, s) => sum + s.tool_error_rate, 0) / sessions.length * 10) / 10 : 0;
|
|
658
|
-
// Hypothesis Check
|
|
853
|
+
// Hypothesis Check — Anthropic-aligned metrics (HT/E, SEI, P99)
|
|
659
854
|
const withSpec = sessions.filter(s => s.has_spec_context && s.total_requests > 0);
|
|
660
855
|
const withoutSpec = sessions.filter(s => !s.has_spec_context && s.total_requests > 0);
|
|
661
|
-
|
|
662
|
-
const
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
const
|
|
669
|
-
const
|
|
670
|
-
const
|
|
671
|
-
? Math.round((1 -
|
|
856
|
+
// HT/E: Human Turns per Edit (lower = more autonomous)
|
|
857
|
+
const computeAvgHTE = (arr) => {
|
|
858
|
+
const valid = arr.filter(s => s.edit_count > 0);
|
|
859
|
+
if (valid.length === 0)
|
|
860
|
+
return 0;
|
|
861
|
+
return Math.round(valid.reduce((sum, s) => sum + (s.human_turns / s.edit_count), 0) / valid.length * 100) / 100;
|
|
862
|
+
};
|
|
863
|
+
const hteWithSpec = computeAvgHTE(withSpec);
|
|
864
|
+
const hteWithoutSpec = computeAvgHTE(withoutSpec);
|
|
865
|
+
const hteImprovement = hteWithoutSpec > 0
|
|
866
|
+
? Math.round((1 - hteWithSpec / hteWithoutSpec) * 1000) / 10 : 0;
|
|
867
|
+
// SEI: Spec Efficiency Index comparison
|
|
868
|
+
const computeAvgSEI = (arr) => {
|
|
869
|
+
const valid = arr.filter(s => s.spec_efficiency && s.spec_efficiency.sei_score > 0);
|
|
870
|
+
if (valid.length === 0)
|
|
871
|
+
return 0;
|
|
872
|
+
return Math.round(valid.reduce((sum, s) => sum + (s.spec_efficiency?.sei_score || 0), 0) / valid.length * 10) / 10;
|
|
873
|
+
};
|
|
874
|
+
// P99 duration: 99th percentile tail-end (minutes)
|
|
875
|
+
const computeP99Duration = (arr) => {
|
|
876
|
+
if (arr.length === 0)
|
|
877
|
+
return 0;
|
|
878
|
+
const sorted = [...arr].map(s => s.duration_minutes).sort((a, b) => a - b);
|
|
879
|
+
const idx = Math.min(Math.floor(sorted.length * 0.99), sorted.length - 1);
|
|
880
|
+
return sorted[idx];
|
|
881
|
+
};
|
|
672
882
|
return {
|
|
673
883
|
summary: {
|
|
674
884
|
total_sessions: sessions.length,
|
|
675
885
|
total_projects: projects.length,
|
|
676
886
|
total_context_tokens: totalContextSent,
|
|
677
|
-
total_output_tokens: totalOutput,
|
|
678
887
|
avg_context_size: avgContextSize,
|
|
679
888
|
danger_level: dangerLevel,
|
|
680
|
-
limit_impact: limitImpact,
|
|
681
889
|
optimal_sessions: sessions.filter(s => s.danger_level === 'optimal').length,
|
|
682
890
|
safe_sessions: sessions.filter(s => s.danger_level === 'safe').length,
|
|
683
891
|
caution_sessions: sessions.filter(s => s.danger_level === 'caution').length,
|
|
@@ -691,18 +899,30 @@ export function getAnalytics(projectsDir, projectPath, startDate, endDate) {
|
|
|
691
899
|
clean_exits: sessions.filter(s => s.session_exit === 'clean').length,
|
|
692
900
|
forced_exits: sessions.filter(s => s.session_exit === 'forced').length,
|
|
693
901
|
hypothesis_check: {
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
normalized_improvement: normalizedImprovement,
|
|
702
|
-
avg_duration_with_spec: avg(withSpec, s => s.duration_minutes),
|
|
703
|
-
avg_duration_without_spec: avg(withoutSpec, s => s.duration_minutes),
|
|
902
|
+
hte_with_spec: hteWithSpec,
|
|
903
|
+
hte_without_spec: hteWithoutSpec,
|
|
904
|
+
hte_improvement: hteImprovement,
|
|
905
|
+
avg_sei_with_spec: computeAvgSEI(withSpec),
|
|
906
|
+
avg_sei_without_spec: computeAvgSEI(withoutSpec),
|
|
907
|
+
p99_duration_with_spec: computeP99Duration(withSpec),
|
|
908
|
+
p99_duration_without_spec: computeP99Duration(withoutSpec),
|
|
704
909
|
sessions_with_spec: withSpec.length,
|
|
705
910
|
sessions_without_spec: withoutSpec.length
|
|
911
|
+
},
|
|
912
|
+
anthropic_aggregate: {
|
|
913
|
+
avg_skill_trigger_rate: sessions.length > 0
|
|
914
|
+
? Math.round(sessions.reduce((sum, s) => sum + s.anthropic_metrics.skill_trigger.cache_hit_rate, 0) / sessions.length * 10) / 10 : 0,
|
|
915
|
+
avg_spec_trigger_rate: sessions.length > 0
|
|
916
|
+
? Math.round(sessions.filter(s => s.has_spec_context).length / sessions.length * 1000) / 10 : 0,
|
|
917
|
+
avg_tool_calls_per_session: sessions.length > 0
|
|
918
|
+
? Math.round(sessions.reduce((sum, s) => sum + s.anthropic_metrics.tool_efficiency.total_tool_calls, 0) / sessions.length * 10) / 10 : 0,
|
|
919
|
+
total_tool_errors: sessions.reduce((sum, s) => sum + s.anthropic_metrics.api_reliability.tool_error_count, 0),
|
|
920
|
+
avg_autonomy_rate: sessions.length > 0
|
|
921
|
+
? Math.round(sessions.reduce((sum, s) => sum + s.anthropic_metrics.user_intervention.autonomy_rate, 0) / sessions.length * 10) / 10 : 0,
|
|
922
|
+
workflow_completion_rate: sessions.length > 0
|
|
923
|
+
? Math.round(sessions.filter(s => s.anthropic_metrics.workflow_autonomy.efficiency_score >= 60).length / sessions.length * 1000) / 10 : 0,
|
|
924
|
+
grade_consistency: sessions.length > 0
|
|
925
|
+
? Math.round(sessions.filter(s => s.anthropic_metrics.session_consistency.grade === 'S' || s.anthropic_metrics.session_consistency.grade === 'A').length / sessions.length * 1000) / 10 : 0,
|
|
706
926
|
}
|
|
707
927
|
},
|
|
708
928
|
projects,
|