cchubber 0.5.5 → 0.5.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/analyzers/value-tracker.js +102 -0
- package/src/cli/index.js +5 -0
- package/src/renderers/html-report.js +33 -0
package/package.json
CHANGED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Value Tracker — measures whether you're getting less for your money over time.
|
|
3
|
+
*
|
|
4
|
+
* Two core metrics:
|
|
5
|
+
* - Output per message: are Claude's responses getting shorter? (model degradation)
|
|
6
|
+
* - Output per dollar: are you getting less work per dollar? (pricing/caching changes)
|
|
7
|
+
*
|
|
8
|
+
* Uses z-score anomaly detection (2 SD) to flag days where value drops significantly.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
export function analyzeValueTrend(dailyFromJSONL) {
|
|
12
|
+
if (!dailyFromJSONL || dailyFromJSONL.length < 3) {
|
|
13
|
+
return { available: false };
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
// Calculate per-day metrics
|
|
17
|
+
const daily = dailyFromJSONL
|
|
18
|
+
.filter(d => d.outputTokens > 0 && d.messageCount > 0)
|
|
19
|
+
.map(d => ({
|
|
20
|
+
date: d.date,
|
|
21
|
+
outputPerMsg: Math.round(d.outputTokens / d.messageCount),
|
|
22
|
+
outputPerDollar: d.cost > 0 ? Math.round(d.outputTokens / d.cost) : 0,
|
|
23
|
+
outputTokens: d.outputTokens,
|
|
24
|
+
messageCount: d.messageCount,
|
|
25
|
+
cost: d.cost || 0,
|
|
26
|
+
}));
|
|
27
|
+
|
|
28
|
+
if (daily.length < 3) return { available: false };
|
|
29
|
+
|
|
30
|
+
// Overall averages
|
|
31
|
+
const avgOutputPerMsg = Math.round(daily.reduce((s, d) => s + d.outputPerMsg, 0) / daily.length);
|
|
32
|
+
const avgOutputPerDollar = daily.filter(d => d.outputPerDollar > 0).length > 0
|
|
33
|
+
? Math.round(daily.filter(d => d.outputPerDollar > 0).reduce((s, d) => s + d.outputPerDollar, 0) / daily.filter(d => d.outputPerDollar > 0).length)
|
|
34
|
+
: 0;
|
|
35
|
+
|
|
36
|
+
// Recent 7 days vs older — is value declining?
|
|
37
|
+
const sorted = [...daily].sort((a, b) => a.date.localeCompare(b.date));
|
|
38
|
+
const recent = sorted.slice(-7);
|
|
39
|
+
const older = sorted.slice(0, -7);
|
|
40
|
+
|
|
41
|
+
let trend = 'stable';
|
|
42
|
+
let trendDetail = '';
|
|
43
|
+
|
|
44
|
+
if (older.length >= 3) {
|
|
45
|
+
const recentAvgPerMsg = Math.round(recent.reduce((s, d) => s + d.outputPerMsg, 0) / recent.length);
|
|
46
|
+
const olderAvgPerMsg = Math.round(older.reduce((s, d) => s + d.outputPerMsg, 0) / older.length);
|
|
47
|
+
|
|
48
|
+
const recentAvgPerDollar = recent.filter(d => d.outputPerDollar > 0).length > 0
|
|
49
|
+
? Math.round(recent.filter(d => d.outputPerDollar > 0).reduce((s, d) => s + d.outputPerDollar, 0) / recent.filter(d => d.outputPerDollar > 0).length)
|
|
50
|
+
: 0;
|
|
51
|
+
const olderAvgPerDollar = older.filter(d => d.outputPerDollar > 0).length > 0
|
|
52
|
+
? Math.round(older.filter(d => d.outputPerDollar > 0).reduce((s, d) => s + d.outputPerDollar, 0) / older.filter(d => d.outputPerDollar > 0).length)
|
|
53
|
+
: 0;
|
|
54
|
+
|
|
55
|
+
const msgChange = olderAvgPerMsg > 0 ? ((recentAvgPerMsg - olderAvgPerMsg) / olderAvgPerMsg * 100) : 0;
|
|
56
|
+
const dollarChange = olderAvgPerDollar > 0 ? ((recentAvgPerDollar - olderAvgPerDollar) / olderAvgPerDollar * 100) : 0;
|
|
57
|
+
|
|
58
|
+
if (msgChange < -20) {
|
|
59
|
+
trend = 'declining';
|
|
60
|
+
trendDetail = `Output per message dropped ${Math.abs(Math.round(msgChange))}% recently (${olderAvgPerMsg} → ${recentAvgPerMsg} tokens/msg)`;
|
|
61
|
+
} else if (msgChange > 20) {
|
|
62
|
+
trend = 'improving';
|
|
63
|
+
trendDetail = `Output per message increased ${Math.round(msgChange)}% recently (${olderAvgPerMsg} → ${recentAvgPerMsg} tokens/msg)`;
|
|
64
|
+
} else {
|
|
65
|
+
trendDetail = `Output per message stable (${recentAvgPerMsg} tokens/msg, was ${olderAvgPerMsg})`;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// Z-score anomaly detection — flag days where value drops >2 SD
|
|
70
|
+
const values = daily.map(d => d.outputPerMsg);
|
|
71
|
+
const mean = values.reduce((s, v) => s + v, 0) / values.length;
|
|
72
|
+
const variance = values.reduce((s, v) => s + Math.pow(v - mean, 2), 0) / values.length;
|
|
73
|
+
const stdDev = Math.sqrt(variance);
|
|
74
|
+
|
|
75
|
+
const anomalies = [];
|
|
76
|
+
if (stdDev > 0) {
|
|
77
|
+
for (const d of daily) {
|
|
78
|
+
const z = (d.outputPerMsg - mean) / stdDev;
|
|
79
|
+
if (z < -2) {
|
|
80
|
+
anomalies.push({
|
|
81
|
+
date: d.date,
|
|
82
|
+
type: 'low_output',
|
|
83
|
+
outputPerMsg: d.outputPerMsg,
|
|
84
|
+
expected: avgOutputPerMsg,
|
|
85
|
+
deviation: Math.abs(Math.round(z * 10) / 10),
|
|
86
|
+
detail: `${d.outputPerMsg} tokens/msg vs ${avgOutputPerMsg} avg (${Math.abs(Math.round(z * 10) / 10)}σ below)`,
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
return {
|
|
93
|
+
available: true,
|
|
94
|
+
daily: daily.map(d => ({ date: d.date, outputPerMsg: d.outputPerMsg, outputPerDollar: d.outputPerDollar })),
|
|
95
|
+
avgOutputPerMsg,
|
|
96
|
+
avgOutputPerDollar,
|
|
97
|
+
trend,
|
|
98
|
+
trendDetail,
|
|
99
|
+
anomalies,
|
|
100
|
+
dayCount: daily.length,
|
|
101
|
+
};
|
|
102
|
+
}
|
package/src/cli/index.js
CHANGED
|
@@ -25,6 +25,7 @@ import { generateRecommendations } from '../analyzers/recommendations.js';
|
|
|
25
25
|
import { detectInflectionPoints } from '../analyzers/inflection-detector.js';
|
|
26
26
|
import { analyzeSessionIntelligence } from '../analyzers/session-intelligence.js';
|
|
27
27
|
import { analyzeModelRouting } from '../analyzers/model-routing.js';
|
|
28
|
+
import { analyzeValueTrend } from '../analyzers/value-tracker.js';
|
|
28
29
|
import { renderHTML } from '../renderers/html-report.js';
|
|
29
30
|
import { renderTerminal } from '../renderers/terminal-summary.js';
|
|
30
31
|
import { shouldSendTelemetry, sendTelemetry } from '../telemetry.js';
|
|
@@ -134,6 +135,9 @@ async function main() {
|
|
|
134
135
|
const inflection = detectInflectionPoints(dailyFromJSONL);
|
|
135
136
|
const sessionIntel = analyzeSessionIntelligence(sessionMeta, jsonlEntries);
|
|
136
137
|
const modelRouting = analyzeModelRouting(costAnalysis, jsonlEntries);
|
|
138
|
+
const valueTrend = analyzeValueTrend(dailyFromJSONL);
|
|
139
|
+
if (valueTrend.available) console.log(` ✓ Value trend: ${valueTrend.avgOutputPerMsg} tokens/msg avg (${valueTrend.trend})`);
|
|
140
|
+
|
|
137
141
|
const recommendations = generateRecommendations(costAnalysis, cacheHealth, claudeMdStack, anomalies, inflection, sessionIntel, modelRouting, projectBreakdown);
|
|
138
142
|
|
|
139
143
|
if (inflection) console.log(` ✓ Inflection: ${inflection.summary}`);
|
|
@@ -166,6 +170,7 @@ async function main() {
|
|
|
166
170
|
claudeMdStack,
|
|
167
171
|
oauthUsage,
|
|
168
172
|
recommendations,
|
|
173
|
+
valueTrend,
|
|
169
174
|
communityStats,
|
|
170
175
|
history: getHistory(),
|
|
171
176
|
};
|
|
@@ -372,6 +372,39 @@ ${inflection && inflection.multiplier >= 1.5 ? `
|
|
|
372
372
|
<svg id="cost-chart-svg" viewBox="0 0 900 200" preserveAspectRatio="xMidYMid meet"></svg>
|
|
373
373
|
</section>
|
|
374
374
|
|
|
375
|
+
${report.valueTrend?.available ? `
|
|
376
|
+
<!-- VALUE TREND -->
|
|
377
|
+
<section class="bg-[#1b1c1d] p-8 rounded-xl border border-[rgba(70,69,84,0.15)]">
|
|
378
|
+
<div class="flex items-center justify-between mb-6">
|
|
379
|
+
<h3 class="text-xl font-bold text-[#e3e2e3]">Output Value</h3>
|
|
380
|
+
<span class="text-[10px] font-mono text-[#908fa0]">${report.valueTrend.trend === 'declining' ? 'DECLINING' : report.valueTrend.trend === 'improving' ? 'IMPROVING' : 'STABLE'}</span>
|
|
381
|
+
</div>
|
|
382
|
+
<div class="grid grid-cols-2 gap-6 mb-6">
|
|
383
|
+
<div class="bg-[#0d0e0f] p-5 rounded-lg">
|
|
384
|
+
<span class="text-[10px] uppercase tracking-[0.05em] text-[#908fa0] block mb-2">Tokens per Message</span>
|
|
385
|
+
<span class="font-mono text-2xl font-bold text-[#e3e2e3]">${report.valueTrend.avgOutputPerMsg.toLocaleString()}</span>
|
|
386
|
+
<span class="text-[10px] text-[#908fa0] block mt-1">avg output tokens per turn</span>
|
|
387
|
+
</div>
|
|
388
|
+
<div class="bg-[#0d0e0f] p-5 rounded-lg">
|
|
389
|
+
<span class="text-[10px] uppercase tracking-[0.05em] text-[#908fa0] block mb-2">Tokens per Dollar</span>
|
|
390
|
+
<span class="font-mono text-2xl font-bold text-[#e3e2e3]">${report.valueTrend.avgOutputPerDollar.toLocaleString()}</span>
|
|
391
|
+
<span class="text-[10px] text-[#908fa0] block mt-1">output tokens per $1 spent</span>
|
|
392
|
+
</div>
|
|
393
|
+
</div>
|
|
394
|
+
${report.valueTrend.trendDetail ? `<p class="text-[12px] text-[#908fa0] mb-4">${esc(report.valueTrend.trendDetail)}</p>` : ''}
|
|
395
|
+
${report.valueTrend.anomalies?.length > 0 ? `
|
|
396
|
+
<div class="space-y-2">
|
|
397
|
+
<span class="text-[10px] uppercase tracking-[0.05em] text-[#908fa0] block">Low Output Days (2+ SD below average)</span>
|
|
398
|
+
${report.valueTrend.anomalies.slice(0, 5).map(a => `
|
|
399
|
+
<div class="p-3 bg-[#0d0e0f] rounded-lg flex items-center justify-between">
|
|
400
|
+
<span class="text-[12px] font-mono text-[#908fa0]">${a.date}</span>
|
|
401
|
+
<span class="text-[12px] text-[#e3e2e3]">${a.outputPerMsg} tokens/msg</span>
|
|
402
|
+
<span class="text-[10px] font-mono px-2 py-0.5 rounded" style="background:rgba(255,180,171,0.15);color:#ffb4ab">${a.deviation}σ below</span>
|
|
403
|
+
</div>`).join('')}
|
|
404
|
+
</div>` : ''}
|
|
405
|
+
</section>
|
|
406
|
+
` : ''}
|
|
407
|
+
|
|
375
408
|
<!-- 5. SESSION INTELLIGENCE + MODEL DISTRIBUTION -->
|
|
376
409
|
<section class="grid grid-cols-1 lg:grid-cols-2 gap-8">
|
|
377
410
|
|