cchubber 0.5.6 → 0.5.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/analyzers/value-tracker.js +66 -11
- package/src/cli/index.js +1 -1
- package/src/renderers/html-report.js +61 -35
package/package.json
CHANGED
|
@@ -8,22 +8,34 @@
|
|
|
8
8
|
* Uses z-score anomaly detection (2 SD) to flag days where value drops significantly.
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
|
-
export function analyzeValueTrend(dailyFromJSONL) {
|
|
11
|
+
export function analyzeValueTrend(dailyFromJSONL, calculatedDailyCosts, inflectionDate) {
|
|
12
12
|
if (!dailyFromJSONL || dailyFromJSONL.length < 3) {
|
|
13
13
|
return { available: false };
|
|
14
14
|
}
|
|
15
15
|
|
|
16
|
+
// Build cost lookup from calculated costs (not raw costUSD which is 0 for subscription)
|
|
17
|
+
const costByDate = {};
|
|
18
|
+
if (calculatedDailyCosts) {
|
|
19
|
+
for (const d of calculatedDailyCosts) {
|
|
20
|
+
costByDate[d.date] = d.cost || 0;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
16
24
|
// Calculate per-day metrics
|
|
17
25
|
const daily = dailyFromJSONL
|
|
18
26
|
.filter(d => d.outputTokens > 0 && d.messageCount > 0)
|
|
19
|
-
.map(d =>
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
+
.map(d => {
|
|
28
|
+
const cost = costByDate[d.date] || 0;
|
|
29
|
+
return {
|
|
30
|
+
date: d.date,
|
|
31
|
+
outputPerMsg: Math.round(d.outputTokens / d.messageCount),
|
|
32
|
+
outputPerDollar: cost > 0 ? Math.round(d.outputTokens / cost) : 0,
|
|
33
|
+
wordsPerMsg: Math.round(d.outputTokens / d.messageCount / 1.3), // ~1.3 tokens per word
|
|
34
|
+
outputTokens: d.outputTokens,
|
|
35
|
+
messageCount: d.messageCount,
|
|
36
|
+
cost,
|
|
37
|
+
};
|
|
38
|
+
});
|
|
27
39
|
|
|
28
40
|
if (daily.length < 3) return { available: false };
|
|
29
41
|
|
|
@@ -36,10 +48,15 @@ export function analyzeValueTrend(dailyFromJSONL) {
|
|
|
36
48
|
// Recent 7 days vs older — is value declining?
|
|
37
49
|
const sorted = [...daily].sort((a, b) => a.date.localeCompare(b.date));
|
|
38
50
|
const recent = sorted.slice(-7);
|
|
39
|
-
|
|
51
|
+
// If inflection date exists, use pre-inflection as the "before" baseline
|
|
52
|
+
// This gives a clean comparison: healthy baseline vs current reality
|
|
53
|
+
const older = inflectionDate
|
|
54
|
+
? sorted.filter(d => d.date < inflectionDate)
|
|
55
|
+
: sorted.slice(0, -7);
|
|
40
56
|
|
|
41
57
|
let trend = 'stable';
|
|
42
58
|
let trendDetail = '';
|
|
59
|
+
let comparison = null;
|
|
43
60
|
|
|
44
61
|
if (older.length >= 3) {
|
|
45
62
|
const recentAvgPerMsg = Math.round(recent.reduce((s, d) => s + d.outputPerMsg, 0) / recent.length);
|
|
@@ -64,6 +81,14 @@ export function analyzeValueTrend(dailyFromJSONL) {
|
|
|
64
81
|
} else {
|
|
65
82
|
trendDetail = `Output per message stable (${recentAvgPerMsg} tokens/msg, was ${olderAvgPerMsg})`;
|
|
66
83
|
}
|
|
84
|
+
|
|
85
|
+
// Expose before/after for visual comparison
|
|
86
|
+
comparison = {
|
|
87
|
+
olderAvgPerMsg, recentAvgPerMsg, msgChangePct: Math.round(msgChange),
|
|
88
|
+
olderAvgPerDollar, recentAvgPerDollar, dollarChangePct: Math.round(dollarChange),
|
|
89
|
+
olderWords: Math.round(olderAvgPerMsg / 1.3),
|
|
90
|
+
recentWords: Math.round(recentAvgPerMsg / 1.3),
|
|
91
|
+
};
|
|
67
92
|
}
|
|
68
93
|
|
|
69
94
|
// Z-score anomaly detection — flag days where value drops >2 SD
|
|
@@ -89,14 +114,44 @@ export function analyzeValueTrend(dailyFromJSONL) {
|
|
|
89
114
|
}
|
|
90
115
|
}
|
|
91
116
|
|
|
117
|
+
// Cost per message anomalies — days where each message cost way more than usual (caching broke)
|
|
118
|
+
const costPerMsgValues = daily.filter(d => d.cost > 0).map(d => d.cost / d.messageCount);
|
|
119
|
+
const cpmMean = costPerMsgValues.length > 0 ? costPerMsgValues.reduce((s, v) => s + v, 0) / costPerMsgValues.length : 0;
|
|
120
|
+
const cpmVariance = costPerMsgValues.length > 0 ? costPerMsgValues.reduce((s, v) => s + Math.pow(v - cpmMean, 2), 0) / costPerMsgValues.length : 0;
|
|
121
|
+
const cpmStdDev = Math.sqrt(cpmVariance);
|
|
122
|
+
|
|
123
|
+
// Use median as baseline — more robust to outliers than mean/z-score
|
|
124
|
+
const sortedCpm = [...costPerMsgValues].sort((a, b) => a - b);
|
|
125
|
+
const cpmMedian = sortedCpm.length > 0 ? sortedCpm[Math.floor(sortedCpm.length / 2)] : 0;
|
|
126
|
+
|
|
127
|
+
const costAnomalies = [];
|
|
128
|
+
if (cpmMedian > 0) {
|
|
129
|
+
for (const d of daily.filter(dd => dd.cost > 0 && dd.messageCount >= 10)) {
|
|
130
|
+
const cpm = d.cost / d.messageCount;
|
|
131
|
+
const multiplier = cpm / cpmMedian;
|
|
132
|
+
if (multiplier >= 1.8) { // 80%+ above median cost per message
|
|
133
|
+
costAnomalies.push({
|
|
134
|
+
date: d.date,
|
|
135
|
+
costPerMsg: Math.round(cpm * 100) / 100,
|
|
136
|
+
medianCostPerMsg: Math.round(cpmMedian * 100) / 100,
|
|
137
|
+
multiplier: Math.round(multiplier * 10) / 10,
|
|
138
|
+
});
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
92
143
|
return {
|
|
93
144
|
available: true,
|
|
94
|
-
daily: daily.map(d => ({ date: d.date, outputPerMsg: d.outputPerMsg, outputPerDollar: d.outputPerDollar })),
|
|
145
|
+
daily: daily.map(d => ({ date: d.date, outputPerMsg: d.outputPerMsg, outputPerDollar: d.outputPerDollar, wordsPerMsg: d.wordsPerMsg })),
|
|
146
|
+
avgWordsPerMsg: Math.round(daily.reduce((s, d) => s + d.wordsPerMsg, 0) / daily.length),
|
|
95
147
|
avgOutputPerMsg,
|
|
96
148
|
avgOutputPerDollar,
|
|
97
149
|
trend,
|
|
98
150
|
trendDetail,
|
|
151
|
+
comparison,
|
|
99
152
|
anomalies,
|
|
153
|
+
costAnomalies,
|
|
154
|
+
avgCostPerMsg: Math.round(cpmMean * 100) / 100,
|
|
100
155
|
dayCount: daily.length,
|
|
101
156
|
};
|
|
102
157
|
}
|
package/src/cli/index.js
CHANGED
|
@@ -135,7 +135,7 @@ async function main() {
|
|
|
135
135
|
const inflection = detectInflectionPoints(dailyFromJSONL);
|
|
136
136
|
const sessionIntel = analyzeSessionIntelligence(sessionMeta, jsonlEntries);
|
|
137
137
|
const modelRouting = analyzeModelRouting(costAnalysis, jsonlEntries);
|
|
138
|
-
const valueTrend = analyzeValueTrend(dailyFromJSONL);
|
|
138
|
+
const valueTrend = analyzeValueTrend(dailyFromJSONL, costAnalysis.dailyCosts, inflection?.date);
|
|
139
139
|
if (valueTrend.available) console.log(` ✓ Value trend: ${valueTrend.avgOutputPerMsg} tokens/msg avg (${valueTrend.trend})`);
|
|
140
140
|
|
|
141
141
|
const recommendations = generateRecommendations(costAnalysis, cacheHealth, claudeMdStack, anomalies, inflection, sessionIntel, modelRouting, projectBreakdown);
|
|
@@ -18,6 +18,7 @@ export function renderHTML(report) {
|
|
|
18
18
|
|
|
19
19
|
const dailyCostsJSON = JSON.stringify(dailyCosts.map(d => ({
|
|
20
20
|
date: d.date, cost: d.cost, cacheOutputRatio: d.cacheOutputRatio || 0, isAnomaly: anomalyDates.has(d.date),
|
|
21
|
+
out: d.outputTokens || 0, inp: d.inputTokens || 0, cr: d.cacheReadTokens || 0,
|
|
21
22
|
})));
|
|
22
23
|
|
|
23
24
|
const projectsJSON = JSON.stringify((projectBreakdown || []).map(p => ({
|
|
@@ -353,6 +354,7 @@ ${inflection && inflection.multiplier >= 1.5 ? `
|
|
|
353
354
|
<span class="font-mono text-2xl font-bold block text-[#e3e2e3]">${costAnalysis.sessions?.total || 0}</span>
|
|
354
355
|
<span class="text-[10px] text-[#908fa0] mt-1 block font-mono">${costAnalysis.sessions?.avgDurationMinutes ? Math.round(costAnalysis.sessions.avgDurationMinutes) + ' min avg' : ''}</span>
|
|
355
356
|
</div>`}
|
|
357
|
+
|
|
356
358
|
</section>
|
|
357
359
|
|
|
358
360
|
<!-- 4. COST TREND CHART -->
|
|
@@ -372,38 +374,61 @@ ${inflection && inflection.multiplier >= 1.5 ? `
|
|
|
372
374
|
<svg id="cost-chart-svg" viewBox="0 0 900 200" preserveAspectRatio="xMidYMid meet"></svg>
|
|
373
375
|
</section>
|
|
374
376
|
|
|
375
|
-
${report.valueTrend?.available ?
|
|
377
|
+
${report.valueTrend?.available ? (() => {
|
|
378
|
+
const vt = report.valueTrend;
|
|
379
|
+
const comp = vt.comparison;
|
|
380
|
+
const trendColor = vt.trend === 'declining' ? '#ffb4ab' : vt.trend === 'improving' ? '#10b981' : '#908fa0';
|
|
381
|
+
|
|
382
|
+
// Build verdict around output per dollar — the metric people actually care about
|
|
383
|
+
let verdict, verdictColor, verdictDetail;
|
|
384
|
+
if (!comp || !comp.olderAvgPerDollar) {
|
|
385
|
+
verdict = 'Not enough data to compare yet. Run again next week.';
|
|
386
|
+
verdictColor = '#908fa0';
|
|
387
|
+
verdictDetail = '';
|
|
388
|
+
} else if (comp.dollarChangePct < -20) {
|
|
389
|
+
verdict = 'Yes. ' + Math.abs(comp.dollarChangePct) + '% less output per dollar recently.';
|
|
390
|
+
verdictColor = '#ffb4ab';
|
|
391
|
+
var limitImpact = Math.round(100 / (100 - Math.abs(comp.dollarChangePct)) * 100 - 100);
|
|
392
|
+
verdictDetail = 'Before: ' + comp.olderAvgPerDollar.toLocaleString() + ' tokens/$1. Now: ' + comp.recentAvgPerDollar.toLocaleString() + ' tokens/$1. In practice, your usage limits run out ~' + limitImpact + '% faster than before. This could be caused by cache changes, heavier Opus usage, or longer sessions.';
|
|
393
|
+
} else if (comp.dollarChangePct > 20) {
|
|
394
|
+
verdict = 'No. Actually getting ' + comp.dollarChangePct + '% more output per dollar recently.';
|
|
395
|
+
verdictColor = '#10b981';
|
|
396
|
+
verdictDetail = 'Before: ' + comp.olderAvgPerDollar.toLocaleString() + ' tokens/$1. Now: ' + comp.recentAvgPerDollar.toLocaleString() + ' tokens/$1.';
|
|
397
|
+
} else {
|
|
398
|
+
verdict = 'No. Your output per dollar is stable.';
|
|
399
|
+
verdictColor = '#10b981';
|
|
400
|
+
verdictDetail = comp.recentAvgPerDollar.toLocaleString() + ' output tokens per $1 (was ' + comp.olderAvgPerDollar.toLocaleString() + ').';
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
return `
|
|
376
404
|
<!-- VALUE TREND -->
|
|
377
405
|
<section class="bg-[#1b1c1d] p-8 rounded-xl border border-[rgba(70,69,84,0.15)]">
|
|
378
|
-
<
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
<div class="bg-[#0d0e0f] p-5 rounded-lg">
|
|
389
|
-
<span class="text-[10px] uppercase tracking-[0.05em] text-[#908fa0] block mb-2">Tokens per Dollar</span>
|
|
390
|
-
<span class="font-mono text-2xl font-bold text-[#e3e2e3]">${report.valueTrend.avgOutputPerDollar.toLocaleString()}</span>
|
|
391
|
-
<span class="text-[10px] text-[#908fa0] block mt-1">output tokens per $1 spent</span>
|
|
392
|
-
</div>
|
|
393
|
-
</div>
|
|
394
|
-
${report.valueTrend.trendDetail ? `<p class="text-[12px] text-[#908fa0] mb-4">${esc(report.valueTrend.trendDetail)}</p>` : ''}
|
|
395
|
-
${report.valueTrend.anomalies?.length > 0 ? `
|
|
396
|
-
<div class="space-y-2">
|
|
397
|
-
<span class="text-[10px] uppercase tracking-[0.05em] text-[#908fa0] block">Low Output Days (2+ SD below average)</span>
|
|
398
|
-
${report.valueTrend.anomalies.slice(0, 5).map(a => `
|
|
406
|
+
<h3 class="text-xl font-bold text-[#e3e2e3] mb-4">Are you getting less for your money?</h3>
|
|
407
|
+
<p class="text-[15px] font-semibold mb-1" style="color:${verdictColor}">${verdict}</p>
|
|
408
|
+
${verdictDetail ? '<p class="text-[12px] text-[#908fa0] mb-2">' + verdictDetail + '</p>' : ''}
|
|
409
|
+
${inflection ? '<p class="text-[11px] text-[#908fa0] mb-4">A cache change was detected on ' + esc(inflection.date) + '. Your recent 7 days reflect where you are now.</p>' : ''}
|
|
410
|
+
|
|
411
|
+
${(vt.anomalies?.length > 0 || vt.costAnomalies?.length > 0) ? `
|
|
412
|
+
<div class="mt-6 space-y-2">
|
|
413
|
+
${vt.anomalies?.length > 0 ? `
|
|
414
|
+
<span class="text-[10px] uppercase tracking-[0.05em] text-[#908fa0] block mb-1">Days with unusually short responses</span>
|
|
415
|
+
${vt.anomalies.slice(0, 3).map(a => `
|
|
399
416
|
<div class="p-3 bg-[#0d0e0f] rounded-lg flex items-center justify-between">
|
|
400
|
-
<span class="text-[
|
|
401
|
-
<span class="text-[
|
|
402
|
-
<span class="text-[10px] font-mono px-2 py-0.5 rounded" style="background:rgba(255,180,171,0.15);color:#ffb4ab">${a.deviation}σ below</span>
|
|
403
|
-
</div>`).join('')}
|
|
417
|
+
<span class="text-[11px] font-mono text-[#908fa0]">${a.date}</span>
|
|
418
|
+
<span class="text-[11px] text-[#e3e2e3]">~${Math.round(a.outputPerMsg / 1.3)} words/response</span>
|
|
419
|
+
<span class="text-[10px] font-mono px-2 py-0.5 rounded" style="background:rgba(255,180,171,0.15);color:#ffb4ab">${a.deviation}σ below avg</span>
|
|
420
|
+
</div>`).join('')}` : ''}
|
|
421
|
+
${vt.costAnomalies?.length > 0 ? `
|
|
422
|
+
<span class="text-[10px] uppercase tracking-[0.05em] text-[#908fa0] block mt-3 mb-1">Days where each message cost more than usual (${vt.costAnomalies.length} days)</span>
|
|
423
|
+
${vt.costAnomalies.slice(0, 5).map(a => `
|
|
424
|
+
<div class="p-3 bg-[#0d0e0f] rounded-lg grid" style="grid-template-columns:100px 1fr auto;gap:12px;align-items:center">
|
|
425
|
+
<span class="text-[11px] font-mono text-[#908fa0]">${a.date}</span>
|
|
426
|
+
<span class="text-[11px] text-[#e3e2e3]">$${a.costPerMsg}/msg vs $${a.medianCostPerMsg} median</span>
|
|
427
|
+
<span class="text-[10px] font-mono px-2 py-0.5 rounded" style="background:rgba(255,180,171,0.15);color:#ffb4ab">${a.multiplier}x</span>
|
|
428
|
+
</div>`).join('')}` : ''}
|
|
404
429
|
</div>` : ''}
|
|
405
|
-
</section
|
|
406
|
-
|
|
430
|
+
</section>`;
|
|
431
|
+
})() : ''}
|
|
407
432
|
|
|
408
433
|
<!-- 5. SESSION INTELLIGENCE + MODEL DISTRIBUTION -->
|
|
409
434
|
<section class="grid grid-cols-1 lg:grid-cols-2 gap-8">
|
|
@@ -554,9 +579,9 @@ ${report.valueTrend?.available ? `
|
|
|
554
579
|
<th class="px-8 py-3 text-[10px] uppercase font-bold tracking-[0.05em] text-[#908fa0] text-left">#</th>
|
|
555
580
|
<th class="px-4 py-3 text-[10px] uppercase font-bold tracking-[0.05em] text-[#908fa0]">Grade</th>
|
|
556
581
|
<th class="px-4 py-3 text-[10px] uppercase font-bold tracking-[0.05em] text-[#908fa0] text-right">Ratio</th>
|
|
557
|
-
<th class="px-4 py-3 text-[10px] uppercase font-bold tracking-[0.05em] text-[#908fa0]">Cost</th>
|
|
582
|
+
<th class="px-4 py-3 text-[10px] uppercase font-bold tracking-[0.05em] text-[#908fa0] text-left">Cost</th>
|
|
558
583
|
<th class="px-4 py-3 text-[10px] uppercase font-bold tracking-[0.05em] text-[#908fa0] text-right">Opus %</th>
|
|
559
|
-
<th class="px-
|
|
584
|
+
<th class="px-4 py-3 text-[10px] uppercase font-bold tracking-[0.05em] text-[#908fa0] text-left">Country</th>
|
|
560
585
|
</tr>
|
|
561
586
|
</thead>
|
|
562
587
|
<tbody id="leaderboard-body"></tbody>
|
|
@@ -817,13 +842,14 @@ ${cacheHealth.totalCacheBreaks > 0 ? `
|
|
|
817
842
|
// hover targets
|
|
818
843
|
d.forEach(function(x,j){
|
|
819
844
|
var px=PD.l+(d.length===1?cW/2:j*step),py=PD.t+cH-(x.cost/mx)*cH;
|
|
820
|
-
s+='<circle cx="'+px+'" cy="'+py+'" r="14" fill="transparent" data-d="'+x.date+'" data-c="'+x.cost+'" data-a="'+(x.isAnomaly?1:0)+'" class="hov" style="cursor:crosshair"/>';
|
|
845
|
+
s+='<circle cx="'+px+'" cy="'+py+'" r="14" fill="transparent" data-d="'+x.date+'" data-c="'+x.cost+'" data-a="'+(x.isAnomaly?1:0)+'" data-o="'+(x.out||0)+'" class="hov" style="cursor:crosshair"/>';
|
|
821
846
|
});
|
|
822
847
|
svg.innerHTML=s;
|
|
823
848
|
svg.querySelectorAll('.hov').forEach(function(el){
|
|
824
849
|
el.addEventListener('mouseenter',function(e){
|
|
825
850
|
ttd.textContent=e.target.dataset.d;
|
|
826
|
-
|
|
851
|
+
var ot=parseInt(e.target.dataset.o||0);
|
|
852
|
+
ttc.textContent=fc(parseFloat(e.target.dataset.c))+(ot>0?' — '+ft(ot)+' output':'');
|
|
827
853
|
tta.textContent=e.target.dataset.a==='1'?'ANOMALY':'';
|
|
828
854
|
tta.style.display=e.target.dataset.a==='1'?'block':'none';
|
|
829
855
|
tt.classList.add('on');
|
|
@@ -838,7 +864,7 @@ ${cacheHealth.totalCacheBreaks > 0 ? `
|
|
|
838
864
|
function setR(r){
|
|
839
865
|
var f=filt(r);chart(f);
|
|
840
866
|
var ci=document.getElementById('chart-info');
|
|
841
|
-
if(ci&&f.length){var t=f.reduce(function(s,x){return s+x.cost},0),a=f.filter(function(x){return x.cost>0}).length;ci.textContent=a+' days \u00b7 '+fc(t)}
|
|
867
|
+
if(ci&&f.length){var t=f.reduce(function(s,x){return s+x.cost},0),a=f.filter(function(x){return x.cost>0}).length,tok=f.reduce(function(s,x){return s+(x.out||0)},0);ci.textContent=a+' days \u00b7 '+fc(t)+' \u00b7 '+ft(tok)+' output'}
|
|
842
868
|
var rl=document.getElementById('range-lbl');if(rl)rl.textContent=RL[r]||'All time';
|
|
843
869
|
CARD.range=RL[r]||'All time';
|
|
844
870
|
var cr=document.getElementById('card-range');if(cr)cr.textContent=CARD.range;
|
|
@@ -1197,9 +1223,9 @@ ${cacheHealth.totalCacheBreaks > 0 ? `
|
|
|
1197
1223
|
html += '<td class="px-8 py-3 text-sm font-mono '+(isMe?'text-[#c0c1ff] font-bold':'text-[#908fa0]')+'">#'+(idx+1)+(isMe?' ← you':'')+'</td>';
|
|
1198
1224
|
html += '<td class="px-4 py-3 text-sm font-bold text-center" style="color:'+gradeColors[g]+'">'+g+'</td>';
|
|
1199
1225
|
html += '<td class="px-4 py-3 text-sm font-mono text-[#c7c4d7] text-right">'+(entry.ratio||'?')+':1</td>';
|
|
1200
|
-
html += '<td class="px-4 py-3 text-sm font-mono text-[#908fa0]">'+(entry.cost
|
|
1226
|
+
html += '<td class="px-4 py-3 text-sm font-mono text-[#908fa0]">'+(entry.cost?'$'+entry.cost:'?')+'</td>';
|
|
1201
1227
|
html += '<td class="px-4 py-3 text-sm font-mono text-[#c7c4d7] text-right">'+(entry.opus!=null?entry.opus:'?')+'%</td>';
|
|
1202
|
-
html += '<td class="px-
|
|
1228
|
+
html += '<td class="px-4 py-3 text-sm text-[#908fa0]">'+(isMe?'You':(entry.country||'?'))+'</td>';
|
|
1203
1229
|
html += '</tr>';
|
|
1204
1230
|
}
|
|
1205
1231
|
tbody.innerHTML = html;
|