@trazum/cli 1.9.0 → 1.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +156 -1
- package/dist/i18n/en.d.ts.map +1 -1
- package/dist/i18n/en.js +273 -3
- package/dist/i18n/en.js.map +1 -1
- package/dist/i18n/es.d.ts.map +1 -1
- package/dist/i18n/es.js +275 -3
- package/dist/i18n/es.js.map +1 -1
- package/dist/i18n/types.d.ts +524 -0
- package/dist/i18n/types.d.ts.map +1 -1
- package/dist/index.js +1778 -31
- package/dist/index.js.map +1 -1
- package/dist/markdown.d.ts +68 -0
- package/dist/markdown.d.ts.map +1 -1
- package/dist/markdown.js +329 -1
- package/dist/markdown.js.map +1 -1
- package/dist/time.d.ts +18 -0
- package/dist/time.d.ts.map +1 -0
- package/dist/time.js +32 -0
- package/dist/time.js.map +1 -0
- package/package.json +2 -2
- package/src/i18n/en.ts +402 -3
- package/src/i18n/es.ts +405 -3
- package/src/i18n/types.ts +547 -0
- package/src/index.ts +2129 -174
- package/src/markdown.ts +415 -1
- package/src/time.ts +32 -0
package/dist/index.js
CHANGED
|
@@ -1,14 +1,16 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { readFile, stat, writeFile } from 'node:fs/promises';
|
|
2
|
+
import { readdir, readFile, stat, writeFile } from 'node:fs/promises';
|
|
3
3
|
import { join, resolve as resolvePath } from 'node:path';
|
|
4
|
-
import {
|
|
4
|
+
import { gunzipSync } from 'node:zlib';
|
|
5
|
+
import { applyRewrites, BASELINE_FILENAME, BASELINE_VERSION, breaches, cacheableMinimum, analyzeCachePrefix, billLevers, cacheEconomics, cacheHitRate, comparePrompts, compareToBaseline, computeSavings, countTokensAnthropic, DEFAULT_USAGE, detectFromSource, driversBetween, estimateTokens, evaluate, extractPrompts, findExamples, formatBaseline, formatSignedUsd, formatUsd, getMessages, getModel, hasMarker, LOCALES, MAX_BASELINE_BYTES, moneyIsComparable, mostSpecificMatch, nearestName, optimize, parseBaseline, PHRASE_LANGUAGES, plannedCalls, profilePrompt, profileToCsv, profileUsage, promptId, providerFromEnv, pruneExamples, refineWithLlm, rejectionText, reorderForCache, repriceProfile, reviewAgeDays, reviewExamples, RULES, sharedPrefixes, sharesOf, SOURCE_EXTENSIONS, suggestRewrites, toOtlpMetrics, toPromptfoo, TTL_1H_MS, UNLABELLED, withExactTokenCounts, } from '@trazum/core';
|
|
5
6
|
import { cacheDir, cacheStats, cachingProvider, clearCache } from './suggest-cache.js';
|
|
7
|
+
import { dayOf, formatGap, median, spanDays } from './time.js';
|
|
6
8
|
// Everything that reads the filesystem, on its own entry point so the web
|
|
7
9
|
// bundle cannot reach it. See packages/core/src/node.ts.
|
|
8
10
|
import { CONFIG_FILENAME, DEFAULT_EXTENSIONS, budgetFor, BUNDLED_CATALOGUE, SAFE_FETCH_INIT, applyPricingOverlay, catalogueFromOverlay, checkedEndpoint, openrouterOverlay, detectHost, loadConfig, walkPrompts, } from '@trazum/core/node';
|
|
9
11
|
import { contentAt, gitAvailable, namesByRevision, pathInRepository, repositoryRoot, revisionsFor, } from './git.js';
|
|
10
12
|
import { detectLocale, getCliMessages } from './i18n/index.js';
|
|
11
|
-
import { MAX_SUMMARY_CHARS, fitWithin, renderBlameMarkdown, renderCheckMarkdown, renderDiffMarkdown, renderRankMarkdown, } from './markdown.js';
|
|
13
|
+
import { MAX_SUMMARY_CHARS, fitWithin, renderBlameMarkdown, renderCheckMarkdown, renderDiffMarkdown, renderRankMarkdown, renderProfileMarkdown, } from './markdown.js';
|
|
12
14
|
// --------------------------------------------------------------------------
|
|
13
15
|
// Presentation
|
|
14
16
|
// --------------------------------------------------------------------------
|
|
@@ -22,6 +24,12 @@ const c = {
|
|
|
22
24
|
cyan: (s) => (useColor ? `\u001b[36m${s}\u001b[39m` : s),
|
|
23
25
|
};
|
|
24
26
|
const VALUE_FLAGS = new Set([
|
|
27
|
+
'against',
|
|
28
|
+
// `route` takes a path here, and the flag is deliberately not `--prompt`:
|
|
29
|
+
// everywhere else in this tool `--prompt` names a marked prompt *inside* a
|
|
30
|
+
// source file, and reusing it for a path would be a trap laid for the reader.
|
|
31
|
+
'prompt-file',
|
|
32
|
+
'label',
|
|
25
33
|
'level',
|
|
26
34
|
'model',
|
|
27
35
|
'calls',
|
|
@@ -32,6 +40,15 @@ const VALUE_FLAGS = new Set([
|
|
|
32
40
|
'cases',
|
|
33
41
|
'concurrency',
|
|
34
42
|
'max-growth',
|
|
43
|
+
'max-usd',
|
|
44
|
+
'max-growth-usd',
|
|
45
|
+
'max-cache-loss-usd',
|
|
46
|
+
'max-day-usd',
|
|
47
|
+
'csv-out',
|
|
48
|
+
'csv-shape',
|
|
49
|
+
'what-if',
|
|
50
|
+
'since',
|
|
51
|
+
'until',
|
|
35
52
|
'export',
|
|
36
53
|
'limit',
|
|
37
54
|
'locale',
|
|
@@ -241,6 +258,8 @@ const COMMAND_FLAGS = {
|
|
|
241
258
|
],
|
|
242
259
|
check: ['max-tokens', 'level', 'exact-tokens', 'markdown-out', 'baseline'],
|
|
243
260
|
baseline: ['model', 'calls', 'output-tokens', 'cache-hit-rate', 'batch', 'exact-tokens', 'out', 'o'],
|
|
261
|
+
profile: ['json', 'pricing', 'pricing-live', 'against', 'what-if', 'markdown-out', 'csv-out', 'csv-shape', 'max-usd', 'max-growth-usd', 'max-cache-loss-usd', 'max-day-usd', 'label', 'since', 'until'],
|
|
262
|
+
route: ['prompt-file', 'cases', 'label', 'concurrency', 'json', 'yes', 'pricing', 'pricing-live'],
|
|
244
263
|
eval: ['cases', 'level', 'concurrency', 'export', 'out', 'o', 'model'],
|
|
245
264
|
prune: ['cases', 'concurrency', 'json', 'yes'],
|
|
246
265
|
diff: ['level', 'model', 'calls', 'output-tokens', 'batch', 'max-growth', 'optimized', 'markdown-out', 'all', 'prompt'],
|
|
@@ -330,7 +349,7 @@ function renderDiff(before, after, t) {
|
|
|
330
349
|
* The provider's name when the estimator was not calibrated for it.
|
|
331
350
|
*
|
|
332
351
|
* `estimateTokens` is a heuristic tuned against Claude's tokenizer, and the
|
|
333
|
-
* ±
|
|
352
|
+
* ±10% band descends from that. Printing the same band beside a GPT or Kimi
|
|
334
353
|
* figure states a precision nobody has measured for that family — and since the
|
|
335
354
|
* catalogue grew past Anthropic, that is most of it. Returns null when the model
|
|
336
355
|
* is Anthropic's, where the band is at least the claim it was written for.
|
|
@@ -353,11 +372,83 @@ function languageNames(codes, t) {
|
|
|
353
372
|
return names[0] ?? '';
|
|
354
373
|
return `${names.slice(0, -1).join(', ')} ${t.languages.and} ${names[names.length - 1]}`;
|
|
355
374
|
}
|
|
356
|
-
|
|
375
|
+
/**
|
|
376
|
+
* Advisories whose entire pitch is money.
|
|
377
|
+
*
|
|
378
|
+
* On a subscription these are not weaker advice, they are not advice: "use a
|
|
379
|
+
* cheaper model" saves nothing on a flat plan, and its detail text quotes dollars
|
|
380
|
+
* per month, so suppressing only the price tag beside the title left the money in
|
|
381
|
+
* the sentence underneath.
|
|
382
|
+
*/
|
|
383
|
+
const MONEY_ONLY_ADVISORIES = new Set([
|
|
384
|
+
'model-downgrade',
|
|
385
|
+
'batch-api',
|
|
386
|
+
'output-dominated',
|
|
387
|
+
'promo-pricing',
|
|
388
|
+
'prompt-caching-not-worth-it',
|
|
389
|
+
]);
|
|
390
|
+
/**
|
|
391
|
+
* The one thing worth doing about this prompt, and how it compares to shortening it.
|
|
392
|
+
*
|
|
393
|
+
* `null` when there is nothing to say: no advisory carries a figure, or the
|
|
394
|
+
* reader is on a subscription where a monthly saving is meaningless. A heading
|
|
395
|
+
* with a shrug under it is worse than no heading.
|
|
396
|
+
*/
|
|
397
|
+
function biggestLever(result, tokensOnly, t) {
|
|
398
|
+
/**
|
|
399
|
+
* One guard, and it is the only thing deciding.
|
|
400
|
+
*
|
|
401
|
+
* The first version also filtered the candidate list by `!tokensOnly`, which
|
|
402
|
+
* duplicated this and made it untestable: removing the guard left the filter
|
|
403
|
+
* still suppressing the line, so a mutation that priced a subscription passed
|
|
404
|
+
* the suite. Two checks for one condition is one check and one place for a bug.
|
|
405
|
+
*/
|
|
406
|
+
if (tokensOnly)
|
|
407
|
+
return null;
|
|
408
|
+
const best = result.advisories.find((a) => (a.estimatedMonthlyUsd ?? 0) > 0);
|
|
409
|
+
if (!best?.estimatedMonthlyUsd)
|
|
410
|
+
return null;
|
|
411
|
+
const ruleSaving = result.savings.monthlySavingsUsd;
|
|
412
|
+
return {
|
|
413
|
+
line: t.report.biggestLeverDetail(best.title, formatUsd(best.estimatedMonthlyUsd),
|
|
414
|
+
// The multiple is the point of the line, and it is only honest when there
|
|
415
|
+
// is something to divide by. A prompt the rules could not improve at all
|
|
416
|
+
// gets the amount and no ratio rather than a division by zero dressed up.
|
|
417
|
+
ruleSaving > 0 ? Math.round(best.estimatedMonthlyUsd / ruleSaving) : null),
|
|
418
|
+
};
|
|
419
|
+
}
|
|
420
|
+
function printReport(result, showDiff, t, examplesReview = null, reorder = null, tokensOnly = false, host = { id: 'terminal', displayName: 'terminal', billing: 'unknown', evidence: null }, suggestions = null,
|
|
421
|
+
/** They named a scenario, and the host is suppressing the money anyway. */
|
|
422
|
+
namedScenario = false) {
|
|
357
423
|
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
358
424
|
const sourceNote = result.tokenSource === 'heuristic'
|
|
359
425
|
? c.dim(t.report.estimated(offFamilyName(result.usage.model)))
|
|
360
426
|
: c.dim(t.report.exactCount());
|
|
427
|
+
/**
|
|
428
|
+
* The largest lever, first.
|
|
429
|
+
*
|
|
430
|
+
* This line used to be the last thing in the report and it is the most useful
|
|
431
|
+
* thing in it. Measured on an ordinary support prompt — already reasonably
|
|
432
|
+
* written, which is what a real one is — the rules recover **three tokens of
|
|
433
|
+
* 306**, worth $0.75 a month, while the cache reorder sitting below them is
|
|
434
|
+
* worth $48. The report opened with the 1.3% and closed with the 64×.
|
|
435
|
+
*
|
|
436
|
+
* That ordering is not a presentation quibble. It teaches the reader that
|
|
437
|
+
* shortening the prompt is what this tool is for, and on any prompt somebody
|
|
438
|
+
* competent wrote, shortening it is the smallest thing available. The rules
|
|
439
|
+
* earn their keep on genuine bloat — a duplicated paragraph, "due to the fact
|
|
440
|
+
* that" — and recover close to nothing once that is gone, because they recover
|
|
441
|
+
* waste rather than creating savings.
|
|
442
|
+
*
|
|
443
|
+
* So the answer to "what should I do about this prompt" goes at the top, and
|
|
444
|
+
* the token count follows as the detail it is.
|
|
445
|
+
*/
|
|
446
|
+
const best = biggestLever(result, tokensOnly, t);
|
|
447
|
+
if (best) {
|
|
448
|
+
console.log();
|
|
449
|
+
console.log(c.bold(t.report.biggestLever()));
|
|
450
|
+
console.log(` ${c.dim(wrap(best.line, 74, ' '))}`);
|
|
451
|
+
}
|
|
361
452
|
console.log();
|
|
362
453
|
console.log(c.bold(t.report.inputTokens()));
|
|
363
454
|
console.log(` ${n(result.tokensBefore)} → ${c.green(n(result.tokensAfter))} ${c.bold(`-${result.reductionPct.toFixed(1)}%`)}${sourceNote}`);
|
|
@@ -447,7 +538,7 @@ function printReport(result, showDiff, t, examplesReview = null, reorder = null,
|
|
|
447
538
|
//
|
|
448
539
|
// What replaces it is the thing that *is* scarce there: the context window.
|
|
449
540
|
if (tokensOnly) {
|
|
450
|
-
printTokensOnly(result, host, t, n);
|
|
541
|
+
printTokensOnly(result, host, t, n, namedScenario);
|
|
451
542
|
}
|
|
452
543
|
else {
|
|
453
544
|
printMoney(result, t, n);
|
|
@@ -460,13 +551,7 @@ function printReport(result, showDiff, t, examplesReview = null, reorder = null,
|
|
|
460
551
|
// The rest stay: an overflowing context window still fails the call, a
|
|
461
552
|
// contradiction is still wrong, redundant examples still cost tokens, and
|
|
462
553
|
// caching still buys latency and rate-limit headroom.
|
|
463
|
-
const MONEY_ONLY =
|
|
464
|
-
'model-downgrade',
|
|
465
|
-
'batch-api',
|
|
466
|
-
'output-dominated',
|
|
467
|
-
'promo-pricing',
|
|
468
|
-
'prompt-caching-not-worth-it',
|
|
469
|
-
]);
|
|
554
|
+
const MONEY_ONLY = MONEY_ONLY_ADVISORIES;
|
|
470
555
|
const advisories = tokensOnly
|
|
471
556
|
? result.advisories.filter((a) => !MONEY_ONLY.has(a.id))
|
|
472
557
|
: result.advisories;
|
|
@@ -497,22 +582,27 @@ function printReport(result, showDiff, t, examplesReview = null, reorder = null,
|
|
|
497
582
|
console.log(` ${marker} ${column}${c.bold(advisory.title)}`);
|
|
498
583
|
console.log(`${gutter}${c.dim(wrap(advisory.detail, 78 - gutter.length, gutter))}`);
|
|
499
584
|
}
|
|
500
|
-
//
|
|
501
|
-
//
|
|
502
|
-
// sentences is how the most valuable line in the report gets skipped.
|
|
503
|
-
const best = advisories.find((a) => (a.estimatedMonthlyUsd ?? 0) > 0);
|
|
504
|
-
if (!tokensOnly && best?.estimatedMonthlyUsd) {
|
|
505
|
-
const ruleSaving = result.savings.monthlySavingsUsd;
|
|
506
|
-
const line = t.report.biggestLeverDetail(best.title, formatUsd(best.estimatedMonthlyUsd), ruleSaving > 0 ? Math.round(best.estimatedMonthlyUsd / ruleSaving) : null);
|
|
507
|
-
console.log();
|
|
508
|
-
// Wrapped to the same width as everything else. An unwrapped closing line
|
|
509
|
-
// is the one that runs off a narrow terminal, and it is the line most
|
|
510
|
-
// worth reading.
|
|
511
|
-
console.log(` ${c.bold(t.report.biggestLever())} ${c.dim(wrap(line, 62, ' '))}`);
|
|
512
|
-
}
|
|
585
|
+
// The "start here" line is printed at the top of the report now, where a
|
|
586
|
+
// reader who stops after four lines still sees it.
|
|
513
587
|
}
|
|
514
588
|
printSuggestions(suggestions, t, n);
|
|
515
589
|
printRest(result, showDiff, t, examplesReview, n);
|
|
590
|
+
/**
|
|
591
|
+
* Where the money actually is, said at the front door.
|
|
592
|
+
*
|
|
593
|
+
* `optimize` is the first command anybody runs, and it reports the smallest
|
|
594
|
+
* line item on the bill: measured, about 1% of a monthly figure. Everything
|
|
595
|
+
* that moves 40% to 80% — which model the call goes to, the Batch API,
|
|
596
|
+
* caching, what re-sending the conversation costs — lives in `profile`, which
|
|
597
|
+
* needs a usage log a new reader does not have and has no reason to go looking
|
|
598
|
+
* for.
|
|
599
|
+
*
|
|
600
|
+
* A tool that learned that and only said it in the command you reach last has
|
|
601
|
+
* not said it. So it prints here, once, at the end, on every run: this is the
|
|
602
|
+
* small lever, and the big ones are one file away.
|
|
603
|
+
*/
|
|
604
|
+
console.log();
|
|
605
|
+
console.log(` ${c.dim(wrap(tokensOnly ? t.report.beyondThisPromptTokensOnly() : t.report.beyondThisPrompt(), 74, ' '))}`);
|
|
516
606
|
}
|
|
517
607
|
/** The cost section, for anyone billed by the token. */
|
|
518
608
|
function printMoney(result, t, n) {
|
|
@@ -542,7 +632,9 @@ function printMoney(result, t, n) {
|
|
|
542
632
|
* a measurable one, and it is the honest answer to "what did I gain" on a plan
|
|
543
633
|
* that costs the same either way.
|
|
544
634
|
*/
|
|
545
|
-
function printTokensOnly(result, host, t, n
|
|
635
|
+
function printTokensOnly(result, host, t, n,
|
|
636
|
+
/** Whether they named a scenario while the money was being withheld. */
|
|
637
|
+
namedScenario = false) {
|
|
546
638
|
const model = getModel(result.usage.model);
|
|
547
639
|
const saved = result.tokensBefore - result.tokensAfter;
|
|
548
640
|
console.log();
|
|
@@ -555,10 +647,34 @@ function printTokensOnly(result, host, t, n) {
|
|
|
555
647
|
: t.report.tokensOnlyAsked()}`);
|
|
556
648
|
console.log();
|
|
557
649
|
console.log(` ${c.green(t.report.tokensSaved(n(saved)))}`);
|
|
558
|
-
|
|
650
|
+
/**
|
|
651
|
+
* Share of the window, which is what a saved token is actually worth here.
|
|
652
|
+
*
|
|
653
|
+
* A 225-token prompt against a million-token window printed `0.0% → 0.0%`: a
|
|
654
|
+
* line whose whole job is to say what a token buys, saying nothing twice. When
|
|
655
|
+
* both sides round to the same figure the honest statement is the other one —
|
|
656
|
+
* that the window is not the constraint on this prompt.
|
|
657
|
+
*/
|
|
559
658
|
const share = (tokens) => `${((tokens / model.contextWindow) * 100).toFixed(1)}%`;
|
|
560
|
-
|
|
561
|
-
|
|
659
|
+
const before = share(result.tokensBefore);
|
|
660
|
+
const after = share(result.tokensAfter);
|
|
661
|
+
/**
|
|
662
|
+
* Three cases, and the first version had two.
|
|
663
|
+
*
|
|
664
|
+
* Equal shares mean either "this prompt is nothing against a million tokens" or
|
|
665
|
+
* "this prompt is 10% of the window and one token did not move it". Using the
|
|
666
|
+
* negligible message for both told a reader holding a tenth of a Haiku window
|
|
667
|
+
* that they were under a tenth of a percent — off by two orders of magnitude,
|
|
668
|
+
* on a line whose only job is to size the prompt against the window.
|
|
669
|
+
*/
|
|
670
|
+
const unchanged = before === after;
|
|
671
|
+
const negligible = after === '0.0%';
|
|
672
|
+
console.log(` ${c.dim(!unchanged
|
|
673
|
+
? t.report.windowUse(before, after, model.displayName, n(model.contextWindow))
|
|
674
|
+
: negligible
|
|
675
|
+
? t.report.windowNegligible(n(result.tokensAfter), model.displayName, n(model.contextWindow))
|
|
676
|
+
: t.report.windowUnmoved(after, model.displayName, n(model.contextWindow)))}`);
|
|
677
|
+
console.log(` ${c.dim(namedScenario ? t.report.tokensOnlyAskedFor() : t.report.tokensOnlyCost())}`);
|
|
562
678
|
}
|
|
563
679
|
/**
|
|
564
680
|
* The proposed rewrites.
|
|
@@ -1062,9 +1178,23 @@ async function commandOptimize(args, config, pricing, t, locale) {
|
|
|
1062
1178
|
const tokensOnly = boolFlag(args, 'cost')
|
|
1063
1179
|
? false
|
|
1064
1180
|
: boolFlag(args, 'tokens-only') || host.billing === 'subscription';
|
|
1181
|
+
/**
|
|
1182
|
+
* Whether they named a scenario while the money was being withheld.
|
|
1183
|
+
*
|
|
1184
|
+
* Not a reason to start printing dollars — `--cost` is the documented way to
|
|
1185
|
+
* ask, and `--calls` is a scenario parameter with a default that several
|
|
1186
|
+
* commands take purely to size a finding. Making it imply `--cost` would hand
|
|
1187
|
+
* dollar figures to somebody who put `--calls` in an alias precisely because
|
|
1188
|
+
* they had configured the tool not to show them.
|
|
1189
|
+
*
|
|
1190
|
+
* It is a reason to stop answering with a generic hint. Somebody who typed
|
|
1191
|
+
* `--calls 50000` and read "pass --cost if this prompt is bound for a metered
|
|
1192
|
+
* API" has been told to do a thing they plainly just tried to do.
|
|
1193
|
+
*/
|
|
1194
|
+
const namedScenario = args.flags.has('calls') || args.flags.has('output-tokens');
|
|
1065
1195
|
printReport(result, boolFlag(args, 'diff'), t, examplesReview, reorder, tokensOnly, host, suggestions
|
|
1066
1196
|
? { result: suggestions, applied: boolFlag(args, 'apply-suggestions'), locale }
|
|
1067
|
-
: null);
|
|
1197
|
+
: null, namedScenario);
|
|
1068
1198
|
if (outPath) {
|
|
1069
1199
|
console.log(c.dim(t.report.wroteTo(outPath)));
|
|
1070
1200
|
console.log();
|
|
@@ -1304,6 +1434,1617 @@ function monthlyCostOf(tokens, usage, pricing) {
|
|
|
1304
1434
|
function isoDate() {
|
|
1305
1435
|
return new Date().toISOString().slice(0, 10);
|
|
1306
1436
|
}
|
|
1437
|
+
/**
|
|
1438
|
+
* `trazum profile <log.jsonl>` — where the money actually went.
|
|
1439
|
+
*
|
|
1440
|
+
* Every other command in this file reads a prompt and reasons forward about what
|
|
1441
|
+
* it would cost. This one reads what the provider charged and reasons backward,
|
|
1442
|
+
* and it exists because the forward direction can only see the smallest line item:
|
|
1443
|
+
* on an ordinary support prompt the rules recover about 1% of the monthly figure
|
|
1444
|
+
* while output alone was 87% of it.
|
|
1445
|
+
*
|
|
1446
|
+
* **Money is never suppressed here, unlike every other report.** The rest of the
|
|
1447
|
+
* CLI hides dollar figures on a subscription host, because a saving quoted to
|
|
1448
|
+
* somebody on a flat plan is money that does not exist. This log is a record of
|
|
1449
|
+
* metered API calls somebody was actually billed for — the bill exists wherever
|
|
1450
|
+
* Trazum happens to be running, so the host has no bearing on it.
|
|
1451
|
+
*/
|
|
1452
|
+
async function commandProfile(args, config, pricing, t) {
|
|
1453
|
+
const path = args.positional[0];
|
|
1454
|
+
if (path === undefined) {
|
|
1455
|
+
console.log();
|
|
1456
|
+
console.log(c.dim(wrap(t.profile.noTarget(), 74, ' ')));
|
|
1457
|
+
console.log();
|
|
1458
|
+
return;
|
|
1459
|
+
}
|
|
1460
|
+
/**
|
|
1461
|
+
* A log, or a directory of them.
|
|
1462
|
+
*
|
|
1463
|
+
* Usage logs rotate: `logs/2026-08-01.jsonl`, `logs/2026-08-02.jsonl`, one
|
|
1464
|
+
* per day for a month. Making somebody `cat` them together before a profile
|
|
1465
|
+
* will read them is a setup cost that gets a tool skipped, and doing it for
|
|
1466
|
+
* them is a directory listing.
|
|
1467
|
+
*
|
|
1468
|
+
* Files are read in name order — which for dated names is time order — and
|
|
1469
|
+
* how many were read is stated, because a report over "the logs" that
|
|
1470
|
+
* silently skipped one is a total that is wrong by an unknown amount. A
|
|
1471
|
+
* directory holding nothing readable is an error naming what it looked for,
|
|
1472
|
+
* not an empty report.
|
|
1473
|
+
*/
|
|
1474
|
+
const LOG_EXTENSIONS = ['.jsonl', '.ndjson', '.log', '.json'];
|
|
1475
|
+
/**
|
|
1476
|
+
* The same names, gzipped — which is what a rotated log actually looks like
|
|
1477
|
+
* a day after it rotates.
|
|
1478
|
+
*
|
|
1479
|
+
* `logrotate`, Docker's json-file driver and every cloud log export compress
|
|
1480
|
+
* yesterday's file, so a directory of a month's logs is one plain file and
|
|
1481
|
+
* twenty-nine `.gz` ones. Reading only the plain one and saying nothing
|
|
1482
|
+
* would report a month's bill from a day of it, in the flattering
|
|
1483
|
+
* direction, which is exactly the failure directory mode was added to
|
|
1484
|
+
* prevent.
|
|
1485
|
+
*/
|
|
1486
|
+
const GZ_EXTENSIONS = LOG_EXTENSIONS.map((ext) => `${ext}.gz`);
|
|
1487
|
+
const READABLE = [...LOG_EXTENSIONS, ...GZ_EXTENSIONS];
|
|
1488
|
+
const target = await stat(path).catch(() => null);
|
|
1489
|
+
let logFiles = [path];
|
|
1490
|
+
if (target?.isDirectory()) {
|
|
1491
|
+
const entries = await readdir(path, { withFileTypes: true });
|
|
1492
|
+
logFiles = entries
|
|
1493
|
+
.filter((entry) => entry.isFile() && READABLE.some((ext) => entry.name.endsWith(ext)))
|
|
1494
|
+
.map((entry) => join(path, entry.name))
|
|
1495
|
+
.sort((a, b) => a.localeCompare(b));
|
|
1496
|
+
if (logFiles.length === 0) {
|
|
1497
|
+
throw new Error(t.profile.noLogsInDirectory(path, READABLE.join(', ')));
|
|
1498
|
+
}
|
|
1499
|
+
}
|
|
1500
|
+
/**
|
|
1501
|
+
* Gzipped files are decompressed in memory; everything else is read as text.
|
|
1502
|
+
*
|
|
1503
|
+
* Decided by **extension**, not by sniffing the first two bytes: a file
|
|
1504
|
+
* named `.jsonl` whose contents happen to start with 0x1f8b is far more
|
|
1505
|
+
* likely to be a corrupt log than a mislabelled archive, and silently
|
|
1506
|
+
* treating it as one would turn a diagnosable error into an empty report.
|
|
1507
|
+
*
|
|
1508
|
+
* A `.gz` that will not decompress is an error naming the file. The
|
|
1509
|
+
* alternative — skipping it — is a total quietly missing a day, which is
|
|
1510
|
+
* the failure this repository refuses in every other place it can occur.
|
|
1511
|
+
*/
|
|
1512
|
+
const readLog = async (file) => {
|
|
1513
|
+
if (!file.endsWith('.gz'))
|
|
1514
|
+
return readFile(file, 'utf8');
|
|
1515
|
+
const compressed = await readFile(file);
|
|
1516
|
+
try {
|
|
1517
|
+
return gunzipSync(compressed).toString('utf8');
|
|
1518
|
+
}
|
|
1519
|
+
catch (error) {
|
|
1520
|
+
throw new Error(t.profile.badGzip(file, error instanceof Error ? error.message : String(error)));
|
|
1521
|
+
}
|
|
1522
|
+
};
|
|
1523
|
+
const logTexts = await Promise.all(logFiles.map((file) => readLog(file)));
|
|
1524
|
+
// A file that does not end in a newline would otherwise glue its last record
|
|
1525
|
+
// to the next file's first one, and both would be reported as unreadable.
|
|
1526
|
+
const raw = logTexts.map((text) => (text.endsWith('\n') ? text : `${text}\n`)).join('');
|
|
1527
|
+
/**
|
|
1528
|
+
* The drill-down. A label that matches nothing is an error naming the labels
|
|
1529
|
+
* that exist — the route command's rule, for the route command's reason: a
|
|
1530
|
+
* report over zero calls silently filtered would read as "this workload is
|
|
1531
|
+
* free".
|
|
1532
|
+
*/
|
|
1533
|
+
const onlyLabel = stringFlag(args, 'label');
|
|
1534
|
+
/**
|
|
1535
|
+
* The drill-down in time. `--since`/`--until` take a UTC day or a full
|
|
1536
|
+
* timestamp; a bare day means the whole of it — since its first instant,
|
|
1537
|
+
* until its last — because "--until 2026-08-14" excluding the named day is
|
|
1538
|
+
* a trap sprung on everyone who reads dates the way humans do. Internally
|
|
1539
|
+
* the window is half-open `[since, until)`, so two adjacent windows share
|
|
1540
|
+
* no record.
|
|
1541
|
+
*/
|
|
1542
|
+
const now = Date.now();
|
|
1543
|
+
let relativeWindow = false;
|
|
1544
|
+
const parseWhen = (flag, endOfDay) => {
|
|
1545
|
+
const value = stringFlag(args, flag);
|
|
1546
|
+
if (value === undefined)
|
|
1547
|
+
return undefined;
|
|
1548
|
+
/**
|
|
1549
|
+
* A relative window — `7d`, `24h` — because "the last week" is what a
|
|
1550
|
+
* nightly job actually wants, and computing a date in a shell to say it
|
|
1551
|
+
* is the step that gets skipped.
|
|
1552
|
+
*
|
|
1553
|
+
* Relative to **the machine's clock, not the log's**, which is a real
|
|
1554
|
+
* difference: a log exported last month answers `--since 7d` with
|
|
1555
|
+
* nothing, and the report says so rather than reporting $0. That caveat
|
|
1556
|
+
* is stated beside the window line, because a period the reader did not
|
|
1557
|
+
* name is a period they will misread.
|
|
1558
|
+
*/
|
|
1559
|
+
const relative = /^(\d+)([dh])$/.exec(value);
|
|
1560
|
+
if (relative) {
|
|
1561
|
+
const amount = Number(relative[1]);
|
|
1562
|
+
if (amount > 0) {
|
|
1563
|
+
relativeWindow = true;
|
|
1564
|
+
const span = relative[2] === 'd' ? 86_400_000 : 3_600_000;
|
|
1565
|
+
return now - amount * span;
|
|
1566
|
+
}
|
|
1567
|
+
}
|
|
1568
|
+
if (/^\d{4}-\d{2}-\d{2}$/.test(value)) {
|
|
1569
|
+
const midnight = Date.parse(`${value}T00:00:00Z`);
|
|
1570
|
+
if (Number.isFinite(midnight))
|
|
1571
|
+
return endOfDay ? midnight + 86_400_000 : midnight;
|
|
1572
|
+
}
|
|
1573
|
+
if (value === 'now')
|
|
1574
|
+
return now;
|
|
1575
|
+
const exact = Date.parse(value);
|
|
1576
|
+
if (Number.isFinite(exact))
|
|
1577
|
+
return exact;
|
|
1578
|
+
throw new Error(t.profile.badWhen(flag, value));
|
|
1579
|
+
};
|
|
1580
|
+
const sinceMs = parseWhen('since', false);
|
|
1581
|
+
const untilMs = parseWhen('until', true);
|
|
1582
|
+
if (sinceMs !== undefined && untilMs !== undefined && sinceMs >= untilMs) {
|
|
1583
|
+
throw new Error(t.profile.sinceAfterUntil());
|
|
1584
|
+
}
|
|
1585
|
+
const windowed = sinceMs !== undefined || untilMs !== undefined;
|
|
1586
|
+
/**
|
|
1587
|
+
* How old the price table behind every dollar below is. Stated only when it
|
|
1588
|
+
* is old enough to matter: `models` and `doctor` always print the date, but
|
|
1589
|
+
* a profile is read for its figures, and the one fact that silently
|
|
1590
|
+
* invalidates all of them is a table the provider has re-priced since.
|
|
1591
|
+
* The threshold is in the sentence, not hidden here.
|
|
1592
|
+
*/
|
|
1593
|
+
const STALE_PRICING_DAYS = 45;
|
|
1594
|
+
const pricingAgeDays = reviewAgeDays(pricing.lastReviewed, new Date());
|
|
1595
|
+
const pricingStale = pricingAgeDays !== null && pricingAgeDays > STALE_PRICING_DAYS
|
|
1596
|
+
? { date: pricing.lastReviewed, days: pricingAgeDays }
|
|
1597
|
+
: null;
|
|
1598
|
+
const report = profileUsage(raw, { catalogue: pricing, label: onlyLabel, sinceMs, untilMs });
|
|
1599
|
+
if (report.total.calls === 0 && report.unpriced.calls === 0) {
|
|
1600
|
+
if (onlyLabel !== undefined || windowed) {
|
|
1601
|
+
// Diagnose against the log without the failed filter, so the error can
|
|
1602
|
+
// name what does exist instead of describing an absence.
|
|
1603
|
+
const unfiltered = profileUsage(raw, { catalogue: pricing });
|
|
1604
|
+
if (unfiltered.total.calls > 0 || unfiltered.unpriced.calls > 0) {
|
|
1605
|
+
if (onlyLabel !== undefined && !unfiltered.byLabel.some((r) => r.label === onlyLabel)) {
|
|
1606
|
+
const available = unfiltered.byLabel
|
|
1607
|
+
.map((r) => (r.label === UNLABELLED ? t.profile.unlabelled() : r.label))
|
|
1608
|
+
.join(', ');
|
|
1609
|
+
throw new Error(t.route.labelNotFound(onlyLabel, available || '—'));
|
|
1610
|
+
}
|
|
1611
|
+
if (windowed) {
|
|
1612
|
+
/**
|
|
1613
|
+
* A window that matches nothing must not become a $0 report — under
|
|
1614
|
+
* `--max-usd` it would pass a budget gate over a period the log
|
|
1615
|
+
* simply does not cover, which is the flattering non-answer. The
|
|
1616
|
+
* error names what the log *does* cover, or says it has no clock at
|
|
1617
|
+
* all, so the fix is visible in the message.
|
|
1618
|
+
*/
|
|
1619
|
+
if (unfiltered.span === null)
|
|
1620
|
+
throw new Error(t.profile.windowNeedsClock());
|
|
1621
|
+
throw new Error(`${t.profile.windowMatchesNothing(dayOf(unfiltered.span.fromMs), dayOf(unfiltered.span.toMs))}${relativeWindow ? ` ${t.profile.windowRelativeEmpty()}` : ''}`);
|
|
1622
|
+
}
|
|
1623
|
+
}
|
|
1624
|
+
}
|
|
1625
|
+
}
|
|
1626
|
+
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
1627
|
+
const pct = (share) => `${(share * 100).toFixed(1)}%`;
|
|
1628
|
+
/**
|
|
1629
|
+
* The previous log, loaded before the output paths split so the growth gate
|
|
1630
|
+
* exists under `--json` too — a CI step reads the JSON and trusts the exit
|
|
1631
|
+
* code, and a gate that only arms in the human rendering is a gate CI never
|
|
1632
|
+
* had.
|
|
1633
|
+
*/
|
|
1634
|
+
const againstPath = stringFlag(args, 'against');
|
|
1635
|
+
// The same filter on both sides: comparing one workload's bill against the
|
|
1636
|
+
// whole previous log would report every sibling workload as vanished savings.
|
|
1637
|
+
const previous = againstPath !== undefined
|
|
1638
|
+
// The same reader as the log itself, so `--against last-month.jsonl.gz`
|
|
1639
|
+
// works: a comparison that could only read one of the two formats would
|
|
1640
|
+
// be a flag that fails on exactly the rotated file it exists to read.
|
|
1641
|
+
? profileUsage(await readLog(againstPath), {
|
|
1642
|
+
catalogue: pricing,
|
|
1643
|
+
label: onlyLabel,
|
|
1644
|
+
// The same window on both sides, for the same reason as the label:
|
|
1645
|
+
// a windowed bill against an unwindowed one compares a slice to a
|
|
1646
|
+
// whole and calls the difference growth.
|
|
1647
|
+
sinceMs,
|
|
1648
|
+
untilMs,
|
|
1649
|
+
})
|
|
1650
|
+
: null;
|
|
1651
|
+
const againstDelta = previous !== null && previous.total.calls > 0
|
|
1652
|
+
? report.total.totalUsd - previous.total.totalUsd
|
|
1653
|
+
: null;
|
|
1654
|
+
/**
|
|
1655
|
+
* The same tokens at another model's rates, computed before the output paths
|
|
1656
|
+
* split so `--json` carries it too.
|
|
1657
|
+
*
|
|
1658
|
+
* An unknown id **throws** rather than printing nothing. A flag that silently
|
|
1659
|
+
* does nothing is worse than a missing feature: the reader typed a question,
|
|
1660
|
+
* got a report with no answer in it, and has no way to tell a typo from a
|
|
1661
|
+
* model this comparison had nothing to say about.
|
|
1662
|
+
*/
|
|
1663
|
+
const whatIfModel = stringFlag(args, 'what-if');
|
|
1664
|
+
const whatIf = whatIfModel !== undefined ? repriceProfile(report, whatIfModel, pricing) : null;
|
|
1665
|
+
if (whatIfModel !== undefined && whatIf === null) {
|
|
1666
|
+
throw new Error(t.profile.whatIfUnknown(whatIfModel, pricing.models.map((m) => m.id).join(', ')));
|
|
1667
|
+
}
|
|
1668
|
+
/**
|
|
1669
|
+
* The drivers of the change, per label and per model, computed once here so
|
|
1670
|
+
* the terminal, the JSON and any future rendering describe the same change.
|
|
1671
|
+
* The model half answers the question the label half cannot: "the growth is
|
|
1672
|
+
* traffic moving from Haiku to Opus" is a fact about the mix, invisible in
|
|
1673
|
+
* per-workload rows whose names did not change.
|
|
1674
|
+
*/
|
|
1675
|
+
const labelDrivers = previous !== null && previous.total.calls > 0
|
|
1676
|
+
? driversBetween(previous.byLabel.map((r) => ({ key: r.label, usd: r.breakdown.totalUsd })), report.byLabel.map((r) => ({ key: r.label, usd: r.breakdown.totalUsd })))
|
|
1677
|
+
: [];
|
|
1678
|
+
const modelDrivers = previous !== null && previous.total.calls > 0
|
|
1679
|
+
? driversBetween(previous.byModel.map((r) => ({ key: r.model, usd: r.breakdown.totalUsd })), report.byModel.map((r) => ({ key: r.model, usd: r.breakdown.totalUsd })))
|
|
1680
|
+
: [];
|
|
1681
|
+
/**
|
|
1682
|
+
* Whether the two logs share any time at all. This comparison is meant for
|
|
1683
|
+
* disjoint periods or snapshots of different systems; when both spans are
|
|
1684
|
+
* known and intersect, the same calls may sit on both sides of the
|
|
1685
|
+
* subtraction and the "growth" is partly the same money counted twice.
|
|
1686
|
+
* Only decidable when both logs carry a clock — three states, as always:
|
|
1687
|
+
* warned, clear, or unknown, and unknown stays silent rather than clear.
|
|
1688
|
+
*/
|
|
1689
|
+
const againstOverlap = previous !== null &&
|
|
1690
|
+
previous.total.calls > 0 &&
|
|
1691
|
+
previous.span !== null &&
|
|
1692
|
+
report.span !== null &&
|
|
1693
|
+
Math.min(report.span.toMs, previous.span.toMs) >=
|
|
1694
|
+
Math.max(report.span.fromMs, previous.span.fromMs)
|
|
1695
|
+
? {
|
|
1696
|
+
fromMs: Math.max(report.span.fromMs, previous.span.fromMs),
|
|
1697
|
+
toMs: Math.min(report.span.toMs, previous.span.toMs),
|
|
1698
|
+
}
|
|
1699
|
+
: null;
|
|
1700
|
+
// A gate flag that silently does nothing is not an answer — same rule as
|
|
1701
|
+
// --apply-suggestions without --suggest.
|
|
1702
|
+
if (typeof args.flags.get('max-growth-usd') === 'string' && againstPath === undefined) {
|
|
1703
|
+
throw new Error(t.profile.maxGrowthNeedsAgainst());
|
|
1704
|
+
}
|
|
1705
|
+
/**
|
|
1706
|
+
* The money gates, armed by flags and applied on every output path.
|
|
1707
|
+
*
|
|
1708
|
+
* `check` gates tokens before the money is spent; these gate the spend
|
|
1709
|
+
* itself, from the provider's own billed counts. No period is assumed —
|
|
1710
|
+
* the budget applies to exactly the log handed in, so a nightly job that
|
|
1711
|
+
* profiles yesterday's log has a daily budget without Trazum ever
|
|
1712
|
+
* guessing what a day is.
|
|
1713
|
+
*/
|
|
1714
|
+
const applyGates = () => {
|
|
1715
|
+
/**
|
|
1716
|
+
* Before any verdict: whether the gated figure is the whole bill. A gate
|
|
1717
|
+
* can only judge the money it can see, and three things hide money from
|
|
1718
|
+
* it — unreadable lines, unpriced models, and clockless calls left
|
|
1719
|
+
* outside a window. Passing on a floor is acceptable; passing on a floor
|
|
1720
|
+
* *silently* is the flattering omission this repository refuses, because
|
|
1721
|
+
* an over-budget bill with three corrupt lines would read as green.
|
|
1722
|
+
*/
|
|
1723
|
+
const anyGate = typeof args.flags.get('max-usd') === 'string' ||
|
|
1724
|
+
typeof args.flags.get('max-growth-usd') === 'string' ||
|
|
1725
|
+
typeof args.flags.get('max-cache-loss-usd') === 'string' ||
|
|
1726
|
+
typeof args.flags.get('max-day-usd') === 'string' ||
|
|
1727
|
+
config.spend !== undefined;
|
|
1728
|
+
if (anyGate) {
|
|
1729
|
+
const reasons = [];
|
|
1730
|
+
if (report.skippedLines.length > 0)
|
|
1731
|
+
reasons.push(t.profile.floorSkipped(report.skippedLines.length));
|
|
1732
|
+
if (report.unpriced.calls > 0)
|
|
1733
|
+
reasons.push(t.profile.floorUnpriced(report.unpriced.calls));
|
|
1734
|
+
if (report.timeWindow !== null && report.timeWindow.undatedExcluded > 0) {
|
|
1735
|
+
reasons.push(t.profile.floorUndated(report.timeWindow.undatedExcluded));
|
|
1736
|
+
}
|
|
1737
|
+
if (reasons.length > 0) {
|
|
1738
|
+
console.error(c.yellow(t.profile.gateOnFloor(reasons.join('; '))));
|
|
1739
|
+
}
|
|
1740
|
+
}
|
|
1741
|
+
/**
|
|
1742
|
+
* Per-workload budgets from the config — the policy in the repository
|
|
1743
|
+
* rather than in one CI invocation. Each label is gated against its own
|
|
1744
|
+
* spend in the same run, and a budgeted label with no calls in this log
|
|
1745
|
+
* is reported as **not measured**: a workload that did not appear is not
|
|
1746
|
+
* a workload that came in under budget, and printing green over an
|
|
1747
|
+
* absence is exactly the flattering direction this tool refuses.
|
|
1748
|
+
*/
|
|
1749
|
+
const byLabel = config.spend?.byLabel;
|
|
1750
|
+
if (byLabel !== undefined && !windowed) {
|
|
1751
|
+
const spent = new Map(report.byLabel.map((r) => [r.label, r.breakdown.totalUsd]));
|
|
1752
|
+
for (const [label, limit] of Object.entries(byLabel)) {
|
|
1753
|
+
const usd = spent.get(label);
|
|
1754
|
+
if (usd === undefined) {
|
|
1755
|
+
console.error(c.dim(t.profile.labelBudgetMissing(label)));
|
|
1756
|
+
continue;
|
|
1757
|
+
}
|
|
1758
|
+
if (usd > limit) {
|
|
1759
|
+
console.error(c.red(t.profile.labelBudgetFailed(label, formatUsd(usd), formatUsd(limit))));
|
|
1760
|
+
process.exitCode = 1;
|
|
1761
|
+
}
|
|
1762
|
+
else {
|
|
1763
|
+
console.error(c.dim(t.profile.labelBudgetOk(label, formatUsd(usd), formatUsd(limit))));
|
|
1764
|
+
}
|
|
1765
|
+
}
|
|
1766
|
+
}
|
|
1767
|
+
else if (byLabel !== undefined && windowed) {
|
|
1768
|
+
// A window changes what "this label spent" means, and a budget written
|
|
1769
|
+
// for a period the caller did not name would gate against a slice.
|
|
1770
|
+
console.error(c.dim(t.profile.labelBudgetWindowed()));
|
|
1771
|
+
}
|
|
1772
|
+
if (typeof args.flags.get('max-usd') === 'string' || config.spend?.maxUsd !== undefined) {
|
|
1773
|
+
const maxUsd = typeof args.flags.get('max-usd') === 'string'
|
|
1774
|
+
? numberFlag(args, 'max-usd', 0, t)
|
|
1775
|
+
: config.spend.maxUsd;
|
|
1776
|
+
if (report.total.totalUsd > maxUsd) {
|
|
1777
|
+
console.error(c.red(t.profile.maxUsdFailed(formatUsd(report.total.totalUsd), formatUsd(maxUsd))));
|
|
1778
|
+
process.exitCode = 1;
|
|
1779
|
+
}
|
|
1780
|
+
else {
|
|
1781
|
+
console.error(c.dim(t.profile.maxUsdOk(formatUsd(report.total.totalUsd), formatUsd(maxUsd))));
|
|
1782
|
+
}
|
|
1783
|
+
}
|
|
1784
|
+
if (typeof args.flags.get('max-growth-usd') === 'string' && againstDelta !== null) {
|
|
1785
|
+
const maxGrowth = numberFlag(args, 'max-growth-usd', 0, t);
|
|
1786
|
+
if (againstDelta > maxGrowth) {
|
|
1787
|
+
console.error(c.red(t.profile.maxGrowthUsdFailed(formatSignedUsd(againstDelta), formatUsd(maxGrowth))));
|
|
1788
|
+
process.exitCode = 1;
|
|
1789
|
+
}
|
|
1790
|
+
}
|
|
1791
|
+
/**
|
|
1792
|
+
* The cache gate, and it reads the worst case on purpose. A log carrying
|
|
1793
|
+
* only the flat cache-write count cannot say which TTL was paid, and the
|
|
1794
|
+
* two verdicts can straddle the limit — a gate reading the flattering
|
|
1795
|
+
* half would pass exactly the bills it exists to catch. The failure
|
|
1796
|
+
* message says which claim fired: a settled loss, or a ceiling only the
|
|
1797
|
+
* missing "cache_creation" field can settle.
|
|
1798
|
+
*/
|
|
1799
|
+
if (typeof args.flags.get('max-cache-loss-usd') === 'string') {
|
|
1800
|
+
const maxLoss = numberFlag(args, 'max-cache-loss-usd', 0, t);
|
|
1801
|
+
const gateCache = cacheEconomics(report.total);
|
|
1802
|
+
if (gateCache.deltaUsd > maxLoss) {
|
|
1803
|
+
console.error(c.red(t.profile.maxCacheLossFailed(formatUsd(gateCache.deltaUsd), formatUsd(maxLoss))));
|
|
1804
|
+
process.exitCode = 1;
|
|
1805
|
+
}
|
|
1806
|
+
else if (gateCache.worstCaseDeltaUsd > maxLoss) {
|
|
1807
|
+
console.error(c.red(t.profile.maxCacheLossWorstCase(report.total.assumedWriteTtlCalls, formatUsd(gateCache.worstCaseDeltaUsd), formatUsd(maxLoss))));
|
|
1808
|
+
process.exitCode = 1;
|
|
1809
|
+
}
|
|
1810
|
+
else {
|
|
1811
|
+
console.error(c.dim(t.profile.maxCacheLossOk(formatUsd(Math.max(0, gateCache.worstCaseDeltaUsd)), formatUsd(maxLoss))));
|
|
1812
|
+
}
|
|
1813
|
+
}
|
|
1814
|
+
/**
|
|
1815
|
+
* The per-day gate — the one a total cannot arm.
|
|
1816
|
+
*
|
|
1817
|
+
* A month at $3,000 against a $4,000 budget passes while one afternoon's
|
|
1818
|
+
* runaway agent loop burned $900 of it in four hours. `--max-usd` gates
|
|
1819
|
+
* the sum handed in; this gates the **worst single UTC day inside it**,
|
|
1820
|
+
* which is the shape a loop, a bad deploy or a retry storm actually has.
|
|
1821
|
+
*
|
|
1822
|
+
* Two refusals it inherits from the rest of the tool:
|
|
1823
|
+
*
|
|
1824
|
+
* A log with **no clock at all** cannot be judged by day, and that is an
|
|
1825
|
+
* error rather than a pass. "Not measured" is not "under budget", and a
|
|
1826
|
+
* gate that silently green-lights an unmeasurable log is worse than one
|
|
1827
|
+
* that was never armed.
|
|
1828
|
+
*
|
|
1829
|
+
* The first and last day of a log are usually **partial**, so a day under
|
|
1830
|
+
* the limit here is under it for the hours the log contains. A day *over*
|
|
1831
|
+
* the limit is over it whatever the missing hours held — the failure is
|
|
1832
|
+
* sound in both directions, the pass is a floor, and the pass message
|
|
1833
|
+
* says so when the span does not start and end on a day boundary.
|
|
1834
|
+
*/
|
|
1835
|
+
if (typeof args.flags.get('max-day-usd') === 'string') {
|
|
1836
|
+
const maxDay = numberFlag(args, 'max-day-usd', 0, t);
|
|
1837
|
+
if (report.spendByDay.length === 0) {
|
|
1838
|
+
console.error(c.red(t.profile.maxDayNoClock()));
|
|
1839
|
+
process.exitCode = 1;
|
|
1840
|
+
}
|
|
1841
|
+
else {
|
|
1842
|
+
const worst = report.spendByDay.reduce((a, b) => (b.usd > a.usd ? b : a));
|
|
1843
|
+
const suspect = worst.topLabel !== null && report.byLabel.length > 1
|
|
1844
|
+
? ` ${t.profile.dayPeakLabel(worst.topLabel === UNLABELLED ? t.profile.unlabelled() : worst.topLabel, formatUsd(worst.topLabelUsd))}`
|
|
1845
|
+
: '';
|
|
1846
|
+
if (worst.usd > maxDay) {
|
|
1847
|
+
console.error(c.red(`${t.profile.maxDayFailed(worst.day, formatUsd(worst.usd), formatUsd(maxDay))}${suspect}`));
|
|
1848
|
+
process.exitCode = 1;
|
|
1849
|
+
}
|
|
1850
|
+
else {
|
|
1851
|
+
console.error(c.dim(t.profile.maxDayOk(worst.day, formatUsd(worst.usd), formatUsd(maxDay))));
|
|
1852
|
+
/**
|
|
1853
|
+
* Calls with no clock are in the bill above and in no day below, so
|
|
1854
|
+
* the worst day is a floor by exactly that much. Said only on a
|
|
1855
|
+
* pass: a failure stands whatever the undated calls held.
|
|
1856
|
+
*/
|
|
1857
|
+
const undated = report.fieldCoverage.parsed - report.fieldCoverage.ts;
|
|
1858
|
+
if (undated > 0) {
|
|
1859
|
+
console.error(c.yellow(t.profile.maxDayUndated(n(undated))));
|
|
1860
|
+
}
|
|
1861
|
+
}
|
|
1862
|
+
}
|
|
1863
|
+
}
|
|
1864
|
+
};
|
|
1865
|
+
/**
|
|
1866
|
+
* The side files the caller asked for. Written on **both** output paths:
|
|
1867
|
+
* under --json the human rendering returns early, and the first version of
|
|
1868
|
+
* --csv-out therefore wrote nothing at all there — a flag that silently did
|
|
1869
|
+
* nothing, which is the fault this repository keeps refusing elsewhere.
|
|
1870
|
+
*/
|
|
1871
|
+
const writeSideFiles = async () => {
|
|
1872
|
+
/**
|
|
1873
|
+
* Where the "wrote to" notice goes. Under `--json`, stdout carries the
|
|
1874
|
+
* report and nothing else — a status line there turns a parseable
|
|
1875
|
+
* document into a parse error, which is how a pipeline discovers the
|
|
1876
|
+
* feature. The gates already route their verdicts to stderr for the same
|
|
1877
|
+
* reason.
|
|
1878
|
+
*/
|
|
1879
|
+
const notice = boolFlag(args, 'json')
|
|
1880
|
+
? (message) => console.error(message)
|
|
1881
|
+
: (message) => console.log(message);
|
|
1882
|
+
/**
|
|
1883
|
+
* The same report as GitHub-flavoured markdown, for a job summary or a PR
|
|
1884
|
+
* comment. Written from the same message catalogue the terminal used, because
|
|
1885
|
+
* two renderings of one finding drift the moment they are worded twice.
|
|
1886
|
+
*/
|
|
1887
|
+
const markdownOut = stringFlag(args, 'markdown-out');
|
|
1888
|
+
if (markdownOut !== undefined) {
|
|
1889
|
+
await writeFile(markdownOut, renderProfileMarkdown({
|
|
1890
|
+
report,
|
|
1891
|
+
levers,
|
|
1892
|
+
cache,
|
|
1893
|
+
t,
|
|
1894
|
+
...(windowed
|
|
1895
|
+
? { window: { since: stringFlag(args, 'since') ?? '—', until: stringFlag(args, 'until') ?? '—' } }
|
|
1896
|
+
: {}),
|
|
1897
|
+
...(pricingStale !== null ? { stalePricing: pricingStale } : {}),
|
|
1898
|
+
// The repricing, when --what-if was given: computed once above and
|
|
1899
|
+
// handed over, so the summary in a pull request cannot disagree
|
|
1900
|
+
// with the terminal about what a move would cost.
|
|
1901
|
+
...(whatIf !== null ? { whatIf } : {}),
|
|
1902
|
+
// The comparison, when there was one — the same figures and the same
|
|
1903
|
+
// drivers the terminal printed, never re-derived here.
|
|
1904
|
+
...(previous !== null
|
|
1905
|
+
? {
|
|
1906
|
+
against: {
|
|
1907
|
+
previousTotalUsd: previous.total.totalUsd,
|
|
1908
|
+
previousCalls: previous.total.calls,
|
|
1909
|
+
labelDrivers,
|
|
1910
|
+
modelDrivers: new Set([
|
|
1911
|
+
...previous.byModel.map((r) => r.model),
|
|
1912
|
+
...report.byModel.map((r) => r.model),
|
|
1913
|
+
]).size > 1
|
|
1914
|
+
? modelDrivers
|
|
1915
|
+
: [],
|
|
1916
|
+
overlap: againstOverlap !== null
|
|
1917
|
+
? { from: dayOf(againstOverlap.fromMs), to: dayOf(againstOverlap.toMs) }
|
|
1918
|
+
: null,
|
|
1919
|
+
nothingPriced: previous.total.calls === 0,
|
|
1920
|
+
},
|
|
1921
|
+
}
|
|
1922
|
+
: {}),
|
|
1923
|
+
}), 'utf8');
|
|
1924
|
+
notice(c.dim(t.report.wroteTo(markdownOut)));
|
|
1925
|
+
}
|
|
1926
|
+
/**
|
|
1927
|
+
* The same report as a spreadsheet, one row per label and model — the grain
|
|
1928
|
+
* a routing or budget decision is made at. Deliberately without a total
|
|
1929
|
+
* row: a total inside a data file is summed with the data and doubles every
|
|
1930
|
+
* figure downstream.
|
|
1931
|
+
*/
|
|
1932
|
+
const csvOut = stringFlag(args, 'csv-out');
|
|
1933
|
+
if (csvOut !== undefined) {
|
|
1934
|
+
/**
|
|
1935
|
+
* Which table the file holds. One row shape per file on purpose: a
|
|
1936
|
+
* spreadsheet that has to filter before it can sum is a spreadsheet
|
|
1937
|
+
* somebody sums wrong.
|
|
1938
|
+
*/
|
|
1939
|
+
const shape = stringFlag(args, 'csv-shape') ?? 'slice';
|
|
1940
|
+
if (shape !== 'slice' && shape !== 'day' && shape !== 'hour') {
|
|
1941
|
+
throw new Error(t.profile.badCsvShape(shape));
|
|
1942
|
+
}
|
|
1943
|
+
await writeFile(csvOut, profileToCsv(report, { unlabelled: t.profile.unlabelled(), shape }), 'utf8');
|
|
1944
|
+
notice(c.dim(t.report.wroteTo(csvOut)));
|
|
1945
|
+
}
|
|
1946
|
+
};
|
|
1947
|
+
if (boolFlag(args, 'json')) {
|
|
1948
|
+
/**
|
|
1949
|
+
* The report, plus everything the human output leads on.
|
|
1950
|
+
*
|
|
1951
|
+
* Additive rather than a reshape: `report` keeps the shape `@trazum/core`
|
|
1952
|
+
* returns. The cache verdict is included because leaving a consumer to
|
|
1953
|
+
* re-derive it means two implementations of a sign convention where positive
|
|
1954
|
+
* means *worse*, and one of them will eventually get it backwards.
|
|
1955
|
+
*
|
|
1956
|
+
* `levers` is included because it was not, and that made the flagship
|
|
1957
|
+
* section terminal-only: "What would actually move this bill" — the reason
|
|
1958
|
+
* the command exists — was invisible to any pipeline, dashboard or CI step
|
|
1959
|
+
* reading the JSON. A finding the machine-readable output omits is a finding
|
|
1960
|
+
* the reader's tooling will never surface.
|
|
1961
|
+
*/
|
|
1962
|
+
console.log(JSON.stringify({
|
|
1963
|
+
/**
|
|
1964
|
+
* The contract version, documented in docs/json-output.md and
|
|
1965
|
+
* enforced by json-contract.test.js. It changes only when a
|
|
1966
|
+
* field's meaning changes or one is removed — new findings arrive
|
|
1967
|
+
* as new keys, so a consumer that ignores unknown ones keeps
|
|
1968
|
+
* working. Without it, every dashboard built on this output has to
|
|
1969
|
+
* guess whether a missing key means "old Trazum" or "no data".
|
|
1970
|
+
*/
|
|
1971
|
+
schemaVersion: 1,
|
|
1972
|
+
...report,
|
|
1973
|
+
cache: cacheEconomics(report.total),
|
|
1974
|
+
cacheByLabel: report.byLabel.map((r) => ({
|
|
1975
|
+
label: r.label,
|
|
1976
|
+
cache: cacheEconomics(r.breakdown),
|
|
1977
|
+
})),
|
|
1978
|
+
// The provenance of every dollar above: which price table, how old.
|
|
1979
|
+
pricing: { lastReviewed: pricing.lastReviewed, ageDays: pricingAgeDays },
|
|
1980
|
+
levers: billLevers(report, { catalogue: pricing }),
|
|
1981
|
+
// Present only when --against was passed: null delta means the
|
|
1982
|
+
// previous log had nothing priced, which is a different answer from
|
|
1983
|
+
// zero growth.
|
|
1984
|
+
...(previous !== null
|
|
1985
|
+
? {
|
|
1986
|
+
against: {
|
|
1987
|
+
previousTotalUsd: previous.total.totalUsd,
|
|
1988
|
+
deltaUsd: againstDelta,
|
|
1989
|
+
// The same drivers the terminal names, as data. A finding
|
|
1990
|
+
// the machine-readable output omits is a finding the
|
|
1991
|
+
// reader's tooling will never surface.
|
|
1992
|
+
byLabel: labelDrivers,
|
|
1993
|
+
byModel: modelDrivers,
|
|
1994
|
+
},
|
|
1995
|
+
}
|
|
1996
|
+
: {}),
|
|
1997
|
+
// Present only when --what-if was passed. `sameTokensAssumed` rides
|
|
1998
|
+
// along inside it so a consumer cannot print the dollar figure
|
|
1999
|
+
// without the caveat being in the same object.
|
|
2000
|
+
...(whatIf !== null ? { whatIf } : {}),
|
|
2001
|
+
}, null, 2));
|
|
2002
|
+
await writeSideFiles();
|
|
2003
|
+
applyGates();
|
|
2004
|
+
return;
|
|
2005
|
+
}
|
|
2006
|
+
/**
|
|
2007
|
+
* Nothing priced means there is no report, not a report of zero.
|
|
2008
|
+
*
|
|
2009
|
+
* The guard was `total.calls === 0 && unpriced.calls === 0`, so a log whose every
|
|
2010
|
+
* model was unknown fell through and printed a full report built from a zeroed
|
|
2011
|
+
* total: `0 calls · $0`, four `$0 / 0.0%` rows, a meaningless "Input is 0.0% of
|
|
2012
|
+
* this bill", and — on a log containing a hundred thousand cache-read tokens —
|
|
2013
|
+
* the flatly false "Caching was never used on these calls".
|
|
2014
|
+
*
|
|
2015
|
+
* Two affirmatively wrong claims and a $0 headline for a real bill. The trailing
|
|
2016
|
+
* unpriced note was the only correct line on screen, and it was the quietest.
|
|
2017
|
+
*/
|
|
2018
|
+
if (report.total.calls === 0) {
|
|
2019
|
+
console.log();
|
|
2020
|
+
console.log(c.dim(report.unpriced.calls === 0 ? t.profile.empty() : t.profile.nothingPriced()));
|
|
2021
|
+
reportProfileGaps(report, t, n, pricingStale);
|
|
2022
|
+
return;
|
|
2023
|
+
}
|
|
2024
|
+
const shares = sharesOf(report.total);
|
|
2025
|
+
const parts = [
|
|
2026
|
+
[t.profile.partInput(), report.total.inputUsd, shares.input, report.total.inputTokens],
|
|
2027
|
+
[t.profile.partCacheRead(), report.total.cacheReadUsd, shares.cacheRead, report.total.cacheReadTokens],
|
|
2028
|
+
[t.profile.partCacheWrite(), report.total.cacheWriteUsd, shares.cacheWrite, report.total.cacheWriteTokens],
|
|
2029
|
+
[t.profile.partOutput(), report.total.outputUsd, shares.output, report.total.outputTokens],
|
|
2030
|
+
];
|
|
2031
|
+
console.log();
|
|
2032
|
+
console.log(c.bold(t.profile.heading()));
|
|
2033
|
+
console.log(` ${t.profile.spent(t.profile.calls(report.total.calls), formatUsd(report.total.totalUsd))}`);
|
|
2034
|
+
/**
|
|
2035
|
+
* The period, when the log carries a clock — stated, never extrapolated. A
|
|
2036
|
+
* span makes the reader's own monthly arithmetic valid; a per-month figure
|
|
2037
|
+
* printed from a partial month would be this tool doing the guessing it
|
|
2038
|
+
* exists to end. Partial coverage is said in the same breath, because a span
|
|
2039
|
+
* over a third of the calls silently presented as the log's period is a
|
|
2040
|
+
* figure attributed to something it does not describe.
|
|
2041
|
+
*/
|
|
2042
|
+
if (report.span !== null) {
|
|
2043
|
+
const totalParsed = report.total.calls + report.unpriced.calls;
|
|
2044
|
+
const partial = report.span.calls < totalParsed
|
|
2045
|
+
? ` ${t.profile.spanPartial(n(report.span.calls), n(totalParsed))}`
|
|
2046
|
+
: '';
|
|
2047
|
+
console.log(` ${c.dim(wrap(`${t.profile.spanLine(dayOf(report.span.fromMs), dayOf(report.span.toMs), spanDays(report.span.fromMs, report.span.toMs))}${partial}`, 74, ' '))}`);
|
|
2048
|
+
}
|
|
2049
|
+
// How many files this report covers, when it covers more than one: a total
|
|
2050
|
+
// over "the logs" that silently skipped one is wrong by an unknown amount.
|
|
2051
|
+
if (logFiles.length > 1) {
|
|
2052
|
+
console.log(` ${c.dim(wrap(t.profile.readFiles(logFiles.length, path), 74, ' '))}`);
|
|
2053
|
+
}
|
|
2054
|
+
/**
|
|
2055
|
+
* A doubled bill, said before anything is believed.
|
|
2056
|
+
*
|
|
2057
|
+
* Reading a directory of rotated logs makes double-counting easy — a log
|
|
2058
|
+
* exported twice, an overlapping export, a copy left in the folder — and
|
|
2059
|
+
* the total then reads high with nothing else able to see it. Only counted
|
|
2060
|
+
* over records with a clock, where an identical line is a claim worth
|
|
2061
|
+
* making. It states the count and the money and stops: whether it is a
|
|
2062
|
+
* double export or a genuinely busy millisecond is the reader's to know.
|
|
2063
|
+
*/
|
|
2064
|
+
if (report.duplicateLines.count > 0) {
|
|
2065
|
+
console.log(` ${c.yellow('!')} ${c.dim(wrap(t.profile.duplicateLines(report.duplicateLines.count, formatUsd(report.duplicateLines.usd)), 74, ' '))}`);
|
|
2066
|
+
}
|
|
2067
|
+
/**
|
|
2068
|
+
* The window, said before any figure is trusted as "the log": everything
|
|
2069
|
+
* below describes a slice, and a slice presented as the whole is a figure
|
|
2070
|
+
* attributed to something it does not describe. The undated count is loud —
|
|
2071
|
+
* those calls' spend is in the log and not in this report, so the window's
|
|
2072
|
+
* figures are a floor on the period, and only this line says so.
|
|
2073
|
+
*/
|
|
2074
|
+
if (report.timeWindow !== null) {
|
|
2075
|
+
console.log(` ${c.dim(wrap(t.profile.windowLine(stringFlag(args, 'since') ?? '—', stringFlag(args, 'until') ?? '—'), 74, ' '))}`);
|
|
2076
|
+
if (relativeWindow) {
|
|
2077
|
+
console.log(` ${c.dim(wrap(t.profile.windowRelative(), 74, ' '))}`);
|
|
2078
|
+
}
|
|
2079
|
+
if (report.timeWindow.undatedExcluded > 0) {
|
|
2080
|
+
console.log(` ${c.yellow(wrap(t.profile.windowUndated(report.timeWindow.undatedExcluded), 74, ' '))}`);
|
|
2081
|
+
}
|
|
2082
|
+
}
|
|
2083
|
+
console.log();
|
|
2084
|
+
// Every part, including the zero ones. A row missing because it was zero reads
|
|
2085
|
+
// as a row somebody forgot, and "you are not caching at all" is a finding.
|
|
2086
|
+
for (const [name, usd, share, tokens] of parts) {
|
|
2087
|
+
console.log(` ${c.dim(t.profile.part(name, formatUsd(usd), pct(share), n(tokens)))}`);
|
|
2088
|
+
}
|
|
2089
|
+
/**
|
|
2090
|
+
* The line the command exists for: which part of the bill to argue with.
|
|
2091
|
+
*
|
|
2092
|
+
* When output is both the biggest part and over half, the two sentences say the
|
|
2093
|
+
* same thing and the second says more — so only the second prints. Reporting a
|
|
2094
|
+
* fact twice in adjacent lines reads as a bug, and it was one.
|
|
2095
|
+
*/
|
|
2096
|
+
const [biggestName, , biggestShare] = parts.reduce((a, b) => (b[1] > a[1] ? b : a));
|
|
2097
|
+
const outputDominates = shares.output > 0.5;
|
|
2098
|
+
console.log();
|
|
2099
|
+
if (outputDominates) {
|
|
2100
|
+
console.log(` ${c.bold(wrap(t.profile.outputDominates(pct(shares.output)), 74, ' '))}`);
|
|
2101
|
+
}
|
|
2102
|
+
else {
|
|
2103
|
+
console.log(` ${c.bold(t.profile.biggestPart(biggestName, pct(biggestShare)))}`);
|
|
2104
|
+
}
|
|
2105
|
+
/**
|
|
2106
|
+
* The most expensive day, with a suspect attached.
|
|
2107
|
+
*
|
|
2108
|
+
* The shape of a bill over time is the finding the total hides: a steady $3 a
|
|
2109
|
+
* day and a quiet week broken by one $40 spike sum to the same number and call
|
|
2110
|
+
* for opposite responses. Rendered against the **median** day — a mean would
|
|
2111
|
+
* let the spike inflate its own yardstick — and loud only when it clears twice
|
|
2112
|
+
* the median, a threshold stated in the sentence rather than hidden in code.
|
|
2113
|
+
*/
|
|
2114
|
+
if (report.spendByDay.length >= 2) {
|
|
2115
|
+
const medianUsd = median(report.spendByDay.map((d) => d.usd));
|
|
2116
|
+
const peak = report.spendByDay.reduce((a, b) => (b.usd > a.usd ? b : a));
|
|
2117
|
+
if (medianUsd > 0) {
|
|
2118
|
+
const ratio = (peak.usd / medianUsd).toFixed(1);
|
|
2119
|
+
const line = t.profile.dayPeak(peak.day, formatUsd(peak.usd), ratio);
|
|
2120
|
+
const labelClause = peak.topLabel !== null && report.byLabel.length > 1
|
|
2121
|
+
? ` ${t.profile.dayPeakLabel(peak.topLabel === UNLABELLED ? t.profile.unlabelled() : peak.topLabel, formatUsd(peak.topLabelUsd))}`
|
|
2122
|
+
: '';
|
|
2123
|
+
const loud = peak.usd > 2 * medianUsd;
|
|
2124
|
+
const text = wrap(`${line}${labelClause}`, 74, ' ');
|
|
2125
|
+
console.log(` ${loud ? c.yellow(text) : c.dim(text)}`);
|
|
2126
|
+
}
|
|
2127
|
+
}
|
|
2128
|
+
/**
|
|
2129
|
+
* The shape of the day, and what it says about batching.
|
|
2130
|
+
*
|
|
2131
|
+
* Spend packed into the hours a country is awake is interactive traffic
|
|
2132
|
+
* somebody is waiting on; spend spread evenly across twenty-four is
|
|
2133
|
+
* background work — and background work is what the Batch API halves. The
|
|
2134
|
+
* measure is exact and needs no threshold to state: the **fewest hours that
|
|
2135
|
+
* hold 80% of the spend**. Two or three means concentrated; sixteen means
|
|
2136
|
+
* flat.
|
|
2137
|
+
*
|
|
2138
|
+
* It says what the shape is and stops. Whether a workload can wait is a
|
|
2139
|
+
* product decision Trazum cannot make from counts, so the sentence names
|
|
2140
|
+
* the lever and never claims the saving — the batch figure the levers
|
|
2141
|
+
* section already prints is the one with money attached.
|
|
2142
|
+
*/
|
|
2143
|
+
if (report.spendByHour.length >= 4 && report.total.totalUsd > 0) {
|
|
2144
|
+
const sorted = [...report.spendByHour].sort((a, b) => b.usd - a.usd);
|
|
2145
|
+
let covered = 0;
|
|
2146
|
+
let hoursForMost = 0;
|
|
2147
|
+
for (const hour of sorted) {
|
|
2148
|
+
covered += hour.usd;
|
|
2149
|
+
hoursForMost += 1;
|
|
2150
|
+
if (covered >= 0.8 * report.total.totalUsd)
|
|
2151
|
+
break;
|
|
2152
|
+
}
|
|
2153
|
+
const busiest = sorted
|
|
2154
|
+
.slice(0, hoursForMost)
|
|
2155
|
+
.map((hour) => hour.hour)
|
|
2156
|
+
.sort((a, b) => a - b)
|
|
2157
|
+
.map((hour) => `${String(hour).padStart(2, '0')}:00`)
|
|
2158
|
+
.join(', ');
|
|
2159
|
+
console.log();
|
|
2160
|
+
console.log(` ${c.dim(wrap(hoursForMost <= 8 ? t.profile.hoursConcentrated(n(hoursForMost), busiest) : t.profile.hoursFlat(n(hoursForMost)), 74, ' '))}`);
|
|
2161
|
+
}
|
|
2162
|
+
/**
|
|
2163
|
+
* The hit rate, and then the question the hit rate does not answer.
|
|
2164
|
+
*
|
|
2165
|
+
* `cacheNever()` is keyed off the **verdict**, not off a null hit rate. Those
|
|
2166
|
+
* two came apart on a log whose calls were entirely cache writes with no plain
|
|
2167
|
+
* input: the rate is undefined there — zero reads over zero attempts — while
|
|
2168
|
+
* caching was plainly in use, and the old branch printed "caching was never
|
|
2169
|
+
* used" over a bill made of cache writes.
|
|
2170
|
+
*/
|
|
2171
|
+
const cache = cacheEconomics(report.total);
|
|
2172
|
+
const hitRate = cacheHitRate(report.total);
|
|
2173
|
+
if (cache.verdict === 'not-attempted') {
|
|
2174
|
+
console.log(` ${c.dim(wrap(t.profile.cacheNever(), 74, ' '))}`);
|
|
2175
|
+
}
|
|
2176
|
+
else if (hitRate !== null) {
|
|
2177
|
+
console.log(` ${c.dim(t.profile.cacheHit(pct(hitRate)))}`);
|
|
2178
|
+
}
|
|
2179
|
+
/**
|
|
2180
|
+
* Whether the caching was worth doing — the one finding here that can
|
|
2181
|
+
* contradict the advice Trazum gives everywhere else.
|
|
2182
|
+
*
|
|
2183
|
+
* A cache write costs 1.25x plain input on Anthropic and 2x at the one-hour
|
|
2184
|
+
* TTL, so a prefix rebuilt faster than it is reused is billed at a premium and
|
|
2185
|
+
* returns nothing: that workload is cheaper with caching switched off. The
|
|
2186
|
+
* counterfactual is exact rather than a projection — caching changes the
|
|
2187
|
+
* multiplier on a token, never the token — so this is the one place in `profile`
|
|
2188
|
+
* where a comparison against what-might-have-been is arithmetic instead of a
|
|
2189
|
+
* guess about a prompt nobody wrote.
|
|
2190
|
+
*/
|
|
2191
|
+
/**
|
|
2192
|
+
* The losing labels, **ranked by what caching cost them** and not by bill size.
|
|
2193
|
+
*
|
|
2194
|
+
* `byLabel` arrives sorted by total spend, which is the right order for the
|
|
2195
|
+
* table above and the wrong one here: the worst cache in an estate usually sits
|
|
2196
|
+
* on a small workload, so taking the first three off a spend-ordered list meant
|
|
2197
|
+
* the biggest loser could be the one that went unnamed.
|
|
2198
|
+
*/
|
|
2199
|
+
const lostLabels = report.byLabel
|
|
2200
|
+
.map((r) => ({ row: r, cache: cacheEconomics(r.breakdown) }))
|
|
2201
|
+
.filter((r) => r.cache.verdict === 'lost-money')
|
|
2202
|
+
.sort((a, b) => b.cache.deltaUsd - a.cache.deltaUsd);
|
|
2203
|
+
const NAMED = 3;
|
|
2204
|
+
const nameOf = (row) => row.label === UNLABELLED ? t.profile.unlabelled() : row.label;
|
|
2205
|
+
/**
|
|
2206
|
+
* The names, with the ones that did not fit **counted rather than dropped**.
|
|
2207
|
+
*
|
|
2208
|
+
* The first version sliced to three silently while the money beside it was
|
|
2209
|
+
* summed over every loser — so four bleeding labels printed three names and a
|
|
2210
|
+
* figure that charged them with a fourth label's loss. Truncating is fine;
|
|
2211
|
+
* truncating without saying so is the flattering omission this repository keeps
|
|
2212
|
+
* catching itself at, and `reportProfileGaps` already had the pattern.
|
|
2213
|
+
*/
|
|
2214
|
+
const listNames = (rows) => {
|
|
2215
|
+
const names = rows.slice(0, NAMED).map((r) => nameOf(r.row)).join(', ');
|
|
2216
|
+
return rows.length <= NAMED
|
|
2217
|
+
? names
|
|
2218
|
+
: `${names} ${t.profile.andMoreLabels(rows.length - NAMED)}`;
|
|
2219
|
+
};
|
|
2220
|
+
const namedLosers = listNames(lostLabels);
|
|
2221
|
+
const bleeding = lostLabels.reduce((sum, r) => sum + r.cache.deltaUsd, 0);
|
|
2222
|
+
/**
|
|
2223
|
+
* Whether the log can settle the question at all.
|
|
2224
|
+
*
|
|
2225
|
+
* Decided before anything prints, because it governs whether the confident
|
|
2226
|
+
* sentence prints — not merely whether a caveat follows it. The first attempt
|
|
2227
|
+
* added the caveat and left the assertion above it, so the reader met `Caching
|
|
2228
|
+
* took $0.1000 off this bill` and only afterwards learned it might be a $3.65
|
|
2229
|
+
* loss. A finding a later line retracts is still a finding somebody acted on.
|
|
2230
|
+
*/
|
|
2231
|
+
const unsettled = cache.worstCaseVerdict !== cache.verdict && report.total.assumedWriteTtlCalls > 0;
|
|
2232
|
+
if (unsettled) {
|
|
2233
|
+
console.log(` ${c.yellow('!')} ${c.bold(wrap(t.profile.cacheTtlUnsettled(report.total.assumedWriteTtlCalls, formatUsd(-cache.deltaUsd), formatUsd(cache.worstCaseDeltaUsd)), 74, ' '))}`);
|
|
2234
|
+
}
|
|
2235
|
+
else if (cache.verdict === 'lost-money') {
|
|
2236
|
+
console.log(` ${c.yellow('!')} ${c.bold(wrap(t.profile.cacheLost(formatUsd(cache.deltaUsd), n(report.total.cacheWriteTokens), n(report.total.cacheReadTokens)), 74, ' '))}`);
|
|
2237
|
+
// Only when it narrows the search. One label is the total again, said twice.
|
|
2238
|
+
if (lostLabels.length > 0 && report.byLabel.length > 1) {
|
|
2239
|
+
console.log(` ${c.dim(wrap(t.profile.cacheLostBy(namedLosers), 74, ' '))}`);
|
|
2240
|
+
}
|
|
2241
|
+
}
|
|
2242
|
+
else {
|
|
2243
|
+
if (cache.verdict === 'paid-off') {
|
|
2244
|
+
console.log(` ${c.dim(wrap(t.profile.cachePaidOff(formatUsd(-cache.deltaUsd)), 74, ' '))}`);
|
|
2245
|
+
}
|
|
2246
|
+
else if (cache.verdict === 'no-difference') {
|
|
2247
|
+
console.log(` ${c.dim(wrap(t.profile.cacheNoDifference(), 74, ' '))}`);
|
|
2248
|
+
}
|
|
2249
|
+
}
|
|
2250
|
+
/**
|
|
2251
|
+
* A workload bleeding underneath a total that does not report a loss.
|
|
2252
|
+
*
|
|
2253
|
+
* The case the aggregate is actively hiding, so it prints as a warning: a cache
|
|
2254
|
+
* paying for itself on one label and losing on another nets out to a comfortable
|
|
2255
|
+
* number, and nothing else on screen would say otherwise.
|
|
2256
|
+
*
|
|
2257
|
+
* The sentence deliberately does not restate the total's verdict. It used to
|
|
2258
|
+
* open "Caching pays off overall", which this position cannot claim — it also
|
|
2259
|
+
* runs under `no-difference`, where the line immediately above has just said the
|
|
2260
|
+
* opposite, and under `unsettled`, where there is no verdict to report at all.
|
|
2261
|
+
*/
|
|
2262
|
+
if (lostLabels.length > 0 && cache.verdict !== 'lost-money') {
|
|
2263
|
+
console.log(` ${c.yellow('!')} ${c.dim(wrap(t.profile.cacheLostHidden(formatUsd(bleeding), namedLosers), 74, ' '))}`);
|
|
2264
|
+
}
|
|
2265
|
+
/**
|
|
2266
|
+
* A label that loses money only if its unstated TTL was the long one.
|
|
2267
|
+
*
|
|
2268
|
+
* The same ambiguity one level down, and it hides better here: a total whose
|
|
2269
|
+
* TTLs are mostly recorded reads as settled while one workload inside it is
|
|
2270
|
+
* entirely unstated. Listed apart from the confirmed losers because it is a
|
|
2271
|
+
* different claim — this one is conditional, and merging the two would make
|
|
2272
|
+
* every name in either list mean less.
|
|
2273
|
+
*/
|
|
2274
|
+
const maybeLostLabels = report.byLabel
|
|
2275
|
+
.map((r) => ({ row: r, cache: cacheEconomics(r.breakdown) }))
|
|
2276
|
+
.filter((r) => r.cache.verdict !== 'lost-money' && r.cache.worstCaseVerdict === 'lost-money');
|
|
2277
|
+
if (maybeLostLabels.length > 0) {
|
|
2278
|
+
console.log(` ${c.dim(wrap(t.profile.cacheTtlUnsettledLabels(listNames(maybeLostLabels)), 74, ' '))}`);
|
|
2279
|
+
}
|
|
2280
|
+
/**
|
|
2281
|
+
* Why, read from the prompt file itself — the loop `profile` could not close.
|
|
2282
|
+
*
|
|
2283
|
+
* The log carries counts, so this command can say *that* caching loses money
|
|
2284
|
+
* on a label and nothing more. `labels` in the config maps a label to the
|
|
2285
|
+
* prompt file it sends, and for each mapped label whose cache is failing —
|
|
2286
|
+
* losing money, or never attempted while money sat in cacheable input — the
|
|
2287
|
+
* file is read and the reason named: a prefix under the model's minimum,
|
|
2288
|
+
* stable tokens stranded behind the first placeholder, or a healthy file
|
|
2289
|
+
* whose problem is byte-identity between calls.
|
|
2290
|
+
*
|
|
2291
|
+
* Every sentence carries "as it is today": the file is whatever the
|
|
2292
|
+
* repository holds now, which may not be what produced the log, and a fresh
|
|
2293
|
+
* file presented as the history's explanation would be a figure attributed to
|
|
2294
|
+
* something it does not describe.
|
|
2295
|
+
*/
|
|
2296
|
+
const labelMap = config.labels ?? {};
|
|
2297
|
+
for (const { label, model: modelId, breakdown } of report.byLabelAndModel) {
|
|
2298
|
+
const file = labelMap[label];
|
|
2299
|
+
if (file === undefined)
|
|
2300
|
+
continue;
|
|
2301
|
+
const labelCache = cacheEconomics(breakdown);
|
|
2302
|
+
const failing = labelCache.verdict === 'lost-money' ||
|
|
2303
|
+
(labelCache.verdict === 'not-attempted' && breakdown.inputUsd > 0);
|
|
2304
|
+
if (!failing)
|
|
2305
|
+
continue;
|
|
2306
|
+
let text;
|
|
2307
|
+
try {
|
|
2308
|
+
text = await readFile(file, 'utf8');
|
|
2309
|
+
}
|
|
2310
|
+
catch {
|
|
2311
|
+
console.log(` ${c.dim(wrap(t.profile.labelFileMissing(label, file), 74, ' '))}`);
|
|
2312
|
+
continue;
|
|
2313
|
+
}
|
|
2314
|
+
const model = pricing.byId.get(modelId);
|
|
2315
|
+
if (!model)
|
|
2316
|
+
continue;
|
|
2317
|
+
const analysis = analyzeCachePrefix(text, estimateTokens);
|
|
2318
|
+
const minimum = model.cacheMinTokens;
|
|
2319
|
+
console.log();
|
|
2320
|
+
if (minimum !== null && analysis.stablePrefixTokens < minimum) {
|
|
2321
|
+
console.log(` ${c.dim(wrap(t.profile.labelPrefixBelowMinimum(file, n(analysis.stablePrefixTokens), n(minimum), model.displayName), 74, ' '))}`);
|
|
2322
|
+
}
|
|
2323
|
+
else if (analysis.staticTokensAfter >= 200) {
|
|
2324
|
+
console.log(` ${c.dim(wrap(t.profile.labelPrefixMovable(file, n(analysis.staticTokensAfter), n(analysis.stablePrefixTokens)), 74, ' '))}`);
|
|
2325
|
+
}
|
|
2326
|
+
else {
|
|
2327
|
+
console.log(` ${c.dim(wrap(t.profile.labelPrefixHealthy(file, n(analysis.stablePrefixTokens), n(minimum ?? 0)), 74, ' '))}`);
|
|
2328
|
+
}
|
|
2329
|
+
}
|
|
2330
|
+
/**
|
|
2331
|
+
* The token budget against what actually goes up the wire.
|
|
2332
|
+
*
|
|
2333
|
+
* `budgets` gates a prompt *file*; the log records what the *call* carried —
|
|
2334
|
+
* system prompt, retrieved context, conversation history, tool results. The
|
|
2335
|
+
* two are related only through `labels`, and when the gap is large the gate
|
|
2336
|
+
* is real but tiny: a 2,000-token budget on a workload sending 47,000
|
|
2337
|
+
* tokens a call governs four per cent of what is sent, and nobody looking
|
|
2338
|
+
* at a green build would know it.
|
|
2339
|
+
*
|
|
2340
|
+
* Only stated when both ends are known — a label mapped to a file, and a
|
|
2341
|
+
* budget covering that file — and the share is named as approximate,
|
|
2342
|
+
* because the budget counts the file's tokens with the estimator while the
|
|
2343
|
+
* log counts what the provider billed. It says which part of the bill the
|
|
2344
|
+
* gate can see, and never that the budget is wrong.
|
|
2345
|
+
*/
|
|
2346
|
+
const budgetPatterns = Object.keys(config.budgets ?? {});
|
|
2347
|
+
if (budgetPatterns.length > 0) {
|
|
2348
|
+
for (const row of report.byLabel) {
|
|
2349
|
+
const file = labelMap[row.label];
|
|
2350
|
+
if (file === undefined || row.breakdown.calls === 0)
|
|
2351
|
+
continue;
|
|
2352
|
+
const pattern = mostSpecificMatch(budgetPatterns, file);
|
|
2353
|
+
if (pattern === null)
|
|
2354
|
+
continue;
|
|
2355
|
+
const budget = config.budgets[pattern];
|
|
2356
|
+
if (budget <= 0)
|
|
2357
|
+
continue;
|
|
2358
|
+
/**
|
|
2359
|
+
* Input tokens per call over this label — every class that is billed
|
|
2360
|
+
* as input, because a cached token was still sent and still counted
|
|
2361
|
+
* against the model's window.
|
|
2362
|
+
*/
|
|
2363
|
+
const perCall = (row.breakdown.inputTokens + row.breakdown.cacheReadTokens + row.breakdown.cacheWriteTokens) /
|
|
2364
|
+
row.breakdown.calls;
|
|
2365
|
+
if (perCall <= 0)
|
|
2366
|
+
continue;
|
|
2367
|
+
const share = budget / perCall;
|
|
2368
|
+
// Only when the gap is wide enough to change what somebody believes.
|
|
2369
|
+
// A budget covering most of the call is doing its job quietly.
|
|
2370
|
+
if (share >= 0.5)
|
|
2371
|
+
continue;
|
|
2372
|
+
console.log();
|
|
2373
|
+
console.log(` ${c.yellow('!')} ${c.dim(wrap(t.profile.budgetVsWire(row.label === UNLABELLED ? t.profile.unlabelled() : row.label, file, n(budget), n(Math.round(perCall)), pct(share)), 74, ' '))}`);
|
|
2374
|
+
}
|
|
2375
|
+
}
|
|
2376
|
+
/**
|
|
2377
|
+
* Whether the TTL fits how fast the turns arrive — the mechanism behind the
|
|
2378
|
+
* verdict above, readable only when the log carries a clock and a session.
|
|
2379
|
+
*
|
|
2380
|
+
* Rendered as four verdicts plus "could not be measured", the same
|
|
2381
|
+
* three-state discipline truncation uses: a workload with cache writes and no
|
|
2382
|
+
* clock has not been cleared, and silence here would read as fine.
|
|
2383
|
+
*/
|
|
2384
|
+
const TTL_SHOWN = 3;
|
|
2385
|
+
for (const fit of report.cacheTtlFit.slice(0, TTL_SHOWN)) {
|
|
2386
|
+
const name = fit.label === UNLABELLED ? t.profile.unlabelled() : fit.label;
|
|
2387
|
+
const gap = formatGap(fit.medianGapMs);
|
|
2388
|
+
if (fit.verdict === 'expires-before-reuse') {
|
|
2389
|
+
const line = fit.medianGapMs > TTL_1H_MS
|
|
2390
|
+
? t.profile.ttlFitExpiresBoth(name, fit.modelName, gap)
|
|
2391
|
+
: t.profile.ttlFitExpires(name, fit.modelName, gap);
|
|
2392
|
+
console.log(` ${c.yellow('!')} ${c.bold(wrap(line, 74, ' '))}`);
|
|
2393
|
+
}
|
|
2394
|
+
else if (fit.verdict === 'overlong-ttl') {
|
|
2395
|
+
console.log(` ${c.yellow('!')} ${c.bold(wrap(t.profile.ttlFitOverlong(name, fit.modelName, gap, formatUsd(fit.overpayUsd)), 74, ' '))}`);
|
|
2396
|
+
}
|
|
2397
|
+
else if (fit.verdict === 'unsettled') {
|
|
2398
|
+
console.log(` ${c.dim(wrap(t.profile.ttlFitUnsettledGap(name, fit.modelName, gap), 74, ' '))}`);
|
|
2399
|
+
}
|
|
2400
|
+
else {
|
|
2401
|
+
console.log(` ${c.dim(wrap(t.profile.ttlFitFits(name, fit.modelName, gap), 74, ' '))}`);
|
|
2402
|
+
}
|
|
2403
|
+
}
|
|
2404
|
+
if (report.total.cacheWriteTokens > 0 && report.cacheTtlFit.length === 0) {
|
|
2405
|
+
console.log(` ${c.dim(wrap(t.profile.ttlFitUnmeasured(), 74, ' '))}`);
|
|
2406
|
+
}
|
|
2407
|
+
/**
|
|
2408
|
+
* Cache writes by conversations that never came back.
|
|
2409
|
+
*
|
|
2410
|
+
* Two sentences for the same tokens, and which one prints is decided by the
|
|
2411
|
+
* slice's own reads: with zero cache reads anywhere in the slice, nothing
|
|
2412
|
+
* read those writes — within the session, across sessions, at all — and the
|
|
2413
|
+
* ceiling collapses into a fact said loudly. With reads present, another
|
|
2414
|
+
* conversation sharing the prefix may have read them, the log cannot see
|
|
2415
|
+
* whose write a read hit, and the figure prints as the ceiling it is.
|
|
2416
|
+
*/
|
|
2417
|
+
const LEDGER_SHOWN = 3;
|
|
2418
|
+
if (report.singleTurnCacheWrites.length > 0) {
|
|
2419
|
+
const readsBySlice = new Map(report.byLabelAndModel.map((r) => [`${r.label}\n${r.model}`, r.breakdown.cacheReadTokens]));
|
|
2420
|
+
for (const row of report.singleTurnCacheWrites.slice(0, LEDGER_SHOWN)) {
|
|
2421
|
+
const name = row.label === UNLABELLED ? t.profile.unlabelled() : row.label;
|
|
2422
|
+
const reads = readsBySlice.get(`${row.label}\n${row.model}`) ?? 0;
|
|
2423
|
+
if (reads === 0) {
|
|
2424
|
+
console.log(` ${c.yellow('!')} ${c.bold(wrap(t.profile.singleTurnConfirmed(name, row.modelName, n(row.singleTurnSessions), n(row.sessions), formatUsd(row.singleTurnWriteUsd)), 74, ' '))}`);
|
|
2425
|
+
}
|
|
2426
|
+
else {
|
|
2427
|
+
console.log(` ${c.dim(wrap(t.profile.singleTurnCeiling(name, row.modelName, n(row.singleTurnSessions), n(row.sessions), formatUsd(row.singleTurnWriteUsd)), 74, ' '))}`);
|
|
2428
|
+
}
|
|
2429
|
+
}
|
|
2430
|
+
}
|
|
2431
|
+
/**
|
|
2432
|
+
* What one conversation costs — the question a total cannot answer, and the
|
|
2433
|
+
* one a per-seat price or a quota is set from. Median against p95, never a
|
|
2434
|
+
* mean: one runaway agent loop would drag a mean up and hide the ordinary
|
|
2435
|
+
* case, which is the figure somebody is actually pricing.
|
|
2436
|
+
*/
|
|
2437
|
+
for (const shape of report.sessionCosts.slice(0, 3)) {
|
|
2438
|
+
const name = shape.label === UNLABELLED ? t.profile.unlabelled() : shape.label;
|
|
2439
|
+
console.log();
|
|
2440
|
+
console.log(` ${c.dim(wrap(t.profile.sessionCost(name, shape.modelName, n(shape.sessions), formatUsd(shape.medianUsd), n(shape.medianTurns), formatUsd(shape.p95Usd), formatUsd(shape.maxUsd)), 74, ' '))}`);
|
|
2441
|
+
/**
|
|
2442
|
+
* The tail, when there is one. A p95 far above the median is a shape a
|
|
2443
|
+
* quota can fix; a p95 beside it is a workload that is simply expensive,
|
|
2444
|
+
* and saying "hunt the tail" there would send somebody after nothing.
|
|
2445
|
+
* The threshold is in the sentence rather than hidden here.
|
|
2446
|
+
*/
|
|
2447
|
+
if (shape.medianUsd > 0 && shape.p95Usd > 10 * shape.medianUsd) {
|
|
2448
|
+
console.log(` ${c.yellow('!')} ${c.dim(wrap(t.profile.sessionCostTail((shape.p95Usd / shape.medianUsd).toFixed(0)), 74, ' '))}`);
|
|
2449
|
+
}
|
|
2450
|
+
}
|
|
2451
|
+
/**
|
|
2452
|
+
* A total that assumed a cache-write rate is a floor, and says so.
|
|
2453
|
+
*
|
|
2454
|
+
* Anthropic's 1-hour entry costs 2x input against the 5-minute entry's 1.25x. A
|
|
2455
|
+
* log carrying only the flat `cache_creation_input_tokens` cannot say which, so
|
|
2456
|
+
* the cheaper one is used — and the flattering direction is exactly the one this
|
|
2457
|
+
* tool refuses to take quietly.
|
|
2458
|
+
*/
|
|
2459
|
+
if (report.total.assumedWriteTtlCalls > 0) {
|
|
2460
|
+
console.log(` ${c.dim(wrap(t.profile.assumedWriteTtl(report.total.assumedWriteTtlCalls), 74, ' '))}`);
|
|
2461
|
+
}
|
|
2462
|
+
/**
|
|
2463
|
+
* The section this command is for, and the answer to the fairest complaint the
|
|
2464
|
+
* product has had: on a bill of twenty thousand, the rules recover two hundred.
|
|
2465
|
+
*
|
|
2466
|
+
* That figure is right — measured, three tokens out of three hundred and six on
|
|
2467
|
+
* an ordinary support prompt. The conclusion is not that the tool is worthless
|
|
2468
|
+
* but that it had been looking at the smallest line item. Which model a call
|
|
2469
|
+
* goes to moves 40% to 80%. The Batch API moves 50% flat. Both are priced here
|
|
2470
|
+
* from the reader's own tokens, at published rates, with no modelling in
|
|
2471
|
+
* between — and printed above the breakdowns, because a lever nobody scrolls to
|
|
2472
|
+
* is a lever nobody pulls.
|
|
2473
|
+
*
|
|
2474
|
+
* The ceiling on prompt shortening prints underneath them on purpose. A 1% win
|
|
2475
|
+
* reported without saying 1% of what is not information, and this repository
|
|
2476
|
+
* would rather say the uncomfortable number itself than let somebody else
|
|
2477
|
+
* discover it.
|
|
2478
|
+
*/
|
|
2479
|
+
const levers = billLevers(report, { catalogue: pricing });
|
|
2480
|
+
console.log();
|
|
2481
|
+
console.log(c.bold(t.profile.leversHeading()));
|
|
2482
|
+
/**
|
|
2483
|
+
* Every lever below describes a mixture when nothing carries a label.
|
|
2484
|
+
*
|
|
2485
|
+
* A 2,000-call classifier and a 400-call RAG pipeline merge into one slice, and
|
|
2486
|
+
* the section then offers a single route for two workloads that need different
|
|
2487
|
+
* answers — and `trazum route` would measure one prompt against a figure
|
|
2488
|
+
* covering both. The session case already tells the reader to add the field;
|
|
2489
|
+
* this one named the row `unlabelled` and said nothing, as though that were a
|
|
2490
|
+
* workload.
|
|
2491
|
+
*/
|
|
2492
|
+
const unlabelledOnly = report.byLabel.length === 1 && report.byLabel[0].label === UNLABELLED;
|
|
2493
|
+
if (unlabelledOnly && levers.slices.length > 0) {
|
|
2494
|
+
console.log(` ${c.yellow('!')} ${c.dim(wrap(t.profile.leversUnlabelled(), 74, ' '))}`);
|
|
2495
|
+
}
|
|
2496
|
+
if (levers.slices.length === 0) {
|
|
2497
|
+
console.log(` ${c.dim(wrap(t.profile.leversNone(), 74, ' '))}`);
|
|
2498
|
+
}
|
|
2499
|
+
else {
|
|
2500
|
+
for (const slice of levers.slices.slice(0, 5)) {
|
|
2501
|
+
const label = slice.label === UNLABELLED ? t.profile.unlabelled() : slice.label;
|
|
2502
|
+
console.log();
|
|
2503
|
+
/**
|
|
2504
|
+
* The headline is the **combined** figure, and the options underneath are
|
|
2505
|
+
* the ways to reach it — not rows to add up. Batching a routed call
|
|
2506
|
+
* discounts the cheaper model's price, so listing them separately printed
|
|
2507
|
+
* $12.60 and $10.50 against a slice that had spent $21.00: a saving larger
|
|
2508
|
+
* than the bill it came from, in the flattering direction.
|
|
2509
|
+
*/
|
|
2510
|
+
console.log(` ${c.green('→')} ${c.bold(wrap(t.profile.leverSlice(label, slice.modelName, formatUsd(slice.combinedUsd), pct(slice.shareOfBill)), 74, ' '))}`);
|
|
2511
|
+
console.log(` ${c.dim(t.profile.leverCalls(t.profile.calls(slice.calls), formatUsd(slice.spentUsd)))}`);
|
|
2512
|
+
if (slice.route) {
|
|
2513
|
+
console.log(` ${c.dim('·')} ${c.dim(wrap(t.profile.leverRoute(slice.route.candidate.displayName, formatUsd(slice.route.savingUsd)), 74, ' '))}`);
|
|
2514
|
+
}
|
|
2515
|
+
if (slice.batch) {
|
|
2516
|
+
console.log(` ${c.dim('·')} ${c.dim(wrap(t.profile.leverBatch(formatUsd(slice.batch.savingUsd)), 74, ' '))}`);
|
|
2517
|
+
}
|
|
2518
|
+
// The arithmetic is exact and the quality question is untouched by it.
|
|
2519
|
+
// Naming the command is the difference between a saving and a gamble.
|
|
2520
|
+
if (slice.route) {
|
|
2521
|
+
console.log(` ${c.dim(wrap(t.profile.leverRouteVerify(slice.route.candidate.id), 74, ' '))}`);
|
|
2522
|
+
}
|
|
2523
|
+
}
|
|
2524
|
+
}
|
|
2525
|
+
console.log();
|
|
2526
|
+
console.log(` ${c.dim(wrap(t.profile.leverPromptCeiling(formatUsd(levers.promptCeilingUsd), pct(levers.promptCeilingShare)), 74, ' '))}`);
|
|
2527
|
+
/**
|
|
2528
|
+
* `--what-if <model>`: these exact calls, at another model's rates.
|
|
2529
|
+
*
|
|
2530
|
+
* The levers above pick their own candidate; this answers the question the
|
|
2531
|
+
* reader arrived with. It is multiplication, not advice, and every part of
|
|
2532
|
+
* this section is built so it cannot be read as advice:
|
|
2533
|
+
*
|
|
2534
|
+
* - the caveat line prints **before** the figure, not after it;
|
|
2535
|
+
* - calls the target's context window could not have accepted are named as
|
|
2536
|
+
* impossible rather than priced as cheap, and their money is in none of
|
|
2537
|
+
* the totals;
|
|
2538
|
+
* - spend already on the target is stated separately, because a difference
|
|
2539
|
+
* computed over money that cannot move is a percentage of the wrong
|
|
2540
|
+
* denominator.
|
|
2541
|
+
*/
|
|
2542
|
+
if (whatIf !== null) {
|
|
2543
|
+
console.log();
|
|
2544
|
+
console.log(c.bold(t.profile.whatIfHeading(whatIf.target.displayName)));
|
|
2545
|
+
console.log(` ${c.dim(wrap(t.profile.whatIfAssumption(), 74, ' '))}`);
|
|
2546
|
+
console.log();
|
|
2547
|
+
if (whatIf.slices.length === 0) {
|
|
2548
|
+
console.log(` ${c.dim(wrap(t.profile.whatIfNothingToMove(), 74, ' '))}`);
|
|
2549
|
+
}
|
|
2550
|
+
else {
|
|
2551
|
+
const cheaper = whatIf.deltaUsd < 0;
|
|
2552
|
+
const line = t.profile.whatIfTotal(formatUsd(whatIf.currentUsd), formatUsd(whatIf.targetUsd), formatUsd(Math.abs(whatIf.deltaUsd)));
|
|
2553
|
+
console.log(` ${cheaper ? c.green('→') : c.yellow('!')} ${c.bold(wrap(line, 74, ' '))}`);
|
|
2554
|
+
console.log(` ${c.dim(wrap(cheaper ? t.profile.whatIfCheaper() : t.profile.whatIfDearer(), 74, ' '))}`);
|
|
2555
|
+
for (const slice of whatIf.slices.slice(0, 5)) {
|
|
2556
|
+
const label = slice.label === UNLABELLED ? t.profile.unlabelled() : slice.label;
|
|
2557
|
+
console.log(` ${c.dim('·')} ${c.dim(wrap(t.profile.whatIfSlice(label, slice.model, formatUsd(slice.currentUsd), formatUsd(slice.targetUsd)), 74, ' '))}`);
|
|
2558
|
+
}
|
|
2559
|
+
}
|
|
2560
|
+
/**
|
|
2561
|
+
* The refusal, and it is loud. A call larger than the target's window is
|
|
2562
|
+
* not a cheaper call, and a comparison that priced it anyway would report
|
|
2563
|
+
* a saving for traffic that would have failed outright.
|
|
2564
|
+
*/
|
|
2565
|
+
for (const slice of whatIf.overContext.slice(0, 3)) {
|
|
2566
|
+
const label = slice.label === UNLABELLED ? t.profile.unlabelled() : slice.label;
|
|
2567
|
+
console.log(` ${c.yellow('!')} ${c.bold(wrap(t.profile.whatIfOverContext(label, n(slice.maxCallInputTokens), n(whatIf.target.contextWindow), formatUsd(slice.currentUsd)), 74, ' '))}`);
|
|
2568
|
+
}
|
|
2569
|
+
// Money that is already there cannot move, and leaving it out of the
|
|
2570
|
+
// totals above is only honest if the reader is told it exists.
|
|
2571
|
+
if (whatIf.alreadyOnTarget.calls > 0) {
|
|
2572
|
+
console.log(` ${c.dim(wrap(t.profile.whatIfAlreadyThere(t.profile.calls(whatIf.alreadyOnTarget.calls), formatUsd(whatIf.alreadyOnTarget.usd)), 74, ' '))}`);
|
|
2573
|
+
}
|
|
2574
|
+
// Models with no current price have no difference to state — their target
|
|
2575
|
+
// cost is knowable and the subtraction is not.
|
|
2576
|
+
if (whatIf.unpricedCalls > 0) {
|
|
2577
|
+
console.log(` ${c.dim(wrap(t.profile.whatIfUnpriced(t.profile.calls(whatIf.unpricedCalls), whatIf.unpricedModels.join(', ')), 74, ' '))}`);
|
|
2578
|
+
}
|
|
2579
|
+
}
|
|
2580
|
+
/**
|
|
2581
|
+
* What re-sending the conversation costs — the line nothing here could see.
|
|
2582
|
+
*
|
|
2583
|
+
* A chat or agent workload sends the whole conversation back on every turn, so
|
|
2584
|
+
* the input grows with the turn count and that growth is routinely the largest
|
|
2585
|
+
* item on the bill. A prompt file shows the system prompt and not the history; a
|
|
2586
|
+
* total shows the sum and not the shape.
|
|
2587
|
+
*
|
|
2588
|
+
* Reported as a **ceiling**, because part of the growth is the user's own new
|
|
2589
|
+
* messages and this reads counts rather than content, so it cannot separate the
|
|
2590
|
+
* two. Saying nothing because the exact split is unknowable would be worse: the
|
|
2591
|
+
* bound is exact, and the reader can act on it.
|
|
2592
|
+
*/
|
|
2593
|
+
if (report.conversations.length > 0) {
|
|
2594
|
+
console.log();
|
|
2595
|
+
console.log(c.bold(t.profile.historyHeading()));
|
|
2596
|
+
for (const growth of report.conversations.slice(0, 3)) {
|
|
2597
|
+
const label = growth.label === UNLABELLED ? t.profile.unlabelled() : growth.label;
|
|
2598
|
+
console.log();
|
|
2599
|
+
console.log(` ${c.bold(wrap(t.profile.historyGrowth(label, growth.modelName, n(Math.round(growth.minTurnTokens)), n(Math.round(growth.maxTurnTokens)), n(growth.longestSession)), 74, ' '))}`);
|
|
2600
|
+
console.log(` ${c.dim(wrap(t.profile.historyCeiling(formatUsd(growth.growthUsd), pct(growth.shareOfBill), formatUsd(growth.flatUsd), formatUsd(growth.inputUsd)), 74, ' '))}`);
|
|
2601
|
+
}
|
|
2602
|
+
}
|
|
2603
|
+
else if (!report.hasSessions) {
|
|
2604
|
+
/**
|
|
2605
|
+
* Not the same as "no growth". A log without a session field cannot be asked
|
|
2606
|
+
* the question at all, and silence there would read as a clean bill of health
|
|
2607
|
+
* on the line most likely to be the biggest.
|
|
2608
|
+
*/
|
|
2609
|
+
console.log();
|
|
2610
|
+
console.log(c.bold(t.profile.historyHeading()));
|
|
2611
|
+
console.log(` ${c.dim(wrap(t.profile.historyNoSessions(), 74, ' '))}`);
|
|
2612
|
+
}
|
|
2613
|
+
/**
|
|
2614
|
+
* Where the output spend concentrates — the actionable half of "output
|
|
2615
|
+
* dominates", which the headline above could only state as a total.
|
|
2616
|
+
*
|
|
2617
|
+
* Two bills with identical output spend want opposite responses. Six per cent
|
|
2618
|
+
* of calls holding half of it is a tail, and a tail has a cause worth a
|
|
2619
|
+
* morning; forty-five per cent is what evenly spread looks like, and the only
|
|
2620
|
+
* lever there is asking every answer to be shorter. The threshold between the
|
|
2621
|
+
* two is a quarter of the calls — far enough from both shapes that rounding
|
|
2622
|
+
* cannot flip the message, and stated here because it is a presentation choice,
|
|
2623
|
+
* not a measurement.
|
|
2624
|
+
*/
|
|
2625
|
+
if (report.outputShapes.length > 0) {
|
|
2626
|
+
console.log();
|
|
2627
|
+
console.log(c.bold(t.profile.outputShapeHeading()));
|
|
2628
|
+
for (const shape of report.outputShapes.slice(0, 3)) {
|
|
2629
|
+
const label = shape.label === UNLABELLED ? t.profile.unlabelled() : shape.label;
|
|
2630
|
+
const isTail = shape.heavyCallShare < 0.25;
|
|
2631
|
+
console.log();
|
|
2632
|
+
if (isTail) {
|
|
2633
|
+
console.log(` ${c.bold(wrap(t.profile.outputTail(label, shape.modelName, pct(shape.heavyCallShare), pct(shape.heavySpendShare), n(shape.aboveTokens), formatUsd(shape.outputUsd)), 74, ' '))}`);
|
|
2634
|
+
console.log(` ${c.dim(wrap(t.profile.outputTailAdvice(), 74, ' '))}`);
|
|
2635
|
+
}
|
|
2636
|
+
else {
|
|
2637
|
+
console.log(` ${c.bold(wrap(t.profile.outputFlat(label, shape.modelName, pct(shape.heavyCallShare), pct(shape.heavySpendShare), formatUsd(shape.outputUsd)), 74, ' '))}`);
|
|
2638
|
+
console.log(` ${c.dim(wrap(t.profile.outputFlatAdvice(), 74, ' '))}`);
|
|
2639
|
+
}
|
|
2640
|
+
/**
|
|
2641
|
+
* The ceilings a max_tokens cap actually wants, exact over the
|
|
2642
|
+
* histogram: every measured answer at or under the named number is
|
|
2643
|
+
* counted, none interpolated. Omitted when the covering bucket is the
|
|
2644
|
+
* open-ended last one, which has no ceiling to name honestly.
|
|
2645
|
+
*/
|
|
2646
|
+
if (shape.medianWithinTokens !== null && shape.p95WithinTokens !== null) {
|
|
2647
|
+
console.log(` ${c.dim(wrap(t.profile.outputPercentiles(n(shape.medianWithinTokens), n(shape.p95WithinTokens)), 74, ' '))}`);
|
|
2648
|
+
}
|
|
2649
|
+
}
|
|
2650
|
+
}
|
|
2651
|
+
/**
|
|
2652
|
+
* How big the calls themselves are — the other half of the bill.
|
|
2653
|
+
*
|
|
2654
|
+
* The section above describes output; on a RAG or agent workload input is
|
|
2655
|
+
* most of the invoice, and a total could only ever say "input is 63% of
|
|
2656
|
+
* this bill", which nobody can act on. The actionable question is whether
|
|
2657
|
+
* the ordinary call is large or a few calls are enormous, and those two
|
|
2658
|
+
* shapes want opposite responses: a cap on something, or a shorter prompt.
|
|
2659
|
+
*
|
|
2660
|
+
* Loud past **four times** the median — far enough from an even
|
|
2661
|
+
* distribution that a bucket boundary cannot flip the message, and stated
|
|
2662
|
+
* in the sentence rather than hidden here. Both figures are bucket
|
|
2663
|
+
* ceilings, so the ratio is coarse by construction and the copy says so.
|
|
2664
|
+
*/
|
|
2665
|
+
if (report.inputShapes.length > 0) {
|
|
2666
|
+
console.log();
|
|
2667
|
+
console.log(c.bold(t.profile.inputShapeHeading()));
|
|
2668
|
+
for (const shape of report.inputShapes.slice(0, 3)) {
|
|
2669
|
+
const label = shape.label === UNLABELLED ? t.profile.unlabelled() : shape.label;
|
|
2670
|
+
console.log();
|
|
2671
|
+
if (shape.medianWithinTokens === null || shape.p95WithinTokens === null || shape.p95OverMedian === null) {
|
|
2672
|
+
/**
|
|
2673
|
+
* The covering bucket is the open-ended last one, so there is no
|
|
2674
|
+
* ceiling to name. Said rather than skipped: a slice whose calls are
|
|
2675
|
+
* all above a million tokens is a finding, and silence would drop it.
|
|
2676
|
+
*/
|
|
2677
|
+
console.log(` ${c.bold(wrap(t.profile.inputHuge(label, shape.modelName, t.profile.calls(shape.calls), formatUsd(shape.inputUsd)), 74, ' '))}`);
|
|
2678
|
+
continue;
|
|
2679
|
+
}
|
|
2680
|
+
const skewed = shape.p95OverMedian >= 4;
|
|
2681
|
+
const line = skewed
|
|
2682
|
+
? t.profile.inputSkewed(label, shape.modelName, n(shape.medianWithinTokens), n(shape.p95WithinTokens), shape.p95OverMedian.toFixed(1), formatUsd(shape.inputUsd))
|
|
2683
|
+
: t.profile.inputEven(label, shape.modelName, n(shape.medianWithinTokens), n(shape.p95WithinTokens), formatUsd(shape.inputUsd));
|
|
2684
|
+
console.log(` ${c.bold(wrap(line, 74, ' '))}`);
|
|
2685
|
+
console.log(` ${c.dim(wrap(skewed ? t.profile.inputSkewedAdvice() : t.profile.inputEvenAdvice(), 74, ' '))}`);
|
|
2686
|
+
/**
|
|
2687
|
+
* What that size actually costs. A cache read is a tenth of input on
|
|
2688
|
+
* Anthropic, so a large slice reading almost everything from cache is a
|
|
2689
|
+
* very different bill from one paying full rate — and the token counts
|
|
2690
|
+
* alone cannot tell them apart.
|
|
2691
|
+
*/
|
|
2692
|
+
if (shape.cachedShare >= 0.5) {
|
|
2693
|
+
console.log(` ${c.dim(wrap(t.profile.inputMostlyCached(pct(shape.cachedShare)), 74, ' '))}`);
|
|
2694
|
+
}
|
|
2695
|
+
else if (shape.cachedShare < 0.1) {
|
|
2696
|
+
console.log(` ${c.dim(wrap(t.profile.inputFullRate(), 74, ' '))}`);
|
|
2697
|
+
}
|
|
2698
|
+
}
|
|
2699
|
+
}
|
|
2700
|
+
/**
|
|
2701
|
+
* The same request, sent again a moment later.
|
|
2702
|
+
*
|
|
2703
|
+
* A conversation's input grows with every turn, so two consecutive calls in
|
|
2704
|
+
* one conversation carrying the same size seconds apart is a thing going
|
|
2705
|
+
* wrong rather than a thing working — a retry after a timeout, an agent
|
|
2706
|
+
* step repeating, a loop. Loud, because the money bought nothing, and
|
|
2707
|
+
* hedged, because this reads counts and cannot see content: the sentence
|
|
2708
|
+
* says the pattern is *usually* a retry, never that it is one.
|
|
2709
|
+
*/
|
|
2710
|
+
if (report.repeatedTurns.length > 0) {
|
|
2711
|
+
console.log();
|
|
2712
|
+
console.log(c.bold(t.profile.repeatsHeading()));
|
|
2713
|
+
for (const row of report.repeatedTurns.slice(0, 3)) {
|
|
2714
|
+
const label = row.label === UNLABELLED ? t.profile.unlabelled() : row.label;
|
|
2715
|
+
console.log();
|
|
2716
|
+
console.log(` ${c.yellow('!')} ${c.bold(wrap(t.profile.repeatsFound(label, row.modelName, n(row.repeats), n(row.checkedCalls), n(Math.round(row.withinMs / 1000)), formatUsd(row.usd)), 74, ' '))}`);
|
|
2717
|
+
console.log(` ${c.dim(wrap(t.profile.repeatsAdvice(), 74, ' '))}`);
|
|
2718
|
+
}
|
|
2719
|
+
}
|
|
2720
|
+
/**
|
|
2721
|
+
* Output spend that bought answers cut off mid-generation — the one slice of
|
|
2722
|
+
* a bill that is waste without a counterpart. Paid in full, frequently
|
|
2723
|
+
* retried and billed again, and the truncated attempt bought nothing.
|
|
2724
|
+
*
|
|
2725
|
+
* Three states, kept apart on purpose: waste found, none found on a log that
|
|
2726
|
+
* measured, and a log that never recorded a stop reason at all — which gets
|
|
2727
|
+
* the missing-field message, because silence there would read as a clean bill
|
|
2728
|
+
* of health on a question the log never asked.
|
|
2729
|
+
*/
|
|
2730
|
+
if (report.total.truncatedCalls > 0 && report.total.outputUsd > 0) {
|
|
2731
|
+
console.log();
|
|
2732
|
+
console.log(` ${c.yellow('!')} ${c.bold(wrap(t.profile.truncatedWaste(t.profile.calls(report.total.truncatedCalls), formatUsd(report.total.truncatedOutputUsd), pct(report.total.truncatedOutputUsd / report.total.outputUsd)), 74, ' '))}`);
|
|
2733
|
+
/**
|
|
2734
|
+
* Which workloads are paying for it, and at what rate — the actionable
|
|
2735
|
+
* half the total hides. A 40% truncation rate is a max_tokens setting
|
|
2736
|
+
* that is simply wrong; 1% is a long tail, and the two call for opposite
|
|
2737
|
+
* responses.
|
|
2738
|
+
*
|
|
2739
|
+
* The rate is over calls that **recorded a stop reason**, never over all
|
|
2740
|
+
* calls: a workload that logs the field on half its traffic must not be
|
|
2741
|
+
* reported as though the unmeasured half completed. Both numbers print,
|
|
2742
|
+
* so the denominator is visible rather than implied.
|
|
2743
|
+
*/
|
|
2744
|
+
const truncatedLabels = report.byLabel
|
|
2745
|
+
.filter((row) => row.breakdown.truncatedCalls > 0)
|
|
2746
|
+
.sort((a, b) => b.breakdown.truncatedOutputUsd - a.breakdown.truncatedOutputUsd);
|
|
2747
|
+
if (truncatedLabels.length > 0 && report.byLabel.length > 1) {
|
|
2748
|
+
for (const row of truncatedLabels.slice(0, 3)) {
|
|
2749
|
+
const name = row.label === UNLABELLED ? t.profile.unlabelled() : row.label;
|
|
2750
|
+
console.log(` ${c.dim(wrap(t.profile.truncatedBy(name, n(row.breakdown.truncatedCalls), n(row.breakdown.stopReasonCalls), pct(row.breakdown.truncatedCalls / row.breakdown.stopReasonCalls), formatUsd(row.breakdown.truncatedOutputUsd)), 74, ' '))}`);
|
|
2751
|
+
}
|
|
2752
|
+
}
|
|
2753
|
+
/**
|
|
2754
|
+
* The ceiling the completed answers actually needed, when the output
|
|
2755
|
+
* shapes measured it: "95% of the answers that finished fit within N
|
|
2756
|
+
* tokens" is the number a max_tokens cap wants, and it sits next to the
|
|
2757
|
+
* evidence that the current cap is too low. Measured on these calls,
|
|
2758
|
+
* promised for nothing.
|
|
2759
|
+
*/
|
|
2760
|
+
const ceiling = report.outputShapes.find((shape) => shape.p95WithinTokens !== null);
|
|
2761
|
+
if (ceiling !== undefined) {
|
|
2762
|
+
console.log(` ${c.dim(wrap(t.profile.truncatedCeiling(n(ceiling.p95WithinTokens)), 74, ' '))}`);
|
|
2763
|
+
}
|
|
2764
|
+
}
|
|
2765
|
+
else if (report.total.stopReasonCalls === 0) {
|
|
2766
|
+
console.log();
|
|
2767
|
+
console.log(` ${c.dim(wrap(t.profile.truncatedNotRecorded(), 74, ' '))}`);
|
|
2768
|
+
}
|
|
2769
|
+
/**
|
|
2770
|
+
* This bill against the previous one — how spend actually gets out of hand.
|
|
2771
|
+
*
|
|
2772
|
+
* Nobody adds five thousand a month in one day; bills grow four percent a week
|
|
2773
|
+
* while every snapshot looks reasonable. This is the baseline gate the prompts
|
|
2774
|
+
* already had, applied to the money itself. **Positive means the bill grew**
|
|
2775
|
+
* (the diff convention), and every figure is between exactly these two files:
|
|
2776
|
+
* no period is assumed, so the call counts print beside the money for the
|
|
2777
|
+
* reader to judge comparability before judging the trend.
|
|
2778
|
+
*/
|
|
2779
|
+
if (previous !== null) {
|
|
2780
|
+
console.log();
|
|
2781
|
+
console.log(c.bold(t.profile.againstHeading()));
|
|
2782
|
+
if (previous.total.calls === 0) {
|
|
2783
|
+
console.log(` ${c.dim(wrap(t.profile.againstNothingPriced(), 74, ' '))}`);
|
|
2784
|
+
}
|
|
2785
|
+
else {
|
|
2786
|
+
const delta = report.total.totalUsd - previous.total.totalUsd;
|
|
2787
|
+
const growthPct = previous.total.totalUsd > 0
|
|
2788
|
+
? `${delta >= 0 ? '+' : ''}${((delta / previous.total.totalUsd) * 100).toFixed(1)}%`
|
|
2789
|
+
: '—';
|
|
2790
|
+
console.log(` ${c.bold(wrap(t.profile.againstTotals(formatUsd(previous.total.totalUsd), formatUsd(report.total.totalUsd), formatSignedUsd(delta), growthPct, t.profile.calls(previous.total.calls), t.profile.calls(report.total.calls)), 74, ' '))}`);
|
|
2791
|
+
// Overlapping spans mean part of this "growth" is the same money on
|
|
2792
|
+
// both sides of the subtraction. Said after the figure it qualifies
|
|
2793
|
+
// and before the drivers built from it.
|
|
2794
|
+
if (againstOverlap !== null) {
|
|
2795
|
+
console.log(` ${c.yellow('!')} ${c.dim(wrap(t.profile.againstOverlap(dayOf(againstOverlap.fromMs), dayOf(againstOverlap.toMs)), 74, ' '))}`);
|
|
2796
|
+
}
|
|
2797
|
+
// Drivers: per-key contribution to the change, largest magnitude first,
|
|
2798
|
+
// computed once beside the gates so no rendering derives its own.
|
|
2799
|
+
const driverLine = (d, shown) => d.was === null
|
|
2800
|
+
? t.profile.againstDriverNew(formatSignedUsd(d.delta), shown)
|
|
2801
|
+
: d.now === null
|
|
2802
|
+
? t.profile.againstDriverGone(formatSignedUsd(d.delta), shown)
|
|
2803
|
+
: t.profile.againstDriver(formatSignedUsd(d.delta), shown, formatUsd(d.was), formatUsd(d.now));
|
|
2804
|
+
console.log();
|
|
2805
|
+
for (const d of labelDrivers.slice(0, 5)) {
|
|
2806
|
+
const line = driverLine(d, d.key === UNLABELLED ? t.profile.unlabelled() : d.key);
|
|
2807
|
+
console.log(` ${d.delta > 0 ? c.yellow(line) : c.dim(line)}`);
|
|
2808
|
+
}
|
|
2809
|
+
if (labelDrivers.length > 5) {
|
|
2810
|
+
console.log(` ${c.dim(t.profile.andMoreLabels(labelDrivers.length - 5))}`);
|
|
2811
|
+
}
|
|
2812
|
+
/**
|
|
2813
|
+
* The same change, by model — where the mix moved. The label rows cannot
|
|
2814
|
+
* show it: a workload that kept its name and switched from Haiku to Opus
|
|
2815
|
+
* reads as "chat grew", and the reason is the model. Only printed when
|
|
2816
|
+
* more than one model is involved; with one model on both sides, this
|
|
2817
|
+
* section restates the totals line and says nothing new.
|
|
2818
|
+
*/
|
|
2819
|
+
const modelsInvolved = new Set([
|
|
2820
|
+
...previous.byModel.map((r) => r.model),
|
|
2821
|
+
...report.byModel.map((r) => r.model),
|
|
2822
|
+
]);
|
|
2823
|
+
if (modelDrivers.length > 0 && modelsInvolved.size > 1) {
|
|
2824
|
+
console.log();
|
|
2825
|
+
console.log(` ${c.dim(t.profile.againstByModel())}`);
|
|
2826
|
+
for (const d of modelDrivers.slice(0, 3)) {
|
|
2827
|
+
const line = driverLine(d, d.key);
|
|
2828
|
+
console.log(` ${d.delta > 0 ? c.yellow(line) : c.dim(line)}`);
|
|
2829
|
+
}
|
|
2830
|
+
}
|
|
2831
|
+
}
|
|
2832
|
+
}
|
|
2833
|
+
for (const [heading, rows] of [
|
|
2834
|
+
[t.profile.byLabelHeading(), report.byLabel.map((r) => [r.label === UNLABELLED ? t.profile.unlabelled() : r.label, r.breakdown])],
|
|
2835
|
+
[t.profile.byModelHeading(), report.byModel.map((r) => [r.model, r.breakdown])],
|
|
2836
|
+
]) {
|
|
2837
|
+
if (rows.length <= 1)
|
|
2838
|
+
continue; // One row is the total again, said twice.
|
|
2839
|
+
console.log();
|
|
2840
|
+
console.log(c.bold(heading));
|
|
2841
|
+
for (const [name, breakdown] of rows) {
|
|
2842
|
+
const share = report.total.totalUsd > 0 ? breakdown.totalUsd / report.total.totalUsd : 0;
|
|
2843
|
+
console.log(` ${t.profile.row(name, formatUsd(breakdown.totalUsd), pct(share), t.profile.calls(breakdown.calls))}`);
|
|
2844
|
+
}
|
|
2845
|
+
}
|
|
2846
|
+
/**
|
|
2847
|
+
* What this log cannot answer, and what would fix it.
|
|
2848
|
+
*
|
|
2849
|
+
* Every finding past the totals needs a field the format does not require,
|
|
2850
|
+
* and a reader who never adds them sees a report quietly missing half of
|
|
2851
|
+
* itself — with no way to tell "nothing to report" from "nothing recorded".
|
|
2852
|
+
* Named with counts rather than booleans: twelve labelled records out of
|
|
2853
|
+
* forty thousand is not a labelled log, and a boolean would call it one.
|
|
2854
|
+
*
|
|
2855
|
+
* Only fields that are actually missing are listed. A complete log gets no
|
|
2856
|
+
* section at all, because a paragraph of things that are fine is the
|
|
2857
|
+
* paragraph readers learn to skip.
|
|
2858
|
+
*/
|
|
2859
|
+
const coverage = report.fieldCoverage;
|
|
2860
|
+
if (coverage.parsed > 0) {
|
|
2861
|
+
const missing = [];
|
|
2862
|
+
const partial = (seen) => `${n(seen)}/${n(coverage.parsed)}`;
|
|
2863
|
+
if (coverage.label < coverage.parsed) {
|
|
2864
|
+
missing.push(t.profile.needsLabel(partial(coverage.label)));
|
|
2865
|
+
}
|
|
2866
|
+
if (coverage.session < coverage.parsed) {
|
|
2867
|
+
missing.push(t.profile.needsSession(partial(coverage.session)));
|
|
2868
|
+
}
|
|
2869
|
+
if (coverage.ts < coverage.parsed) {
|
|
2870
|
+
missing.push(t.profile.needsTs(partial(coverage.ts)));
|
|
2871
|
+
}
|
|
2872
|
+
if (coverage.stopReason < coverage.parsed) {
|
|
2873
|
+
missing.push(t.profile.needsStopReason(partial(coverage.stopReason)));
|
|
2874
|
+
}
|
|
2875
|
+
if (coverage.cacheWrites > 0 && coverage.cacheTtl < coverage.cacheWrites) {
|
|
2876
|
+
missing.push(t.profile.needsCacheTtl(`${n(coverage.cacheTtl)}/${n(coverage.cacheWrites)}`));
|
|
2877
|
+
}
|
|
2878
|
+
if (missing.length > 0) {
|
|
2879
|
+
console.log();
|
|
2880
|
+
console.log(c.bold(t.profile.coverageHeading()));
|
|
2881
|
+
for (const line of missing)
|
|
2882
|
+
console.log(` ${c.dim(wrap(line, 74, ' '))}`);
|
|
2883
|
+
}
|
|
2884
|
+
}
|
|
2885
|
+
reportProfileGaps(report, t, n, pricingStale);
|
|
2886
|
+
await writeSideFiles();
|
|
2887
|
+
applyGates();
|
|
2888
|
+
}
|
|
2889
|
+
/**
|
|
2890
|
+
* What the profile could not account for, said out loud.
|
|
2891
|
+
*
|
|
2892
|
+
* Separated so both the empty and the populated path print it. A total that
|
|
2893
|
+
* silently omits calls is wrong in the flattering direction, which is the fault
|
|
2894
|
+
* this repository keeps finding in itself.
|
|
2895
|
+
*/
|
|
2896
|
+
function reportProfileGaps(report, t, n, stalePricing = null) {
|
|
2897
|
+
/**
|
|
2898
|
+
* The one fact that silently invalidates every dollar above: a price table
|
|
2899
|
+
* the provider may have re-priced since. Loud, because unlike a skipped
|
|
2900
|
+
* line it does not name its own size — the error is exactly whatever the
|
|
2901
|
+
* provider changed, and only refreshing the table can say.
|
|
2902
|
+
*/
|
|
2903
|
+
if (stalePricing !== null) {
|
|
2904
|
+
console.log();
|
|
2905
|
+
console.log(` ${c.yellow('!')} ${c.dim(wrap(t.profile.pricesStale(stalePricing.date, stalePricing.days), 74, ' '))}`);
|
|
2906
|
+
}
|
|
2907
|
+
if (report.unpricedModels.length > 0) {
|
|
2908
|
+
console.log();
|
|
2909
|
+
console.log(` ${c.yellow('!')} ${c.dim(wrap(t.profile.unpriced(report.unpricedModels.join(', '), report.unpriced.calls), 74, ' '))}`);
|
|
2910
|
+
}
|
|
2911
|
+
if (report.skippedLines.length > 0) {
|
|
2912
|
+
const shown = report.skippedLines.slice(0, 5).join(', ');
|
|
2913
|
+
console.log(` ${c.dim(t.profile.skipped(report.skippedLines.length, report.skippedLines.length > 5 ? `${shown}…` : shown))}`);
|
|
2914
|
+
}
|
|
2915
|
+
}
|
|
2916
|
+
/**
|
|
2917
|
+
* `trazum route <log> --prompt-file <p> --cases <c>` — the loop the levers could
|
|
2918
|
+
* only point at.
|
|
2919
|
+
*
|
|
2920
|
+
* `profile` prices a route exactly and can say nothing whatever about whether the
|
|
2921
|
+
* cheaper model still does the job. So it printed a figure and a homework
|
|
2922
|
+
* assignment, and homework does not get done — the report said "$16.80 available,
|
|
2923
|
+
* go and test it" and the reader closed the terminal.
|
|
2924
|
+
*
|
|
2925
|
+
* This runs the test. Same prompt, two models, judged against **the expensive
|
|
2926
|
+
* model's own run-to-run variance** measured on the same cases in the same run. No
|
|
2927
|
+
* threshold anybody picked: the question is whether the cheaper model agrees with
|
|
2928
|
+
* the original more closely than the original agrees with itself.
|
|
2929
|
+
*
|
|
2930
|
+
* It costs three provider calls per case and says so before spending one of them,
|
|
2931
|
+
* exactly as `prune` does. A command that can spend somebody's money without
|
|
2932
|
+
* telling them first is a command they stop trusting.
|
|
2933
|
+
*/
|
|
2934
|
+
async function commandRoute(args, pricing, t) {
|
|
2935
|
+
const path = args.positional[0];
|
|
2936
|
+
if (path === undefined) {
|
|
2937
|
+
console.log();
|
|
2938
|
+
console.log(c.dim(wrap(t.route.noTarget(), 74, ' ')));
|
|
2939
|
+
console.log();
|
|
2940
|
+
return;
|
|
2941
|
+
}
|
|
2942
|
+
const promptPath = stringFlag(args, 'prompt-file');
|
|
2943
|
+
const casesPath = stringFlag(args, 'cases');
|
|
2944
|
+
if (!promptPath || !casesPath)
|
|
2945
|
+
throw new Error(t.route.needsPrompt());
|
|
2946
|
+
const report = profileUsage(await readFile(path, 'utf8'), { catalogue: pricing });
|
|
2947
|
+
const levers = billLevers(report, { catalogue: pricing });
|
|
2948
|
+
const wanted = stringFlag(args, 'label');
|
|
2949
|
+
/**
|
|
2950
|
+
* A `--label` nothing carries is a typo, and it gets the typo answer.
|
|
2951
|
+
*
|
|
2952
|
+
* Falling through to the generic "no route clears 1% of the bill: these calls
|
|
2953
|
+
* are already on the cheapest model of their family" asserted two falsehoods
|
|
2954
|
+
* at once when the log had a 60% route under a different name — a verdict
|
|
2955
|
+
* about calls the flag never selected.
|
|
2956
|
+
*/
|
|
2957
|
+
if (wanted !== undefined && !report.byLabel.some((r) => r.label === wanted)) {
|
|
2958
|
+
const available = report.byLabel
|
|
2959
|
+
.map((r) => (r.label === UNLABELLED ? t.profile.unlabelled() : r.label))
|
|
2960
|
+
.join(', ');
|
|
2961
|
+
console.log();
|
|
2962
|
+
console.log(c.dim(wrap(t.route.labelNotFound(wanted, available), 74, ' ')));
|
|
2963
|
+
console.log();
|
|
2964
|
+
return;
|
|
2965
|
+
}
|
|
2966
|
+
const slice = levers.slices.find((s) => s.route !== null && (wanted === undefined || s.label === wanted));
|
|
2967
|
+
if (!slice?.route) {
|
|
2968
|
+
console.log();
|
|
2969
|
+
console.log(c.dim(wrap(t.route.noRoute(), 74, ' ')));
|
|
2970
|
+
console.log();
|
|
2971
|
+
return;
|
|
2972
|
+
}
|
|
2973
|
+
const prompt = await readFile(promptPath, 'utf8');
|
|
2974
|
+
const inputs = parseCases(await readFile(casesPath, 'utf8'));
|
|
2975
|
+
if (inputs.length === 0)
|
|
2976
|
+
throw new Error(t.errors.evalNoCases(casesPath));
|
|
2977
|
+
const provider = providerFromEnv();
|
|
2978
|
+
if (!provider)
|
|
2979
|
+
throw new Error(t.errors.llmNotConfigured());
|
|
2980
|
+
/**
|
|
2981
|
+
* The candidate on the same endpoint and key, with the model swapped. Built
|
|
2982
|
+
* through the same factory rather than by hand so a provider that needs more
|
|
2983
|
+
* than a model id — a Bedrock region, a Vertex project — keeps whatever the
|
|
2984
|
+
* environment already gave it.
|
|
2985
|
+
*/
|
|
2986
|
+
const candidate = providerFromEnv({
|
|
2987
|
+
...process.env,
|
|
2988
|
+
TRAZUM_LLM_MODEL: slice.route.candidate.id,
|
|
2989
|
+
});
|
|
2990
|
+
if (!candidate)
|
|
2991
|
+
throw new Error(t.errors.llmNotConfigured());
|
|
2992
|
+
const label = slice.label === UNLABELLED ? t.profile.unlabelled() : slice.label;
|
|
2993
|
+
const worth = formatUsd(slice.route.savingUsd);
|
|
2994
|
+
console.log();
|
|
2995
|
+
console.log(` ${c.bold(t.route.picked(label, slice.modelName, slice.route.candidate.displayName, worth, `${(slice.shareOfBill * 100).toFixed(1)}%`))}`);
|
|
2996
|
+
/**
|
|
2997
|
+
* The money and the measurement have to describe the same calls.
|
|
2998
|
+
*
|
|
2999
|
+
* An unlabelled slice can hold a classifier and a RAG pipeline at once, and
|
|
3000
|
+
* this measures exactly one prompt. Attributing the verdict to a figure that
|
|
3001
|
+
* covers both is the fault this repository keeps finding in itself — a number
|
|
3002
|
+
* describing something other than what was measured. It cannot be detected from
|
|
3003
|
+
* counts, so it is stated rather than guessed at.
|
|
3004
|
+
*/
|
|
3005
|
+
if (slice.label === UNLABELLED) {
|
|
3006
|
+
console.log(` ${c.yellow('!')} ${c.dim(wrap(t.route.unlabelledSlice(), 74, ' '))}`);
|
|
3007
|
+
}
|
|
3008
|
+
console.log();
|
|
3009
|
+
console.log(` ${c.dim(wrap(t.route.willSpend(inputs.length * 3, provider.model, candidate.model), 74, ' '))}`);
|
|
3010
|
+
if (!boolFlag(args, 'yes')) {
|
|
3011
|
+
console.log(` ${c.dim(t.route.dryRun())}`);
|
|
3012
|
+
console.log();
|
|
3013
|
+
return;
|
|
3014
|
+
}
|
|
3015
|
+
console.log(` ${c.dim(t.route.running(inputs.length))}`);
|
|
3016
|
+
// Same prompt on both sides. The axis under test is the model, and passing the
|
|
3017
|
+
// prompt twice is what says so at the call site.
|
|
3018
|
+
const result = await evaluate(prompt, prompt, inputs, provider, {
|
|
3019
|
+
candidateProvider: candidate,
|
|
3020
|
+
concurrency: numberFlag(args, 'concurrency', 3, t),
|
|
3021
|
+
});
|
|
3022
|
+
if (boolFlag(args, 'json')) {
|
|
3023
|
+
console.log(JSON.stringify({ slice, evaluation: result }, null, 2));
|
|
3024
|
+
return;
|
|
3025
|
+
}
|
|
3026
|
+
const asPct = (v) => `${(v * 100).toFixed(0)}%`;
|
|
3027
|
+
console.log();
|
|
3028
|
+
console.log(` ${c.dim(wrap(t.route.agreement(asPct(result.crossAgreement), asPct(result.selfAgreement)), 74, ' '))}`);
|
|
3029
|
+
console.log();
|
|
3030
|
+
if (result.verdict === 'inconclusive') {
|
|
3031
|
+
console.log(` ${c.bold(wrap(t.route.inconclusive(), 74, ' '))}`);
|
|
3032
|
+
}
|
|
3033
|
+
else if (result.verdict === 'diverges') {
|
|
3034
|
+
console.log(` ${c.yellow('!')} ${c.bold(wrap(t.route.diverges(worth), 74, ' '))}`);
|
|
3035
|
+
}
|
|
3036
|
+
else {
|
|
3037
|
+
console.log(` ${c.green('✓')} ${c.bold(wrap(t.route.holds(worth), 74, ' '))}`);
|
|
3038
|
+
}
|
|
3039
|
+
/**
|
|
3040
|
+
* Printed on every verdict including the good one. Agreement is not
|
|
3041
|
+
* correctness: this measures whether the answers moved, not whether they were
|
|
3042
|
+
* ever right, and a green tick that let somebody forget that would be the tool
|
|
3043
|
+
* overstating what it knows.
|
|
3044
|
+
*/
|
|
3045
|
+
console.log(` ${c.dim(wrap(t.route.yours(), 74, ' '))}`);
|
|
3046
|
+
console.log();
|
|
3047
|
+
}
|
|
1307
3048
|
/**
|
|
1308
3049
|
* `trazum baseline <dir>` — record what the estate costs now.
|
|
1309
3050
|
*
|
|
@@ -2401,6 +4142,12 @@ async function main() {
|
|
|
2401
4142
|
case 'baseline':
|
|
2402
4143
|
await commandBaseline(args, config, pricing, t, locale);
|
|
2403
4144
|
break;
|
|
4145
|
+
case 'profile':
|
|
4146
|
+
await commandProfile(args, config, pricing, t);
|
|
4147
|
+
break;
|
|
4148
|
+
case 'route':
|
|
4149
|
+
await commandRoute(args, pricing, t);
|
|
4150
|
+
break;
|
|
2404
4151
|
case 'eval':
|
|
2405
4152
|
await commandEval(args, config, t, locale);
|
|
2406
4153
|
break;
|