@trazum/cli 1.10.0 → 1.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +120 -2
- package/dist/i18n/en.d.ts.map +1 -1
- package/dist/i18n/en.js +249 -4
- package/dist/i18n/en.js.map +1 -1
- package/dist/i18n/es.d.ts.map +1 -1
- package/dist/i18n/es.js +251 -4
- package/dist/i18n/es.js.map +1 -1
- package/dist/i18n/types.d.ts +499 -1
- package/dist/i18n/types.d.ts.map +1 -1
- package/dist/index.js +1586 -24
- package/dist/index.js.map +1 -1
- package/dist/markdown.d.ts +68 -0
- package/dist/markdown.d.ts.map +1 -1
- package/dist/markdown.js +329 -1
- package/dist/markdown.js.map +1 -1
- package/dist/time.d.ts +18 -0
- package/dist/time.d.ts.map +1 -0
- package/dist/time.js +32 -0
- package/dist/time.js.map +1 -0
- package/package.json +2 -2
- package/src/i18n/en.ts +370 -4
- package/src/i18n/es.ts +373 -4
- package/src/i18n/types.ts +522 -1
- package/src/index.ts +1857 -118
- package/src/markdown.ts +415 -1
- package/src/time.ts +32 -0
package/src/index.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { readFile, stat, writeFile } from 'node:fs/promises';
|
|
2
|
+
import { readdir, readFile, stat, writeFile } from 'node:fs/promises';
|
|
3
3
|
import { join, resolve as resolvePath } from 'node:path';
|
|
4
|
+
import { gunzipSync } from 'node:zlib';
|
|
4
5
|
|
|
5
6
|
import {
|
|
6
7
|
applyRewrites,
|
|
@@ -8,6 +9,9 @@ import {
|
|
|
8
9
|
BASELINE_VERSION,
|
|
9
10
|
breaches,
|
|
10
11
|
cacheableMinimum,
|
|
12
|
+
analyzeCachePrefix,
|
|
13
|
+
billLevers,
|
|
14
|
+
cacheEconomics,
|
|
11
15
|
cacheHitRate,
|
|
12
16
|
comparePrompts,
|
|
13
17
|
compareToBaseline,
|
|
@@ -15,6 +19,7 @@ import {
|
|
|
15
19
|
countTokensAnthropic,
|
|
16
20
|
DEFAULT_USAGE,
|
|
17
21
|
detectFromSource,
|
|
22
|
+
driversBetween,
|
|
18
23
|
estimateTokens,
|
|
19
24
|
evaluate,
|
|
20
25
|
extractPrompts,
|
|
@@ -29,6 +34,7 @@ import {
|
|
|
29
34
|
LOCALES,
|
|
30
35
|
MAX_BASELINE_BYTES,
|
|
31
36
|
moneyIsComparable,
|
|
37
|
+
mostSpecificMatch,
|
|
32
38
|
nearestName,
|
|
33
39
|
optimize,
|
|
34
40
|
parseBaseline,
|
|
@@ -36,6 +42,7 @@ import {
|
|
|
36
42
|
plannedCalls,
|
|
37
43
|
PRICING_LAST_REVIEWED,
|
|
38
44
|
profilePrompt,
|
|
45
|
+
profileToCsv,
|
|
39
46
|
profileUsage,
|
|
40
47
|
promptId,
|
|
41
48
|
providerFromEnv,
|
|
@@ -43,6 +50,7 @@ import {
|
|
|
43
50
|
refineWithLlm,
|
|
44
51
|
rejectionText,
|
|
45
52
|
reorderForCache,
|
|
53
|
+
repriceProfile,
|
|
46
54
|
reviewAgeDays,
|
|
47
55
|
reviewExamples,
|
|
48
56
|
RULES,
|
|
@@ -52,10 +60,12 @@ import {
|
|
|
52
60
|
suggestRewrites,
|
|
53
61
|
toOtlpMetrics,
|
|
54
62
|
toPromptfoo,
|
|
63
|
+
TTL_1H_MS,
|
|
55
64
|
UNLABELLED,
|
|
56
65
|
withExactTokenCounts,
|
|
57
66
|
} from '@trazum/core';
|
|
58
67
|
import { cacheDir, cacheStats, cachingProvider, clearCache } from './suggest-cache.js';
|
|
68
|
+
import { dayOf, formatGap, median, spanDays } from './time.js';
|
|
59
69
|
import type {
|
|
60
70
|
BaselineBreach,
|
|
61
71
|
BaselineChange,
|
|
@@ -112,6 +122,7 @@ import {
|
|
|
112
122
|
renderCheckMarkdown,
|
|
113
123
|
renderDiffMarkdown,
|
|
114
124
|
renderRankMarkdown,
|
|
125
|
+
renderProfileMarkdown,
|
|
115
126
|
} from './markdown.js';
|
|
116
127
|
import type { CliMessages } from './i18n/index.js';
|
|
117
128
|
|
|
@@ -148,6 +159,12 @@ interface Args {
|
|
|
148
159
|
}
|
|
149
160
|
|
|
150
161
|
const VALUE_FLAGS = new Set([
|
|
162
|
+
'against',
|
|
163
|
+
// `route` takes a path here, and the flag is deliberately not `--prompt`:
|
|
164
|
+
// everywhere else in this tool `--prompt` names a marked prompt *inside* a
|
|
165
|
+
// source file, and reusing it for a path would be a trap laid for the reader.
|
|
166
|
+
'prompt-file',
|
|
167
|
+
'label',
|
|
151
168
|
'level',
|
|
152
169
|
'model',
|
|
153
170
|
'calls',
|
|
@@ -158,6 +175,15 @@ const VALUE_FLAGS = new Set([
|
|
|
158
175
|
'cases',
|
|
159
176
|
'concurrency',
|
|
160
177
|
'max-growth',
|
|
178
|
+
'max-usd',
|
|
179
|
+
'max-growth-usd',
|
|
180
|
+
'max-cache-loss-usd',
|
|
181
|
+
'max-day-usd',
|
|
182
|
+
'csv-out',
|
|
183
|
+
'csv-shape',
|
|
184
|
+
'what-if',
|
|
185
|
+
'since',
|
|
186
|
+
'until',
|
|
161
187
|
'export',
|
|
162
188
|
'limit',
|
|
163
189
|
'locale',
|
|
@@ -408,7 +434,8 @@ const COMMAND_FLAGS: Record<string, string[]> = {
|
|
|
408
434
|
],
|
|
409
435
|
check: ['max-tokens', 'level', 'exact-tokens', 'markdown-out', 'baseline'],
|
|
410
436
|
baseline: ['model', 'calls', 'output-tokens', 'cache-hit-rate', 'batch', 'exact-tokens', 'out', 'o'],
|
|
411
|
-
profile: ['json', 'pricing', 'pricing-live'],
|
|
437
|
+
profile: ['json', 'pricing', 'pricing-live', 'against', 'what-if', 'markdown-out', 'csv-out', 'csv-shape', 'max-usd', 'max-growth-usd', 'max-cache-loss-usd', 'max-day-usd', 'label', 'since', 'until'],
|
|
438
|
+
route: ['prompt-file', 'cases', 'label', 'concurrency', 'json', 'yes', 'pricing', 'pricing-live'],
|
|
412
439
|
eval: ['cases', 'level', 'concurrency', 'export', 'out', 'o', 'model'],
|
|
413
440
|
prune: ['cases', 'concurrency', 'json', 'yes'],
|
|
414
441
|
diff: ['level', 'model', 'calls', 'output-tokens', 'batch', 'max-growth', 'optimized', 'markdown-out', 'all', 'prompt'],
|
|
@@ -594,6 +621,8 @@ function printReport(
|
|
|
594
621
|
tokensOnly = false,
|
|
595
622
|
host: HostEnvironment = { id: 'terminal', displayName: 'terminal', billing: 'unknown', evidence: null },
|
|
596
623
|
suggestions: { result: SuggestResult; applied: boolean; locale: Locale } | null = null,
|
|
624
|
+
/** They named a scenario, and the host is suppressing the money anyway. */
|
|
625
|
+
namedScenario = false,
|
|
597
626
|
): void {
|
|
598
627
|
const n = (value: number): string => value.toLocaleString(t.numberLocale);
|
|
599
628
|
const sourceNote =
|
|
@@ -739,7 +768,7 @@ function printReport(
|
|
|
739
768
|
//
|
|
740
769
|
// What replaces it is the thing that *is* scarce there: the context window.
|
|
741
770
|
if (tokensOnly) {
|
|
742
|
-
printTokensOnly(result, host, t, n);
|
|
771
|
+
printTokensOnly(result, host, t, n, namedScenario);
|
|
743
772
|
} else {
|
|
744
773
|
printMoney(result, t, n);
|
|
745
774
|
}
|
|
@@ -795,6 +824,25 @@ function printReport(
|
|
|
795
824
|
|
|
796
825
|
printSuggestions(suggestions, t, n);
|
|
797
826
|
printRest(result, showDiff, t, examplesReview, n);
|
|
827
|
+
|
|
828
|
+
/**
|
|
829
|
+
* Where the money actually is, said at the front door.
|
|
830
|
+
*
|
|
831
|
+
* `optimize` is the first command anybody runs, and it reports the smallest
|
|
832
|
+
* line item on the bill: measured, about 1% of a monthly figure. Everything
|
|
833
|
+
* that moves 40% to 80% — which model the call goes to, the Batch API,
|
|
834
|
+
* caching, what re-sending the conversation costs — lives in `profile`, which
|
|
835
|
+
* needs a usage log a new reader does not have and has no reason to go looking
|
|
836
|
+
* for.
|
|
837
|
+
*
|
|
838
|
+
* A tool that learned that and only said it in the command you reach last has
|
|
839
|
+
* not said it. So it prints here, once, at the end, on every run: this is the
|
|
840
|
+
* small lever, and the big ones are one file away.
|
|
841
|
+
*/
|
|
842
|
+
console.log();
|
|
843
|
+
console.log(
|
|
844
|
+
` ${c.dim(wrap(tokensOnly ? t.report.beyondThisPromptTokensOnly() : t.report.beyondThisPrompt(), 74, ' '))}`,
|
|
845
|
+
);
|
|
798
846
|
}
|
|
799
847
|
|
|
800
848
|
/** The cost section, for anyone billed by the token. */
|
|
@@ -847,6 +895,8 @@ function printTokensOnly(
|
|
|
847
895
|
host: HostEnvironment,
|
|
848
896
|
t: CliMessages,
|
|
849
897
|
n: (v: number) => string,
|
|
898
|
+
/** Whether they named a scenario while the money was being withheld. */
|
|
899
|
+
namedScenario = false,
|
|
850
900
|
): void {
|
|
851
901
|
const model = getModel(result.usage.model);
|
|
852
902
|
const saved = result.tokensBefore - result.tokensAfter;
|
|
@@ -866,20 +916,39 @@ function printTokensOnly(
|
|
|
866
916
|
console.log();
|
|
867
917
|
console.log(` ${c.green(t.report.tokensSaved(n(saved)))}`);
|
|
868
918
|
|
|
869
|
-
|
|
919
|
+
/**
|
|
920
|
+
* Share of the window, which is what a saved token is actually worth here.
|
|
921
|
+
*
|
|
922
|
+
* A 225-token prompt against a million-token window printed `0.0% → 0.0%`: a
|
|
923
|
+
* line whose whole job is to say what a token buys, saying nothing twice. When
|
|
924
|
+
* both sides round to the same figure the honest statement is the other one —
|
|
925
|
+
* that the window is not the constraint on this prompt.
|
|
926
|
+
*/
|
|
870
927
|
const share = (tokens: number): string =>
|
|
871
928
|
`${((tokens / model.contextWindow) * 100).toFixed(1)}%`;
|
|
929
|
+
const before = share(result.tokensBefore);
|
|
930
|
+
const after = share(result.tokensAfter);
|
|
931
|
+
/**
|
|
932
|
+
* Three cases, and the first version had two.
|
|
933
|
+
*
|
|
934
|
+
* Equal shares mean either "this prompt is nothing against a million tokens" or
|
|
935
|
+
* "this prompt is 10% of the window and one token did not move it". Using the
|
|
936
|
+
* negligible message for both told a reader holding a tenth of a Haiku window
|
|
937
|
+
* that they were under a tenth of a percent — off by two orders of magnitude,
|
|
938
|
+
* on a line whose only job is to size the prompt against the window.
|
|
939
|
+
*/
|
|
940
|
+
const unchanged = before === after;
|
|
941
|
+
const negligible = after === '0.0%';
|
|
872
942
|
console.log(
|
|
873
943
|
` ${c.dim(
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
),
|
|
944
|
+
!unchanged
|
|
945
|
+
? t.report.windowUse(before, after, model.displayName, n(model.contextWindow))
|
|
946
|
+
: negligible
|
|
947
|
+
? t.report.windowNegligible(n(result.tokensAfter), model.displayName, n(model.contextWindow))
|
|
948
|
+
: t.report.windowUnmoved(after, model.displayName, n(model.contextWindow)),
|
|
880
949
|
)}`,
|
|
881
950
|
);
|
|
882
|
-
console.log(` ${c.dim(t.report.tokensOnlyCost())}`);
|
|
951
|
+
console.log(` ${c.dim(namedScenario ? t.report.tokensOnlyAskedFor() : t.report.tokensOnlyCost())}`);
|
|
883
952
|
}
|
|
884
953
|
|
|
885
954
|
/**
|
|
@@ -1514,11 +1583,26 @@ async function commandOptimize(
|
|
|
1514
1583
|
const tokensOnly = boolFlag(args, 'cost')
|
|
1515
1584
|
? false
|
|
1516
1585
|
: boolFlag(args, 'tokens-only') || host.billing === 'subscription';
|
|
1586
|
+
/**
|
|
1587
|
+
* Whether they named a scenario while the money was being withheld.
|
|
1588
|
+
*
|
|
1589
|
+
* Not a reason to start printing dollars — `--cost` is the documented way to
|
|
1590
|
+
* ask, and `--calls` is a scenario parameter with a default that several
|
|
1591
|
+
* commands take purely to size a finding. Making it imply `--cost` would hand
|
|
1592
|
+
* dollar figures to somebody who put `--calls` in an alias precisely because
|
|
1593
|
+
* they had configured the tool not to show them.
|
|
1594
|
+
*
|
|
1595
|
+
* It is a reason to stop answering with a generic hint. Somebody who typed
|
|
1596
|
+
* `--calls 50000` and read "pass --cost if this prompt is bound for a metered
|
|
1597
|
+
* API" has been told to do a thing they plainly just tried to do.
|
|
1598
|
+
*/
|
|
1599
|
+
const namedScenario = args.flags.has('calls') || args.flags.has('output-tokens');
|
|
1517
1600
|
|
|
1518
1601
|
printReport(result, boolFlag(args, 'diff'), t, examplesReview, reorder, tokensOnly, host,
|
|
1519
1602
|
suggestions
|
|
1520
1603
|
? { result: suggestions, applied: boolFlag(args, 'apply-suggestions'), locale }
|
|
1521
1604
|
: null,
|
|
1605
|
+
namedScenario,
|
|
1522
1606
|
);
|
|
1523
1607
|
if (outPath) {
|
|
1524
1608
|
console.log(c.dim(t.report.wroteTo(outPath)));
|
|
@@ -1844,133 +1928,1585 @@ async function checkEmbedded(
|
|
|
1844
1928
|
}
|
|
1845
1929
|
}
|
|
1846
1930
|
|
|
1847
|
-
console.log();
|
|
1848
|
-
if (!ok) process.exitCode = 1;
|
|
1849
|
-
}
|
|
1850
|
-
|
|
1851
|
-
/**
|
|
1852
|
-
* The scenario a baseline is recorded under, and the money it implies.
|
|
1853
|
-
*
|
|
1854
|
-
* Shared by `baseline` and the gate so both compute the monthly figure the same
|
|
1855
|
-
* way. `computeSavings` is asked for a before/after where both sides are the
|
|
1856
|
-
* same token count, because what is wanted here is the cost of a total, not a
|
|
1857
|
-
* saving — `perMonth.before.totalUsd` is that number.
|
|
1858
|
-
*/
|
|
1859
|
-
function monthlyCostOf(tokens: number, usage: UsageProfile, pricing: PricingCatalogue): number {
|
|
1860
|
-
return computeSavings(tokens, tokens, usage, new Date(), pricing).perMonth.before.totalUsd;
|
|
1861
|
-
}
|
|
1862
|
-
|
|
1863
|
-
/** Today, as the ISO date a baseline records. */
|
|
1864
|
-
function isoDate(): string {
|
|
1865
|
-
return new Date().toISOString().slice(0, 10);
|
|
1866
|
-
}
|
|
1867
|
-
|
|
1868
|
-
/**
|
|
1869
|
-
* `trazum profile <log.jsonl>` — where the money actually went.
|
|
1870
|
-
*
|
|
1871
|
-
* Every other command in this file reads a prompt and reasons forward about what
|
|
1872
|
-
* it would cost. This one reads what the provider charged and reasons backward,
|
|
1873
|
-
* and it exists because the forward direction can only see the smallest line item:
|
|
1874
|
-
* on an ordinary support prompt the rules recover about 1% of the monthly figure
|
|
1875
|
-
* while output alone was 87% of it.
|
|
1876
|
-
*
|
|
1877
|
-
* **Money is never suppressed here, unlike every other report.** The rest of the
|
|
1878
|
-
* CLI hides dollar figures on a subscription host, because a saving quoted to
|
|
1879
|
-
* somebody on a flat plan is money that does not exist. This log is a record of
|
|
1880
|
-
* metered API calls somebody was actually billed for — the bill exists wherever
|
|
1881
|
-
* Trazum happens to be running, so the host has no bearing on it.
|
|
1882
|
-
*/
|
|
1883
|
-
async function commandProfile(args: Args, pricing: PricingCatalogue, t: CliMessages): Promise<void> {
|
|
1884
|
-
const path = args.positional[0];
|
|
1885
|
-
if (path === undefined) {
|
|
1931
|
+
console.log();
|
|
1932
|
+
if (!ok) process.exitCode = 1;
|
|
1933
|
+
}
|
|
1934
|
+
|
|
1935
|
+
/**
|
|
1936
|
+
* The scenario a baseline is recorded under, and the money it implies.
|
|
1937
|
+
*
|
|
1938
|
+
* Shared by `baseline` and the gate so both compute the monthly figure the same
|
|
1939
|
+
* way. `computeSavings` is asked for a before/after where both sides are the
|
|
1940
|
+
* same token count, because what is wanted here is the cost of a total, not a
|
|
1941
|
+
* saving — `perMonth.before.totalUsd` is that number.
|
|
1942
|
+
*/
|
|
1943
|
+
function monthlyCostOf(tokens: number, usage: UsageProfile, pricing: PricingCatalogue): number {
|
|
1944
|
+
return computeSavings(tokens, tokens, usage, new Date(), pricing).perMonth.before.totalUsd;
|
|
1945
|
+
}
|
|
1946
|
+
|
|
1947
|
+
/** Today, as the ISO date a baseline records. */
|
|
1948
|
+
function isoDate(): string {
|
|
1949
|
+
return new Date().toISOString().slice(0, 10);
|
|
1950
|
+
}
|
|
1951
|
+
|
|
1952
|
+
/**
|
|
1953
|
+
* `trazum profile <log.jsonl>` — where the money actually went.
|
|
1954
|
+
*
|
|
1955
|
+
* Every other command in this file reads a prompt and reasons forward about what
|
|
1956
|
+
* it would cost. This one reads what the provider charged and reasons backward,
|
|
1957
|
+
* and it exists because the forward direction can only see the smallest line item:
|
|
1958
|
+
* on an ordinary support prompt the rules recover about 1% of the monthly figure
|
|
1959
|
+
* while output alone was 87% of it.
|
|
1960
|
+
*
|
|
1961
|
+
* **Money is never suppressed here, unlike every other report.** The rest of the
|
|
1962
|
+
* CLI hides dollar figures on a subscription host, because a saving quoted to
|
|
1963
|
+
* somebody on a flat plan is money that does not exist. This log is a record of
|
|
1964
|
+
* metered API calls somebody was actually billed for — the bill exists wherever
|
|
1965
|
+
* Trazum happens to be running, so the host has no bearing on it.
|
|
1966
|
+
*/
|
|
1967
|
+
async function commandProfile(args: Args, config: TrazumConfig, pricing: PricingCatalogue, t: CliMessages): Promise<void> {
|
|
1968
|
+
const path = args.positional[0];
|
|
1969
|
+
if (path === undefined) {
|
|
1970
|
+
console.log();
|
|
1971
|
+
console.log(c.dim(wrap(t.profile.noTarget(), 74, ' ')));
|
|
1972
|
+
console.log();
|
|
1973
|
+
return;
|
|
1974
|
+
}
|
|
1975
|
+
|
|
1976
|
+
/**
|
|
1977
|
+
* A log, or a directory of them.
|
|
1978
|
+
*
|
|
1979
|
+
* Usage logs rotate: `logs/2026-08-01.jsonl`, `logs/2026-08-02.jsonl`, one
|
|
1980
|
+
* per day for a month. Making somebody `cat` them together before a profile
|
|
1981
|
+
* will read them is a setup cost that gets a tool skipped, and doing it for
|
|
1982
|
+
* them is a directory listing.
|
|
1983
|
+
*
|
|
1984
|
+
* Files are read in name order — which for dated names is time order — and
|
|
1985
|
+
* how many were read is stated, because a report over "the logs" that
|
|
1986
|
+
* silently skipped one is a total that is wrong by an unknown amount. A
|
|
1987
|
+
* directory holding nothing readable is an error naming what it looked for,
|
|
1988
|
+
* not an empty report.
|
|
1989
|
+
*/
|
|
1990
|
+
const LOG_EXTENSIONS = ['.jsonl', '.ndjson', '.log', '.json'];
|
|
1991
|
+
/**
|
|
1992
|
+
* The same names, gzipped — which is what a rotated log actually looks like
|
|
1993
|
+
* a day after it rotates.
|
|
1994
|
+
*
|
|
1995
|
+
* `logrotate`, Docker's json-file driver and every cloud log export compress
|
|
1996
|
+
* yesterday's file, so a directory of a month's logs is one plain file and
|
|
1997
|
+
* twenty-nine `.gz` ones. Reading only the plain one and saying nothing
|
|
1998
|
+
* would report a month's bill from a day of it, in the flattering
|
|
1999
|
+
* direction, which is exactly the failure directory mode was added to
|
|
2000
|
+
* prevent.
|
|
2001
|
+
*/
|
|
2002
|
+
const GZ_EXTENSIONS = LOG_EXTENSIONS.map((ext) => `${ext}.gz`);
|
|
2003
|
+
const READABLE = [...LOG_EXTENSIONS, ...GZ_EXTENSIONS];
|
|
2004
|
+
const target = await stat(path).catch(() => null);
|
|
2005
|
+
let logFiles: string[] = [path];
|
|
2006
|
+
if (target?.isDirectory()) {
|
|
2007
|
+
const entries = await readdir(path, { withFileTypes: true });
|
|
2008
|
+
logFiles = entries
|
|
2009
|
+
.filter((entry) => entry.isFile() && READABLE.some((ext) => entry.name.endsWith(ext)))
|
|
2010
|
+
.map((entry) => join(path, entry.name))
|
|
2011
|
+
.sort((a, b) => a.localeCompare(b));
|
|
2012
|
+
if (logFiles.length === 0) {
|
|
2013
|
+
throw new Error(t.profile.noLogsInDirectory(path, READABLE.join(', ')));
|
|
2014
|
+
}
|
|
2015
|
+
}
|
|
2016
|
+
/**
|
|
2017
|
+
* Gzipped files are decompressed in memory; everything else is read as text.
|
|
2018
|
+
*
|
|
2019
|
+
* Decided by **extension**, not by sniffing the first two bytes: a file
|
|
2020
|
+
* named `.jsonl` whose contents happen to start with 0x1f8b is far more
|
|
2021
|
+
* likely to be a corrupt log than a mislabelled archive, and silently
|
|
2022
|
+
* treating it as one would turn a diagnosable error into an empty report.
|
|
2023
|
+
*
|
|
2024
|
+
* A `.gz` that will not decompress is an error naming the file. The
|
|
2025
|
+
* alternative — skipping it — is a total quietly missing a day, which is
|
|
2026
|
+
* the failure this repository refuses in every other place it can occur.
|
|
2027
|
+
*/
|
|
2028
|
+
const readLog = async (file: string): Promise<string> => {
|
|
2029
|
+
if (!file.endsWith('.gz')) return readFile(file, 'utf8');
|
|
2030
|
+
const compressed = await readFile(file);
|
|
2031
|
+
try {
|
|
2032
|
+
return gunzipSync(compressed).toString('utf8');
|
|
2033
|
+
} catch (error) {
|
|
2034
|
+
throw new Error(t.profile.badGzip(file, error instanceof Error ? error.message : String(error)));
|
|
2035
|
+
}
|
|
2036
|
+
};
|
|
2037
|
+
const logTexts = await Promise.all(logFiles.map((file) => readLog(file)));
|
|
2038
|
+
// A file that does not end in a newline would otherwise glue its last record
|
|
2039
|
+
// to the next file's first one, and both would be reported as unreadable.
|
|
2040
|
+
const raw = logTexts.map((text) => (text.endsWith('\n') ? text : `${text}\n`)).join('');
|
|
2041
|
+
/**
|
|
2042
|
+
* The drill-down. A label that matches nothing is an error naming the labels
|
|
2043
|
+
* that exist — the route command's rule, for the route command's reason: a
|
|
2044
|
+
* report over zero calls silently filtered would read as "this workload is
|
|
2045
|
+
* free".
|
|
2046
|
+
*/
|
|
2047
|
+
const onlyLabel = stringFlag(args, 'label');
|
|
2048
|
+
/**
|
|
2049
|
+
* The drill-down in time. `--since`/`--until` take a UTC day or a full
|
|
2050
|
+
* timestamp; a bare day means the whole of it — since its first instant,
|
|
2051
|
+
* until its last — because "--until 2026-08-14" excluding the named day is
|
|
2052
|
+
* a trap sprung on everyone who reads dates the way humans do. Internally
|
|
2053
|
+
* the window is half-open `[since, until)`, so two adjacent windows share
|
|
2054
|
+
* no record.
|
|
2055
|
+
*/
|
|
2056
|
+
const now = Date.now();
|
|
2057
|
+
let relativeWindow = false;
|
|
2058
|
+
const parseWhen = (flag: string, endOfDay: boolean): number | undefined => {
|
|
2059
|
+
const value = stringFlag(args, flag);
|
|
2060
|
+
if (value === undefined) return undefined;
|
|
2061
|
+
/**
|
|
2062
|
+
* A relative window — `7d`, `24h` — because "the last week" is what a
|
|
2063
|
+
* nightly job actually wants, and computing a date in a shell to say it
|
|
2064
|
+
* is the step that gets skipped.
|
|
2065
|
+
*
|
|
2066
|
+
* Relative to **the machine's clock, not the log's**, which is a real
|
|
2067
|
+
* difference: a log exported last month answers `--since 7d` with
|
|
2068
|
+
* nothing, and the report says so rather than reporting $0. That caveat
|
|
2069
|
+
* is stated beside the window line, because a period the reader did not
|
|
2070
|
+
* name is a period they will misread.
|
|
2071
|
+
*/
|
|
2072
|
+
const relative = /^(\d+)([dh])$/.exec(value);
|
|
2073
|
+
if (relative) {
|
|
2074
|
+
const amount = Number(relative[1]);
|
|
2075
|
+
if (amount > 0) {
|
|
2076
|
+
relativeWindow = true;
|
|
2077
|
+
const span = relative[2] === 'd' ? 86_400_000 : 3_600_000;
|
|
2078
|
+
return now - amount * span;
|
|
2079
|
+
}
|
|
2080
|
+
}
|
|
2081
|
+
if (/^\d{4}-\d{2}-\d{2}$/.test(value)) {
|
|
2082
|
+
const midnight = Date.parse(`${value}T00:00:00Z`);
|
|
2083
|
+
if (Number.isFinite(midnight)) return endOfDay ? midnight + 86_400_000 : midnight;
|
|
2084
|
+
}
|
|
2085
|
+
if (value === 'now') return now;
|
|
2086
|
+
const exact = Date.parse(value);
|
|
2087
|
+
if (Number.isFinite(exact)) return exact;
|
|
2088
|
+
throw new Error(t.profile.badWhen(flag, value));
|
|
2089
|
+
};
|
|
2090
|
+
const sinceMs = parseWhen('since', false);
|
|
2091
|
+
const untilMs = parseWhen('until', true);
|
|
2092
|
+
if (sinceMs !== undefined && untilMs !== undefined && sinceMs >= untilMs) {
|
|
2093
|
+
throw new Error(t.profile.sinceAfterUntil());
|
|
2094
|
+
}
|
|
2095
|
+
const windowed = sinceMs !== undefined || untilMs !== undefined;
|
|
2096
|
+
|
|
2097
|
+
/**
|
|
2098
|
+
* How old the price table behind every dollar below is. Stated only when it
|
|
2099
|
+
* is old enough to matter: `models` and `doctor` always print the date, but
|
|
2100
|
+
* a profile is read for its figures, and the one fact that silently
|
|
2101
|
+
* invalidates all of them is a table the provider has re-priced since.
|
|
2102
|
+
* The threshold is in the sentence, not hidden here.
|
|
2103
|
+
*/
|
|
2104
|
+
const STALE_PRICING_DAYS = 45;
|
|
2105
|
+
const pricingAgeDays = reviewAgeDays(pricing.lastReviewed, new Date());
|
|
2106
|
+
const pricingStale =
|
|
2107
|
+
pricingAgeDays !== null && pricingAgeDays > STALE_PRICING_DAYS
|
|
2108
|
+
? { date: pricing.lastReviewed, days: pricingAgeDays }
|
|
2109
|
+
: null;
|
|
2110
|
+
|
|
2111
|
+
const report = profileUsage(raw, { catalogue: pricing, label: onlyLabel, sinceMs, untilMs });
|
|
2112
|
+
if (report.total.calls === 0 && report.unpriced.calls === 0) {
|
|
2113
|
+
if (onlyLabel !== undefined || windowed) {
|
|
2114
|
+
// Diagnose against the log without the failed filter, so the error can
|
|
2115
|
+
// name what does exist instead of describing an absence.
|
|
2116
|
+
const unfiltered = profileUsage(raw, { catalogue: pricing });
|
|
2117
|
+
if (unfiltered.total.calls > 0 || unfiltered.unpriced.calls > 0) {
|
|
2118
|
+
if (onlyLabel !== undefined && !unfiltered.byLabel.some((r) => r.label === onlyLabel)) {
|
|
2119
|
+
const available = unfiltered.byLabel
|
|
2120
|
+
.map((r) => (r.label === UNLABELLED ? t.profile.unlabelled() : r.label))
|
|
2121
|
+
.join(', ');
|
|
2122
|
+
throw new Error(t.route.labelNotFound(onlyLabel, available || '—'));
|
|
2123
|
+
}
|
|
2124
|
+
if (windowed) {
|
|
2125
|
+
/**
|
|
2126
|
+
* A window that matches nothing must not become a $0 report — under
|
|
2127
|
+
* `--max-usd` it would pass a budget gate over a period the log
|
|
2128
|
+
* simply does not cover, which is the flattering non-answer. The
|
|
2129
|
+
* error names what the log *does* cover, or says it has no clock at
|
|
2130
|
+
* all, so the fix is visible in the message.
|
|
2131
|
+
*/
|
|
2132
|
+
if (unfiltered.span === null) throw new Error(t.profile.windowNeedsClock());
|
|
2133
|
+
throw new Error(
|
|
2134
|
+
`${t.profile.windowMatchesNothing(dayOf(unfiltered.span.fromMs), dayOf(unfiltered.span.toMs))}${relativeWindow ? ` ${t.profile.windowRelativeEmpty()}` : ''}`,
|
|
2135
|
+
);
|
|
2136
|
+
}
|
|
2137
|
+
}
|
|
2138
|
+
}
|
|
2139
|
+
}
|
|
2140
|
+
const n = (value: number): string => value.toLocaleString(t.numberLocale);
|
|
2141
|
+
const pct = (share: number): string => `${(share * 100).toFixed(1)}%`;
|
|
2142
|
+
|
|
2143
|
+
/**
|
|
2144
|
+
* The previous log, loaded before the output paths split so the growth gate
|
|
2145
|
+
* exists under `--json` too — a CI step reads the JSON and trusts the exit
|
|
2146
|
+
* code, and a gate that only arms in the human rendering is a gate CI never
|
|
2147
|
+
* had.
|
|
2148
|
+
*/
|
|
2149
|
+
const againstPath = stringFlag(args, 'against');
|
|
2150
|
+
// The same filter on both sides: comparing one workload's bill against the
|
|
2151
|
+
// whole previous log would report every sibling workload as vanished savings.
|
|
2152
|
+
const previous =
|
|
2153
|
+
againstPath !== undefined
|
|
2154
|
+
// The same reader as the log itself, so `--against last-month.jsonl.gz`
|
|
2155
|
+
// works: a comparison that could only read one of the two formats would
|
|
2156
|
+
// be a flag that fails on exactly the rotated file it exists to read.
|
|
2157
|
+
? profileUsage(await readLog(againstPath), {
|
|
2158
|
+
catalogue: pricing,
|
|
2159
|
+
label: onlyLabel,
|
|
2160
|
+
// The same window on both sides, for the same reason as the label:
|
|
2161
|
+
// a windowed bill against an unwindowed one compares a slice to a
|
|
2162
|
+
// whole and calls the difference growth.
|
|
2163
|
+
sinceMs,
|
|
2164
|
+
untilMs,
|
|
2165
|
+
})
|
|
2166
|
+
: null;
|
|
2167
|
+
const againstDelta =
|
|
2168
|
+
previous !== null && previous.total.calls > 0
|
|
2169
|
+
? report.total.totalUsd - previous.total.totalUsd
|
|
2170
|
+
: null;
|
|
2171
|
+
|
|
2172
|
+
/**
|
|
2173
|
+
* The same tokens at another model's rates, computed before the output paths
|
|
2174
|
+
* split so `--json` carries it too.
|
|
2175
|
+
*
|
|
2176
|
+
* An unknown id **throws** rather than printing nothing. A flag that silently
|
|
2177
|
+
* does nothing is worse than a missing feature: the reader typed a question,
|
|
2178
|
+
* got a report with no answer in it, and has no way to tell a typo from a
|
|
2179
|
+
* model this comparison had nothing to say about.
|
|
2180
|
+
*/
|
|
2181
|
+
const whatIfModel = stringFlag(args, 'what-if');
|
|
2182
|
+
const whatIf = whatIfModel !== undefined ? repriceProfile(report, whatIfModel, pricing) : null;
|
|
2183
|
+
if (whatIfModel !== undefined && whatIf === null) {
|
|
2184
|
+
throw new Error(
|
|
2185
|
+
t.profile.whatIfUnknown(whatIfModel, pricing.models.map((m) => m.id).join(', ')),
|
|
2186
|
+
);
|
|
2187
|
+
}
|
|
2188
|
+
/**
|
|
2189
|
+
* The drivers of the change, per label and per model, computed once here so
|
|
2190
|
+
* the terminal, the JSON and any future rendering describe the same change.
|
|
2191
|
+
* The model half answers the question the label half cannot: "the growth is
|
|
2192
|
+
* traffic moving from Haiku to Opus" is a fact about the mix, invisible in
|
|
2193
|
+
* per-workload rows whose names did not change.
|
|
2194
|
+
*/
|
|
2195
|
+
const labelDrivers =
|
|
2196
|
+
previous !== null && previous.total.calls > 0
|
|
2197
|
+
? driversBetween(
|
|
2198
|
+
previous.byLabel.map((r) => ({ key: r.label, usd: r.breakdown.totalUsd })),
|
|
2199
|
+
report.byLabel.map((r) => ({ key: r.label, usd: r.breakdown.totalUsd })),
|
|
2200
|
+
)
|
|
2201
|
+
: [];
|
|
2202
|
+
const modelDrivers =
|
|
2203
|
+
previous !== null && previous.total.calls > 0
|
|
2204
|
+
? driversBetween(
|
|
2205
|
+
previous.byModel.map((r) => ({ key: r.model, usd: r.breakdown.totalUsd })),
|
|
2206
|
+
report.byModel.map((r) => ({ key: r.model, usd: r.breakdown.totalUsd })),
|
|
2207
|
+
)
|
|
2208
|
+
: [];
|
|
2209
|
+
/**
|
|
2210
|
+
* Whether the two logs share any time at all. This comparison is meant for
|
|
2211
|
+
* disjoint periods or snapshots of different systems; when both spans are
|
|
2212
|
+
* known and intersect, the same calls may sit on both sides of the
|
|
2213
|
+
* subtraction and the "growth" is partly the same money counted twice.
|
|
2214
|
+
* Only decidable when both logs carry a clock — three states, as always:
|
|
2215
|
+
* warned, clear, or unknown, and unknown stays silent rather than clear.
|
|
2216
|
+
*/
|
|
2217
|
+
const againstOverlap =
|
|
2218
|
+
previous !== null &&
|
|
2219
|
+
previous.total.calls > 0 &&
|
|
2220
|
+
previous.span !== null &&
|
|
2221
|
+
report.span !== null &&
|
|
2222
|
+
Math.min(report.span.toMs, previous.span.toMs) >=
|
|
2223
|
+
Math.max(report.span.fromMs, previous.span.fromMs)
|
|
2224
|
+
? {
|
|
2225
|
+
fromMs: Math.max(report.span.fromMs, previous.span.fromMs),
|
|
2226
|
+
toMs: Math.min(report.span.toMs, previous.span.toMs),
|
|
2227
|
+
}
|
|
2228
|
+
: null;
|
|
2229
|
+
// A gate flag that silently does nothing is not an answer — same rule as
|
|
2230
|
+
// --apply-suggestions without --suggest.
|
|
2231
|
+
if (typeof args.flags.get('max-growth-usd') === 'string' && againstPath === undefined) {
|
|
2232
|
+
throw new Error(t.profile.maxGrowthNeedsAgainst());
|
|
2233
|
+
}
|
|
2234
|
+
|
|
2235
|
+
/**
|
|
2236
|
+
* The money gates, armed by flags and applied on every output path.
|
|
2237
|
+
*
|
|
2238
|
+
* `check` gates tokens before the money is spent; these gate the spend
|
|
2239
|
+
* itself, from the provider's own billed counts. No period is assumed —
|
|
2240
|
+
* the budget applies to exactly the log handed in, so a nightly job that
|
|
2241
|
+
* profiles yesterday's log has a daily budget without Trazum ever
|
|
2242
|
+
* guessing what a day is.
|
|
2243
|
+
*/
|
|
2244
|
+
const applyGates = (): void => {
|
|
2245
|
+
/**
|
|
2246
|
+
* Before any verdict: whether the gated figure is the whole bill. A gate
|
|
2247
|
+
* can only judge the money it can see, and three things hide money from
|
|
2248
|
+
* it — unreadable lines, unpriced models, and clockless calls left
|
|
2249
|
+
* outside a window. Passing on a floor is acceptable; passing on a floor
|
|
2250
|
+
* *silently* is the flattering omission this repository refuses, because
|
|
2251
|
+
* an over-budget bill with three corrupt lines would read as green.
|
|
2252
|
+
*/
|
|
2253
|
+
const anyGate =
|
|
2254
|
+
typeof args.flags.get('max-usd') === 'string' ||
|
|
2255
|
+
typeof args.flags.get('max-growth-usd') === 'string' ||
|
|
2256
|
+
typeof args.flags.get('max-cache-loss-usd') === 'string' ||
|
|
2257
|
+
typeof args.flags.get('max-day-usd') === 'string' ||
|
|
2258
|
+
config.spend !== undefined;
|
|
2259
|
+
if (anyGate) {
|
|
2260
|
+
const reasons: string[] = [];
|
|
2261
|
+
if (report.skippedLines.length > 0) reasons.push(t.profile.floorSkipped(report.skippedLines.length));
|
|
2262
|
+
if (report.unpriced.calls > 0) reasons.push(t.profile.floorUnpriced(report.unpriced.calls));
|
|
2263
|
+
if (report.timeWindow !== null && report.timeWindow.undatedExcluded > 0) {
|
|
2264
|
+
reasons.push(t.profile.floorUndated(report.timeWindow.undatedExcluded));
|
|
2265
|
+
}
|
|
2266
|
+
if (reasons.length > 0) {
|
|
2267
|
+
console.error(c.yellow(t.profile.gateOnFloor(reasons.join('; '))));
|
|
2268
|
+
}
|
|
2269
|
+
}
|
|
2270
|
+
/**
|
|
2271
|
+
* Per-workload budgets from the config — the policy in the repository
|
|
2272
|
+
* rather than in one CI invocation. Each label is gated against its own
|
|
2273
|
+
* spend in the same run, and a budgeted label with no calls in this log
|
|
2274
|
+
* is reported as **not measured**: a workload that did not appear is not
|
|
2275
|
+
* a workload that came in under budget, and printing green over an
|
|
2276
|
+
* absence is exactly the flattering direction this tool refuses.
|
|
2277
|
+
*/
|
|
2278
|
+
const byLabel = config.spend?.byLabel;
|
|
2279
|
+
if (byLabel !== undefined && !windowed) {
|
|
2280
|
+
const spent = new Map(report.byLabel.map((r) => [r.label, r.breakdown.totalUsd]));
|
|
2281
|
+
for (const [label, limit] of Object.entries(byLabel)) {
|
|
2282
|
+
const usd = spent.get(label);
|
|
2283
|
+
if (usd === undefined) {
|
|
2284
|
+
console.error(c.dim(t.profile.labelBudgetMissing(label)));
|
|
2285
|
+
continue;
|
|
2286
|
+
}
|
|
2287
|
+
if (usd > limit) {
|
|
2288
|
+
console.error(c.red(t.profile.labelBudgetFailed(label, formatUsd(usd), formatUsd(limit))));
|
|
2289
|
+
process.exitCode = 1;
|
|
2290
|
+
} else {
|
|
2291
|
+
console.error(c.dim(t.profile.labelBudgetOk(label, formatUsd(usd), formatUsd(limit))));
|
|
2292
|
+
}
|
|
2293
|
+
}
|
|
2294
|
+
} else if (byLabel !== undefined && windowed) {
|
|
2295
|
+
// A window changes what "this label spent" means, and a budget written
|
|
2296
|
+
// for a period the caller did not name would gate against a slice.
|
|
2297
|
+
console.error(c.dim(t.profile.labelBudgetWindowed()));
|
|
2298
|
+
}
|
|
2299
|
+
|
|
2300
|
+
if (typeof args.flags.get('max-usd') === 'string' || config.spend?.maxUsd !== undefined) {
|
|
2301
|
+
const maxUsd =
|
|
2302
|
+
typeof args.flags.get('max-usd') === 'string'
|
|
2303
|
+
? numberFlag(args, 'max-usd', 0, t)
|
|
2304
|
+
: config.spend!.maxUsd!;
|
|
2305
|
+
if (report.total.totalUsd > maxUsd) {
|
|
2306
|
+
console.error(c.red(t.profile.maxUsdFailed(formatUsd(report.total.totalUsd), formatUsd(maxUsd))));
|
|
2307
|
+
process.exitCode = 1;
|
|
2308
|
+
} else {
|
|
2309
|
+
console.error(c.dim(t.profile.maxUsdOk(formatUsd(report.total.totalUsd), formatUsd(maxUsd))));
|
|
2310
|
+
}
|
|
2311
|
+
}
|
|
2312
|
+
if (typeof args.flags.get('max-growth-usd') === 'string' && againstDelta !== null) {
|
|
2313
|
+
const maxGrowth = numberFlag(args, 'max-growth-usd', 0, t);
|
|
2314
|
+
if (againstDelta > maxGrowth) {
|
|
2315
|
+
console.error(c.red(t.profile.maxGrowthUsdFailed(formatSignedUsd(againstDelta), formatUsd(maxGrowth))));
|
|
2316
|
+
process.exitCode = 1;
|
|
2317
|
+
}
|
|
2318
|
+
}
|
|
2319
|
+
/**
|
|
2320
|
+
* The cache gate, and it reads the worst case on purpose. A log carrying
|
|
2321
|
+
* only the flat cache-write count cannot say which TTL was paid, and the
|
|
2322
|
+
* two verdicts can straddle the limit — a gate reading the flattering
|
|
2323
|
+
* half would pass exactly the bills it exists to catch. The failure
|
|
2324
|
+
* message says which claim fired: a settled loss, or a ceiling only the
|
|
2325
|
+
* missing "cache_creation" field can settle.
|
|
2326
|
+
*/
|
|
2327
|
+
if (typeof args.flags.get('max-cache-loss-usd') === 'string') {
|
|
2328
|
+
const maxLoss = numberFlag(args, 'max-cache-loss-usd', 0, t);
|
|
2329
|
+
const gateCache = cacheEconomics(report.total);
|
|
2330
|
+
if (gateCache.deltaUsd > maxLoss) {
|
|
2331
|
+
console.error(
|
|
2332
|
+
c.red(t.profile.maxCacheLossFailed(formatUsd(gateCache.deltaUsd), formatUsd(maxLoss))),
|
|
2333
|
+
);
|
|
2334
|
+
process.exitCode = 1;
|
|
2335
|
+
} else if (gateCache.worstCaseDeltaUsd > maxLoss) {
|
|
2336
|
+
console.error(
|
|
2337
|
+
c.red(
|
|
2338
|
+
t.profile.maxCacheLossWorstCase(
|
|
2339
|
+
report.total.assumedWriteTtlCalls,
|
|
2340
|
+
formatUsd(gateCache.worstCaseDeltaUsd),
|
|
2341
|
+
formatUsd(maxLoss),
|
|
2342
|
+
),
|
|
2343
|
+
),
|
|
2344
|
+
);
|
|
2345
|
+
process.exitCode = 1;
|
|
2346
|
+
} else {
|
|
2347
|
+
console.error(
|
|
2348
|
+
c.dim(t.profile.maxCacheLossOk(formatUsd(Math.max(0, gateCache.worstCaseDeltaUsd)), formatUsd(maxLoss))),
|
|
2349
|
+
);
|
|
2350
|
+
}
|
|
2351
|
+
}
|
|
2352
|
+
/**
|
|
2353
|
+
* The per-day gate — the one a total cannot arm.
|
|
2354
|
+
*
|
|
2355
|
+
* A month at $3,000 against a $4,000 budget passes while one afternoon's
|
|
2356
|
+
* runaway agent loop burned $900 of it in four hours. `--max-usd` gates
|
|
2357
|
+
* the sum handed in; this gates the **worst single UTC day inside it**,
|
|
2358
|
+
* which is the shape a loop, a bad deploy or a retry storm actually has.
|
|
2359
|
+
*
|
|
2360
|
+
* Two refusals it inherits from the rest of the tool:
|
|
2361
|
+
*
|
|
2362
|
+
* A log with **no clock at all** cannot be judged by day, and that is an
|
|
2363
|
+
* error rather than a pass. "Not measured" is not "under budget", and a
|
|
2364
|
+
* gate that silently green-lights an unmeasurable log is worse than one
|
|
2365
|
+
* that was never armed.
|
|
2366
|
+
*
|
|
2367
|
+
* The first and last day of a log are usually **partial**, so a day under
|
|
2368
|
+
* the limit here is under it for the hours the log contains. A day *over*
|
|
2369
|
+
* the limit is over it whatever the missing hours held — the failure is
|
|
2370
|
+
* sound in both directions, the pass is a floor, and the pass message
|
|
2371
|
+
* says so when the span does not start and end on a day boundary.
|
|
2372
|
+
*/
|
|
2373
|
+
if (typeof args.flags.get('max-day-usd') === 'string') {
|
|
2374
|
+
const maxDay = numberFlag(args, 'max-day-usd', 0, t);
|
|
2375
|
+
if (report.spendByDay.length === 0) {
|
|
2376
|
+
console.error(c.red(t.profile.maxDayNoClock()));
|
|
2377
|
+
process.exitCode = 1;
|
|
2378
|
+
} else {
|
|
2379
|
+
const worst = report.spendByDay.reduce((a, b) => (b.usd > a.usd ? b : a));
|
|
2380
|
+
const suspect =
|
|
2381
|
+
worst.topLabel !== null && report.byLabel.length > 1
|
|
2382
|
+
? ` ${t.profile.dayPeakLabel(worst.topLabel === UNLABELLED ? t.profile.unlabelled() : worst.topLabel, formatUsd(worst.topLabelUsd))}`
|
|
2383
|
+
: '';
|
|
2384
|
+
if (worst.usd > maxDay) {
|
|
2385
|
+
console.error(
|
|
2386
|
+
c.red(`${t.profile.maxDayFailed(worst.day, formatUsd(worst.usd), formatUsd(maxDay))}${suspect}`),
|
|
2387
|
+
);
|
|
2388
|
+
process.exitCode = 1;
|
|
2389
|
+
} else {
|
|
2390
|
+
console.error(c.dim(t.profile.maxDayOk(worst.day, formatUsd(worst.usd), formatUsd(maxDay))));
|
|
2391
|
+
/**
|
|
2392
|
+
* Calls with no clock are in the bill above and in no day below, so
|
|
2393
|
+
* the worst day is a floor by exactly that much. Said only on a
|
|
2394
|
+
* pass: a failure stands whatever the undated calls held.
|
|
2395
|
+
*/
|
|
2396
|
+
const undated = report.fieldCoverage.parsed - report.fieldCoverage.ts;
|
|
2397
|
+
if (undated > 0) {
|
|
2398
|
+
console.error(c.yellow(t.profile.maxDayUndated(n(undated))));
|
|
2399
|
+
}
|
|
2400
|
+
}
|
|
2401
|
+
}
|
|
2402
|
+
}
|
|
2403
|
+
};
|
|
2404
|
+
|
|
2405
|
+
/**
|
|
2406
|
+
* The side files the caller asked for. Written on **both** output paths:
|
|
2407
|
+
* under --json the human rendering returns early, and the first version of
|
|
2408
|
+
* --csv-out therefore wrote nothing at all there — a flag that silently did
|
|
2409
|
+
* nothing, which is the fault this repository keeps refusing elsewhere.
|
|
2410
|
+
*/
|
|
2411
|
+
const writeSideFiles = async (): Promise<void> => {
|
|
2412
|
+
/**
|
|
2413
|
+
* Where the "wrote to" notice goes. Under `--json`, stdout carries the
|
|
2414
|
+
* report and nothing else — a status line there turns a parseable
|
|
2415
|
+
* document into a parse error, which is how a pipeline discovers the
|
|
2416
|
+
* feature. The gates already route their verdicts to stderr for the same
|
|
2417
|
+
* reason.
|
|
2418
|
+
*/
|
|
2419
|
+
const notice = boolFlag(args, 'json')
|
|
2420
|
+
? (message: string): void => console.error(message)
|
|
2421
|
+
: (message: string): void => console.log(message);
|
|
2422
|
+
/**
|
|
2423
|
+
* The same report as GitHub-flavoured markdown, for a job summary or a PR
|
|
2424
|
+
* comment. Written from the same message catalogue the terminal used, because
|
|
2425
|
+
* two renderings of one finding drift the moment they are worded twice.
|
|
2426
|
+
*/
|
|
2427
|
+
const markdownOut = stringFlag(args, 'markdown-out');
|
|
2428
|
+
if (markdownOut !== undefined) {
|
|
2429
|
+
await writeFile(
|
|
2430
|
+
markdownOut,
|
|
2431
|
+
renderProfileMarkdown({
|
|
2432
|
+
report,
|
|
2433
|
+
levers,
|
|
2434
|
+
cache,
|
|
2435
|
+
t,
|
|
2436
|
+
...(windowed
|
|
2437
|
+
? { window: { since: stringFlag(args, 'since') ?? '—', until: stringFlag(args, 'until') ?? '—' } }
|
|
2438
|
+
: {}),
|
|
2439
|
+
...(pricingStale !== null ? { stalePricing: pricingStale } : {}),
|
|
2440
|
+
// The repricing, when --what-if was given: computed once above and
|
|
2441
|
+
// handed over, so the summary in a pull request cannot disagree
|
|
2442
|
+
// with the terminal about what a move would cost.
|
|
2443
|
+
...(whatIf !== null ? { whatIf } : {}),
|
|
2444
|
+
// The comparison, when there was one — the same figures and the same
|
|
2445
|
+
// drivers the terminal printed, never re-derived here.
|
|
2446
|
+
...(previous !== null
|
|
2447
|
+
? {
|
|
2448
|
+
against: {
|
|
2449
|
+
previousTotalUsd: previous.total.totalUsd,
|
|
2450
|
+
previousCalls: previous.total.calls,
|
|
2451
|
+
labelDrivers,
|
|
2452
|
+
modelDrivers:
|
|
2453
|
+
new Set([
|
|
2454
|
+
...previous.byModel.map((r) => r.model),
|
|
2455
|
+
...report.byModel.map((r) => r.model),
|
|
2456
|
+
]).size > 1
|
|
2457
|
+
? modelDrivers
|
|
2458
|
+
: [],
|
|
2459
|
+
overlap:
|
|
2460
|
+
againstOverlap !== null
|
|
2461
|
+
? { from: dayOf(againstOverlap.fromMs), to: dayOf(againstOverlap.toMs) }
|
|
2462
|
+
: null,
|
|
2463
|
+
nothingPriced: previous.total.calls === 0,
|
|
2464
|
+
},
|
|
2465
|
+
}
|
|
2466
|
+
: {}),
|
|
2467
|
+
}),
|
|
2468
|
+
'utf8',
|
|
2469
|
+
);
|
|
2470
|
+
notice(c.dim(t.report.wroteTo(markdownOut)));
|
|
2471
|
+
}
|
|
2472
|
+
|
|
2473
|
+
/**
|
|
2474
|
+
* The same report as a spreadsheet, one row per label and model — the grain
|
|
2475
|
+
* a routing or budget decision is made at. Deliberately without a total
|
|
2476
|
+
* row: a total inside a data file is summed with the data and doubles every
|
|
2477
|
+
* figure downstream.
|
|
2478
|
+
*/
|
|
2479
|
+
const csvOut = stringFlag(args, 'csv-out');
|
|
2480
|
+
if (csvOut !== undefined) {
|
|
2481
|
+
/**
|
|
2482
|
+
* Which table the file holds. One row shape per file on purpose: a
|
|
2483
|
+
* spreadsheet that has to filter before it can sum is a spreadsheet
|
|
2484
|
+
* somebody sums wrong.
|
|
2485
|
+
*/
|
|
2486
|
+
const shape = stringFlag(args, 'csv-shape') ?? 'slice';
|
|
2487
|
+
if (shape !== 'slice' && shape !== 'day' && shape !== 'hour') {
|
|
2488
|
+
throw new Error(t.profile.badCsvShape(shape));
|
|
2489
|
+
}
|
|
2490
|
+
await writeFile(
|
|
2491
|
+
csvOut,
|
|
2492
|
+
profileToCsv(report, { unlabelled: t.profile.unlabelled(), shape }),
|
|
2493
|
+
'utf8',
|
|
2494
|
+
);
|
|
2495
|
+
notice(c.dim(t.report.wroteTo(csvOut)));
|
|
2496
|
+
}
|
|
2497
|
+
};
|
|
2498
|
+
|
|
2499
|
+
if (boolFlag(args, 'json')) {
|
|
2500
|
+
/**
|
|
2501
|
+
* The report, plus everything the human output leads on.
|
|
2502
|
+
*
|
|
2503
|
+
* Additive rather than a reshape: `report` keeps the shape `@trazum/core`
|
|
2504
|
+
* returns. The cache verdict is included because leaving a consumer to
|
|
2505
|
+
* re-derive it means two implementations of a sign convention where positive
|
|
2506
|
+
* means *worse*, and one of them will eventually get it backwards.
|
|
2507
|
+
*
|
|
2508
|
+
* `levers` is included because it was not, and that made the flagship
|
|
2509
|
+
* section terminal-only: "What would actually move this bill" — the reason
|
|
2510
|
+
* the command exists — was invisible to any pipeline, dashboard or CI step
|
|
2511
|
+
* reading the JSON. A finding the machine-readable output omits is a finding
|
|
2512
|
+
* the reader's tooling will never surface.
|
|
2513
|
+
*/
|
|
2514
|
+
console.log(
|
|
2515
|
+
JSON.stringify(
|
|
2516
|
+
{
|
|
2517
|
+
/**
|
|
2518
|
+
* The contract version, documented in docs/json-output.md and
|
|
2519
|
+
* enforced by json-contract.test.js. It changes only when a
|
|
2520
|
+
* field's meaning changes or one is removed — new findings arrive
|
|
2521
|
+
* as new keys, so a consumer that ignores unknown ones keeps
|
|
2522
|
+
* working. Without it, every dashboard built on this output has to
|
|
2523
|
+
* guess whether a missing key means "old Trazum" or "no data".
|
|
2524
|
+
*/
|
|
2525
|
+
schemaVersion: 1,
|
|
2526
|
+
...report,
|
|
2527
|
+
cache: cacheEconomics(report.total),
|
|
2528
|
+
cacheByLabel: report.byLabel.map((r) => ({
|
|
2529
|
+
label: r.label,
|
|
2530
|
+
cache: cacheEconomics(r.breakdown),
|
|
2531
|
+
})),
|
|
2532
|
+
// The provenance of every dollar above: which price table, how old.
|
|
2533
|
+
pricing: { lastReviewed: pricing.lastReviewed, ageDays: pricingAgeDays },
|
|
2534
|
+
levers: billLevers(report, { catalogue: pricing }),
|
|
2535
|
+
// Present only when --against was passed: null delta means the
|
|
2536
|
+
// previous log had nothing priced, which is a different answer from
|
|
2537
|
+
// zero growth.
|
|
2538
|
+
...(previous !== null
|
|
2539
|
+
? {
|
|
2540
|
+
against: {
|
|
2541
|
+
previousTotalUsd: previous.total.totalUsd,
|
|
2542
|
+
deltaUsd: againstDelta,
|
|
2543
|
+
// The same drivers the terminal names, as data. A finding
|
|
2544
|
+
// the machine-readable output omits is a finding the
|
|
2545
|
+
// reader's tooling will never surface.
|
|
2546
|
+
byLabel: labelDrivers,
|
|
2547
|
+
byModel: modelDrivers,
|
|
2548
|
+
},
|
|
2549
|
+
}
|
|
2550
|
+
: {}),
|
|
2551
|
+
// Present only when --what-if was passed. `sameTokensAssumed` rides
|
|
2552
|
+
// along inside it so a consumer cannot print the dollar figure
|
|
2553
|
+
// without the caveat being in the same object.
|
|
2554
|
+
...(whatIf !== null ? { whatIf } : {}),
|
|
2555
|
+
},
|
|
2556
|
+
null,
|
|
2557
|
+
2,
|
|
2558
|
+
),
|
|
2559
|
+
);
|
|
2560
|
+
await writeSideFiles();
|
|
2561
|
+
applyGates();
|
|
2562
|
+
return;
|
|
2563
|
+
}
|
|
2564
|
+
|
|
2565
|
+
/**
|
|
2566
|
+
* Nothing priced means there is no report, not a report of zero.
|
|
2567
|
+
*
|
|
2568
|
+
* The guard was `total.calls === 0 && unpriced.calls === 0`, so a log whose every
|
|
2569
|
+
* model was unknown fell through and printed a full report built from a zeroed
|
|
2570
|
+
* total: `0 calls · $0`, four `$0 / 0.0%` rows, a meaningless "Input is 0.0% of
|
|
2571
|
+
* this bill", and — on a log containing a hundred thousand cache-read tokens —
|
|
2572
|
+
* the flatly false "Caching was never used on these calls".
|
|
2573
|
+
*
|
|
2574
|
+
* Two affirmatively wrong claims and a $0 headline for a real bill. The trailing
|
|
2575
|
+
* unpriced note was the only correct line on screen, and it was the quietest.
|
|
2576
|
+
*/
|
|
2577
|
+
if (report.total.calls === 0) {
|
|
2578
|
+
console.log();
|
|
2579
|
+
console.log(c.dim(report.unpriced.calls === 0 ? t.profile.empty() : t.profile.nothingPriced()));
|
|
2580
|
+
reportProfileGaps(report, t, n, pricingStale);
|
|
2581
|
+
return;
|
|
2582
|
+
}
|
|
2583
|
+
|
|
2584
|
+
const shares = sharesOf(report.total);
|
|
2585
|
+
const parts: Array<[string, number, number, number]> = [
|
|
2586
|
+
[t.profile.partInput(), report.total.inputUsd, shares.input, report.total.inputTokens],
|
|
2587
|
+
[t.profile.partCacheRead(), report.total.cacheReadUsd, shares.cacheRead, report.total.cacheReadTokens],
|
|
2588
|
+
[t.profile.partCacheWrite(), report.total.cacheWriteUsd, shares.cacheWrite, report.total.cacheWriteTokens],
|
|
2589
|
+
[t.profile.partOutput(), report.total.outputUsd, shares.output, report.total.outputTokens],
|
|
2590
|
+
];
|
|
2591
|
+
|
|
2592
|
+
console.log();
|
|
2593
|
+
console.log(c.bold(t.profile.heading()));
|
|
2594
|
+
console.log(` ${t.profile.spent(t.profile.calls(report.total.calls), formatUsd(report.total.totalUsd))}`);
|
|
2595
|
+
/**
|
|
2596
|
+
* The period, when the log carries a clock — stated, never extrapolated. A
|
|
2597
|
+
* span makes the reader's own monthly arithmetic valid; a per-month figure
|
|
2598
|
+
* printed from a partial month would be this tool doing the guessing it
|
|
2599
|
+
* exists to end. Partial coverage is said in the same breath, because a span
|
|
2600
|
+
* over a third of the calls silently presented as the log's period is a
|
|
2601
|
+
* figure attributed to something it does not describe.
|
|
2602
|
+
*/
|
|
2603
|
+
if (report.span !== null) {
|
|
2604
|
+
const totalParsed = report.total.calls + report.unpriced.calls;
|
|
2605
|
+
const partial =
|
|
2606
|
+
report.span.calls < totalParsed
|
|
2607
|
+
? ` ${t.profile.spanPartial(n(report.span.calls), n(totalParsed))}`
|
|
2608
|
+
: '';
|
|
2609
|
+
console.log(
|
|
2610
|
+
` ${c.dim(wrap(`${t.profile.spanLine(dayOf(report.span.fromMs), dayOf(report.span.toMs), spanDays(report.span.fromMs, report.span.toMs))}${partial}`, 74, ' '))}`,
|
|
2611
|
+
);
|
|
2612
|
+
}
|
|
2613
|
+
// How many files this report covers, when it covers more than one: a total
|
|
2614
|
+
// over "the logs" that silently skipped one is wrong by an unknown amount.
|
|
2615
|
+
if (logFiles.length > 1) {
|
|
2616
|
+
console.log(` ${c.dim(wrap(t.profile.readFiles(logFiles.length, path), 74, ' '))}`);
|
|
2617
|
+
}
|
|
2618
|
+
|
|
2619
|
+
/**
|
|
2620
|
+
* A doubled bill, said before anything is believed.
|
|
2621
|
+
*
|
|
2622
|
+
* Reading a directory of rotated logs makes double-counting easy — a log
|
|
2623
|
+
* exported twice, an overlapping export, a copy left in the folder — and
|
|
2624
|
+
* the total then reads high with nothing else able to see it. Only counted
|
|
2625
|
+
* over records with a clock, where an identical line is a claim worth
|
|
2626
|
+
* making. It states the count and the money and stops: whether it is a
|
|
2627
|
+
* double export or a genuinely busy millisecond is the reader's to know.
|
|
2628
|
+
*/
|
|
2629
|
+
if (report.duplicateLines.count > 0) {
|
|
2630
|
+
console.log(
|
|
2631
|
+
` ${c.yellow('!')} ${c.dim(wrap(t.profile.duplicateLines(report.duplicateLines.count, formatUsd(report.duplicateLines.usd)), 74, ' '))}`,
|
|
2632
|
+
);
|
|
2633
|
+
}
|
|
2634
|
+
|
|
2635
|
+
/**
|
|
2636
|
+
* The window, said before any figure is trusted as "the log": everything
|
|
2637
|
+
* below describes a slice, and a slice presented as the whole is a figure
|
|
2638
|
+
* attributed to something it does not describe. The undated count is loud —
|
|
2639
|
+
* those calls' spend is in the log and not in this report, so the window's
|
|
2640
|
+
* figures are a floor on the period, and only this line says so.
|
|
2641
|
+
*/
|
|
2642
|
+
if (report.timeWindow !== null) {
|
|
2643
|
+
console.log(
|
|
2644
|
+
` ${c.dim(wrap(t.profile.windowLine(stringFlag(args, 'since') ?? '—', stringFlag(args, 'until') ?? '—'), 74, ' '))}`,
|
|
2645
|
+
);
|
|
2646
|
+
if (relativeWindow) {
|
|
2647
|
+
console.log(` ${c.dim(wrap(t.profile.windowRelative(), 74, ' '))}`);
|
|
2648
|
+
}
|
|
2649
|
+
if (report.timeWindow.undatedExcluded > 0) {
|
|
2650
|
+
console.log(
|
|
2651
|
+
` ${c.yellow(wrap(t.profile.windowUndated(report.timeWindow.undatedExcluded), 74, ' '))}`,
|
|
2652
|
+
);
|
|
2653
|
+
}
|
|
2654
|
+
}
|
|
2655
|
+
console.log();
|
|
2656
|
+
// Every part, including the zero ones. A row missing because it was zero reads
|
|
2657
|
+
// as a row somebody forgot, and "you are not caching at all" is a finding.
|
|
2658
|
+
for (const [name, usd, share, tokens] of parts) {
|
|
2659
|
+
console.log(` ${c.dim(t.profile.part(name, formatUsd(usd), pct(share), n(tokens)))}`);
|
|
2660
|
+
}
|
|
2661
|
+
|
|
2662
|
+
/**
|
|
2663
|
+
* The line the command exists for: which part of the bill to argue with.
|
|
2664
|
+
*
|
|
2665
|
+
* When output is both the biggest part and over half, the two sentences say the
|
|
2666
|
+
* same thing and the second says more — so only the second prints. Reporting a
|
|
2667
|
+
* fact twice in adjacent lines reads as a bug, and it was one.
|
|
2668
|
+
*/
|
|
2669
|
+
const [biggestName, , biggestShare] = parts.reduce((a, b) => (b[1] > a[1] ? b : a));
|
|
2670
|
+
const outputDominates = shares.output > 0.5;
|
|
2671
|
+
console.log();
|
|
2672
|
+
if (outputDominates) {
|
|
2673
|
+
console.log(` ${c.bold(wrap(t.profile.outputDominates(pct(shares.output)), 74, ' '))}`);
|
|
2674
|
+
} else {
|
|
2675
|
+
console.log(` ${c.bold(t.profile.biggestPart(biggestName, pct(biggestShare)))}`);
|
|
2676
|
+
}
|
|
2677
|
+
|
|
2678
|
+
/**
|
|
2679
|
+
* The most expensive day, with a suspect attached.
|
|
2680
|
+
*
|
|
2681
|
+
* The shape of a bill over time is the finding the total hides: a steady $3 a
|
|
2682
|
+
* day and a quiet week broken by one $40 spike sum to the same number and call
|
|
2683
|
+
* for opposite responses. Rendered against the **median** day — a mean would
|
|
2684
|
+
* let the spike inflate its own yardstick — and loud only when it clears twice
|
|
2685
|
+
* the median, a threshold stated in the sentence rather than hidden in code.
|
|
2686
|
+
*/
|
|
2687
|
+
if (report.spendByDay.length >= 2) {
|
|
2688
|
+
const medianUsd = median(report.spendByDay.map((d) => d.usd));
|
|
2689
|
+
const peak = report.spendByDay.reduce((a, b) => (b.usd > a.usd ? b : a));
|
|
2690
|
+
if (medianUsd > 0) {
|
|
2691
|
+
const ratio = (peak.usd / medianUsd).toFixed(1);
|
|
2692
|
+
const line = t.profile.dayPeak(peak.day, formatUsd(peak.usd), ratio);
|
|
2693
|
+
const labelClause =
|
|
2694
|
+
peak.topLabel !== null && report.byLabel.length > 1
|
|
2695
|
+
? ` ${t.profile.dayPeakLabel(peak.topLabel === UNLABELLED ? t.profile.unlabelled() : peak.topLabel, formatUsd(peak.topLabelUsd))}`
|
|
2696
|
+
: '';
|
|
2697
|
+
const loud = peak.usd > 2 * medianUsd;
|
|
2698
|
+
const text = wrap(`${line}${labelClause}`, 74, ' ');
|
|
2699
|
+
console.log(` ${loud ? c.yellow(text) : c.dim(text)}`);
|
|
2700
|
+
}
|
|
2701
|
+
}
|
|
2702
|
+
|
|
2703
|
+
/**
|
|
2704
|
+
* The shape of the day, and what it says about batching.
|
|
2705
|
+
*
|
|
2706
|
+
* Spend packed into the hours a country is awake is interactive traffic
|
|
2707
|
+
* somebody is waiting on; spend spread evenly across twenty-four is
|
|
2708
|
+
* background work — and background work is what the Batch API halves. The
|
|
2709
|
+
* measure is exact and needs no threshold to state: the **fewest hours that
|
|
2710
|
+
* hold 80% of the spend**. Two or three means concentrated; sixteen means
|
|
2711
|
+
* flat.
|
|
2712
|
+
*
|
|
2713
|
+
* It says what the shape is and stops. Whether a workload can wait is a
|
|
2714
|
+
* product decision Trazum cannot make from counts, so the sentence names
|
|
2715
|
+
* the lever and never claims the saving — the batch figure the levers
|
|
2716
|
+
* section already prints is the one with money attached.
|
|
2717
|
+
*/
|
|
2718
|
+
if (report.spendByHour.length >= 4 && report.total.totalUsd > 0) {
|
|
2719
|
+
const sorted = [...report.spendByHour].sort((a, b) => b.usd - a.usd);
|
|
2720
|
+
let covered = 0;
|
|
2721
|
+
let hoursForMost = 0;
|
|
2722
|
+
for (const hour of sorted) {
|
|
2723
|
+
covered += hour.usd;
|
|
2724
|
+
hoursForMost += 1;
|
|
2725
|
+
if (covered >= 0.8 * report.total.totalUsd) break;
|
|
2726
|
+
}
|
|
2727
|
+
const busiest = sorted
|
|
2728
|
+
.slice(0, hoursForMost)
|
|
2729
|
+
.map((hour) => hour.hour)
|
|
2730
|
+
.sort((a, b) => a - b)
|
|
2731
|
+
.map((hour) => `${String(hour).padStart(2, '0')}:00`)
|
|
2732
|
+
.join(', ');
|
|
2733
|
+
console.log();
|
|
2734
|
+
console.log(
|
|
2735
|
+
` ${c.dim(wrap(hoursForMost <= 8 ? t.profile.hoursConcentrated(n(hoursForMost), busiest) : t.profile.hoursFlat(n(hoursForMost)), 74, ' '))}`,
|
|
2736
|
+
);
|
|
2737
|
+
}
|
|
2738
|
+
|
|
2739
|
+
/**
|
|
2740
|
+
* The hit rate, and then the question the hit rate does not answer.
|
|
2741
|
+
*
|
|
2742
|
+
* `cacheNever()` is keyed off the **verdict**, not off a null hit rate. Those
|
|
2743
|
+
* two came apart on a log whose calls were entirely cache writes with no plain
|
|
2744
|
+
* input: the rate is undefined there — zero reads over zero attempts — while
|
|
2745
|
+
* caching was plainly in use, and the old branch printed "caching was never
|
|
2746
|
+
* used" over a bill made of cache writes.
|
|
2747
|
+
*/
|
|
2748
|
+
const cache = cacheEconomics(report.total);
|
|
2749
|
+
const hitRate = cacheHitRate(report.total);
|
|
2750
|
+
if (cache.verdict === 'not-attempted') {
|
|
2751
|
+
console.log(` ${c.dim(wrap(t.profile.cacheNever(), 74, ' '))}`);
|
|
2752
|
+
} else if (hitRate !== null) {
|
|
2753
|
+
console.log(` ${c.dim(t.profile.cacheHit(pct(hitRate)))}`);
|
|
2754
|
+
}
|
|
2755
|
+
|
|
2756
|
+
/**
|
|
2757
|
+
* Whether the caching was worth doing — the one finding here that can
|
|
2758
|
+
* contradict the advice Trazum gives everywhere else.
|
|
2759
|
+
*
|
|
2760
|
+
* A cache write costs 1.25x plain input on Anthropic and 2x at the one-hour
|
|
2761
|
+
* TTL, so a prefix rebuilt faster than it is reused is billed at a premium and
|
|
2762
|
+
* returns nothing: that workload is cheaper with caching switched off. The
|
|
2763
|
+
* counterfactual is exact rather than a projection — caching changes the
|
|
2764
|
+
* multiplier on a token, never the token — so this is the one place in `profile`
|
|
2765
|
+
* where a comparison against what-might-have-been is arithmetic instead of a
|
|
2766
|
+
* guess about a prompt nobody wrote.
|
|
2767
|
+
*/
|
|
2768
|
+
/**
|
|
2769
|
+
* The losing labels, **ranked by what caching cost them** and not by bill size.
|
|
2770
|
+
*
|
|
2771
|
+
* `byLabel` arrives sorted by total spend, which is the right order for the
|
|
2772
|
+
* table above and the wrong one here: the worst cache in an estate usually sits
|
|
2773
|
+
* on a small workload, so taking the first three off a spend-ordered list meant
|
|
2774
|
+
* the biggest loser could be the one that went unnamed.
|
|
2775
|
+
*/
|
|
2776
|
+
const lostLabels = report.byLabel
|
|
2777
|
+
.map((r) => ({ row: r, cache: cacheEconomics(r.breakdown) }))
|
|
2778
|
+
.filter((r) => r.cache.verdict === 'lost-money')
|
|
2779
|
+
.sort((a, b) => b.cache.deltaUsd - a.cache.deltaUsd);
|
|
2780
|
+
|
|
2781
|
+
const NAMED = 3;
|
|
2782
|
+
const nameOf = (row: { label: string }): string =>
|
|
2783
|
+
row.label === UNLABELLED ? t.profile.unlabelled() : row.label;
|
|
2784
|
+
/**
|
|
2785
|
+
* The names, with the ones that did not fit **counted rather than dropped**.
|
|
2786
|
+
*
|
|
2787
|
+
* The first version sliced to three silently while the money beside it was
|
|
2788
|
+
* summed over every loser — so four bleeding labels printed three names and a
|
|
2789
|
+
* figure that charged them with a fourth label's loss. Truncating is fine;
|
|
2790
|
+
* truncating without saying so is the flattering omission this repository keeps
|
|
2791
|
+
* catching itself at, and `reportProfileGaps` already had the pattern.
|
|
2792
|
+
*/
|
|
2793
|
+
const listNames = (rows: Array<{ row: { label: string } }>): string => {
|
|
2794
|
+
const names = rows.slice(0, NAMED).map((r) => nameOf(r.row)).join(', ');
|
|
2795
|
+
return rows.length <= NAMED
|
|
2796
|
+
? names
|
|
2797
|
+
: `${names} ${t.profile.andMoreLabels(rows.length - NAMED)}`;
|
|
2798
|
+
};
|
|
2799
|
+
const namedLosers = listNames(lostLabels);
|
|
2800
|
+
const bleeding = lostLabels.reduce((sum, r) => sum + r.cache.deltaUsd, 0);
|
|
2801
|
+
|
|
2802
|
+
/**
|
|
2803
|
+
* Whether the log can settle the question at all.
|
|
2804
|
+
*
|
|
2805
|
+
* Decided before anything prints, because it governs whether the confident
|
|
2806
|
+
* sentence prints — not merely whether a caveat follows it. The first attempt
|
|
2807
|
+
* added the caveat and left the assertion above it, so the reader met `Caching
|
|
2808
|
+
* took $0.1000 off this bill` and only afterwards learned it might be a $3.65
|
|
2809
|
+
* loss. A finding a later line retracts is still a finding somebody acted on.
|
|
2810
|
+
*/
|
|
2811
|
+
const unsettled =
|
|
2812
|
+
cache.worstCaseVerdict !== cache.verdict && report.total.assumedWriteTtlCalls > 0;
|
|
2813
|
+
|
|
2814
|
+
if (unsettled) {
|
|
2815
|
+
console.log(
|
|
2816
|
+
` ${c.yellow('!')} ${c.bold(wrap(t.profile.cacheTtlUnsettled(report.total.assumedWriteTtlCalls, formatUsd(-cache.deltaUsd), formatUsd(cache.worstCaseDeltaUsd)), 74, ' '))}`,
|
|
2817
|
+
);
|
|
2818
|
+
} else if (cache.verdict === 'lost-money') {
|
|
2819
|
+
console.log(
|
|
2820
|
+
` ${c.yellow('!')} ${c.bold(wrap(t.profile.cacheLost(formatUsd(cache.deltaUsd), n(report.total.cacheWriteTokens), n(report.total.cacheReadTokens)), 74, ' '))}`,
|
|
2821
|
+
);
|
|
2822
|
+
// Only when it narrows the search. One label is the total again, said twice.
|
|
2823
|
+
if (lostLabels.length > 0 && report.byLabel.length > 1) {
|
|
2824
|
+
console.log(` ${c.dim(wrap(t.profile.cacheLostBy(namedLosers), 74, ' '))}`);
|
|
2825
|
+
}
|
|
2826
|
+
} else {
|
|
2827
|
+
if (cache.verdict === 'paid-off') {
|
|
2828
|
+
console.log(` ${c.dim(wrap(t.profile.cachePaidOff(formatUsd(-cache.deltaUsd)), 74, ' '))}`);
|
|
2829
|
+
} else if (cache.verdict === 'no-difference') {
|
|
2830
|
+
console.log(` ${c.dim(wrap(t.profile.cacheNoDifference(), 74, ' '))}`);
|
|
2831
|
+
}
|
|
2832
|
+
}
|
|
2833
|
+
|
|
2834
|
+
/**
|
|
2835
|
+
* A workload bleeding underneath a total that does not report a loss.
|
|
2836
|
+
*
|
|
2837
|
+
* The case the aggregate is actively hiding, so it prints as a warning: a cache
|
|
2838
|
+
* paying for itself on one label and losing on another nets out to a comfortable
|
|
2839
|
+
* number, and nothing else on screen would say otherwise.
|
|
2840
|
+
*
|
|
2841
|
+
* The sentence deliberately does not restate the total's verdict. It used to
|
|
2842
|
+
* open "Caching pays off overall", which this position cannot claim — it also
|
|
2843
|
+
* runs under `no-difference`, where the line immediately above has just said the
|
|
2844
|
+
* opposite, and under `unsettled`, where there is no verdict to report at all.
|
|
2845
|
+
*/
|
|
2846
|
+
if (lostLabels.length > 0 && cache.verdict !== 'lost-money') {
|
|
2847
|
+
console.log(
|
|
2848
|
+
` ${c.yellow('!')} ${c.dim(wrap(t.profile.cacheLostHidden(formatUsd(bleeding), namedLosers), 74, ' '))}`,
|
|
2849
|
+
);
|
|
2850
|
+
}
|
|
2851
|
+
|
|
2852
|
+
/**
|
|
2853
|
+
* A label that loses money only if its unstated TTL was the long one.
|
|
2854
|
+
*
|
|
2855
|
+
* The same ambiguity one level down, and it hides better here: a total whose
|
|
2856
|
+
* TTLs are mostly recorded reads as settled while one workload inside it is
|
|
2857
|
+
* entirely unstated. Listed apart from the confirmed losers because it is a
|
|
2858
|
+
* different claim — this one is conditional, and merging the two would make
|
|
2859
|
+
* every name in either list mean less.
|
|
2860
|
+
*/
|
|
2861
|
+
const maybeLostLabels = report.byLabel
|
|
2862
|
+
.map((r) => ({ row: r, cache: cacheEconomics(r.breakdown) }))
|
|
2863
|
+
.filter((r) => r.cache.verdict !== 'lost-money' && r.cache.worstCaseVerdict === 'lost-money');
|
|
2864
|
+
if (maybeLostLabels.length > 0) {
|
|
2865
|
+
console.log(
|
|
2866
|
+
` ${c.dim(wrap(t.profile.cacheTtlUnsettledLabels(listNames(maybeLostLabels)), 74, ' '))}`,
|
|
2867
|
+
);
|
|
2868
|
+
}
|
|
2869
|
+
|
|
2870
|
+
/**
|
|
2871
|
+
* Why, read from the prompt file itself — the loop `profile` could not close.
|
|
2872
|
+
*
|
|
2873
|
+
* The log carries counts, so this command can say *that* caching loses money
|
|
2874
|
+
* on a label and nothing more. `labels` in the config maps a label to the
|
|
2875
|
+
* prompt file it sends, and for each mapped label whose cache is failing —
|
|
2876
|
+
* losing money, or never attempted while money sat in cacheable input — the
|
|
2877
|
+
* file is read and the reason named: a prefix under the model's minimum,
|
|
2878
|
+
* stable tokens stranded behind the first placeholder, or a healthy file
|
|
2879
|
+
* whose problem is byte-identity between calls.
|
|
2880
|
+
*
|
|
2881
|
+
* Every sentence carries "as it is today": the file is whatever the
|
|
2882
|
+
* repository holds now, which may not be what produced the log, and a fresh
|
|
2883
|
+
* file presented as the history's explanation would be a figure attributed to
|
|
2884
|
+
* something it does not describe.
|
|
2885
|
+
*/
|
|
2886
|
+
const labelMap = config.labels ?? {};
|
|
2887
|
+
for (const { label, model: modelId, breakdown } of report.byLabelAndModel) {
|
|
2888
|
+
const file = labelMap[label];
|
|
2889
|
+
if (file === undefined) continue;
|
|
2890
|
+
const labelCache = cacheEconomics(breakdown);
|
|
2891
|
+
const failing =
|
|
2892
|
+
labelCache.verdict === 'lost-money' ||
|
|
2893
|
+
(labelCache.verdict === 'not-attempted' && breakdown.inputUsd > 0);
|
|
2894
|
+
if (!failing) continue;
|
|
2895
|
+
|
|
2896
|
+
let text: string;
|
|
2897
|
+
try {
|
|
2898
|
+
text = await readFile(file, 'utf8');
|
|
2899
|
+
} catch {
|
|
2900
|
+
console.log(` ${c.dim(wrap(t.profile.labelFileMissing(label, file), 74, ' '))}`);
|
|
2901
|
+
continue;
|
|
2902
|
+
}
|
|
2903
|
+
const model = pricing.byId.get(modelId);
|
|
2904
|
+
if (!model) continue;
|
|
2905
|
+
const analysis = analyzeCachePrefix(text, estimateTokens);
|
|
2906
|
+
const minimum = model.cacheMinTokens;
|
|
2907
|
+
console.log();
|
|
2908
|
+
if (minimum !== null && analysis.stablePrefixTokens < minimum) {
|
|
2909
|
+
console.log(
|
|
2910
|
+
` ${c.dim(wrap(t.profile.labelPrefixBelowMinimum(file, n(analysis.stablePrefixTokens), n(minimum), model.displayName), 74, ' '))}`,
|
|
2911
|
+
);
|
|
2912
|
+
} else if (analysis.staticTokensAfter >= 200) {
|
|
2913
|
+
console.log(
|
|
2914
|
+
` ${c.dim(wrap(t.profile.labelPrefixMovable(file, n(analysis.staticTokensAfter), n(analysis.stablePrefixTokens)), 74, ' '))}`,
|
|
2915
|
+
);
|
|
2916
|
+
} else {
|
|
2917
|
+
console.log(
|
|
2918
|
+
` ${c.dim(wrap(t.profile.labelPrefixHealthy(file, n(analysis.stablePrefixTokens), n(minimum ?? 0)), 74, ' '))}`,
|
|
2919
|
+
);
|
|
2920
|
+
}
|
|
2921
|
+
}
|
|
2922
|
+
|
|
2923
|
+
/**
|
|
2924
|
+
* The token budget against what actually goes up the wire.
|
|
2925
|
+
*
|
|
2926
|
+
* `budgets` gates a prompt *file*; the log records what the *call* carried —
|
|
2927
|
+
* system prompt, retrieved context, conversation history, tool results. The
|
|
2928
|
+
* two are related only through `labels`, and when the gap is large the gate
|
|
2929
|
+
* is real but tiny: a 2,000-token budget on a workload sending 47,000
|
|
2930
|
+
* tokens a call governs four per cent of what is sent, and nobody looking
|
|
2931
|
+
* at a green build would know it.
|
|
2932
|
+
*
|
|
2933
|
+
* Only stated when both ends are known — a label mapped to a file, and a
|
|
2934
|
+
* budget covering that file — and the share is named as approximate,
|
|
2935
|
+
* because the budget counts the file's tokens with the estimator while the
|
|
2936
|
+
* log counts what the provider billed. It says which part of the bill the
|
|
2937
|
+
* gate can see, and never that the budget is wrong.
|
|
2938
|
+
*/
|
|
2939
|
+
const budgetPatterns = Object.keys(config.budgets ?? {});
|
|
2940
|
+
if (budgetPatterns.length > 0) {
|
|
2941
|
+
for (const row of report.byLabel) {
|
|
2942
|
+
const file = labelMap[row.label];
|
|
2943
|
+
if (file === undefined || row.breakdown.calls === 0) continue;
|
|
2944
|
+
const pattern = mostSpecificMatch(budgetPatterns, file);
|
|
2945
|
+
if (pattern === null) continue;
|
|
2946
|
+
const budget = config.budgets![pattern]!;
|
|
2947
|
+
if (budget <= 0) continue;
|
|
2948
|
+
/**
|
|
2949
|
+
* Input tokens per call over this label — every class that is billed
|
|
2950
|
+
* as input, because a cached token was still sent and still counted
|
|
2951
|
+
* against the model's window.
|
|
2952
|
+
*/
|
|
2953
|
+
const perCall =
|
|
2954
|
+
(row.breakdown.inputTokens + row.breakdown.cacheReadTokens + row.breakdown.cacheWriteTokens) /
|
|
2955
|
+
row.breakdown.calls;
|
|
2956
|
+
if (perCall <= 0) continue;
|
|
2957
|
+
const share = budget / perCall;
|
|
2958
|
+
// Only when the gap is wide enough to change what somebody believes.
|
|
2959
|
+
// A budget covering most of the call is doing its job quietly.
|
|
2960
|
+
if (share >= 0.5) continue;
|
|
2961
|
+
console.log();
|
|
2962
|
+
console.log(
|
|
2963
|
+
` ${c.yellow('!')} ${c.dim(wrap(t.profile.budgetVsWire(row.label === UNLABELLED ? t.profile.unlabelled() : row.label, file, n(budget), n(Math.round(perCall)), pct(share)), 74, ' '))}`,
|
|
2964
|
+
);
|
|
2965
|
+
}
|
|
2966
|
+
}
|
|
2967
|
+
|
|
2968
|
+
/**
|
|
2969
|
+
* Whether the TTL fits how fast the turns arrive — the mechanism behind the
|
|
2970
|
+
* verdict above, readable only when the log carries a clock and a session.
|
|
2971
|
+
*
|
|
2972
|
+
* Rendered as four verdicts plus "could not be measured", the same
|
|
2973
|
+
* three-state discipline truncation uses: a workload with cache writes and no
|
|
2974
|
+
* clock has not been cleared, and silence here would read as fine.
|
|
2975
|
+
*/
|
|
2976
|
+
const TTL_SHOWN = 3;
|
|
2977
|
+
for (const fit of report.cacheTtlFit.slice(0, TTL_SHOWN)) {
|
|
2978
|
+
const name = fit.label === UNLABELLED ? t.profile.unlabelled() : fit.label;
|
|
2979
|
+
const gap = formatGap(fit.medianGapMs);
|
|
2980
|
+
if (fit.verdict === 'expires-before-reuse') {
|
|
2981
|
+
const line =
|
|
2982
|
+
fit.medianGapMs > TTL_1H_MS
|
|
2983
|
+
? t.profile.ttlFitExpiresBoth(name, fit.modelName, gap)
|
|
2984
|
+
: t.profile.ttlFitExpires(name, fit.modelName, gap);
|
|
2985
|
+
console.log(` ${c.yellow('!')} ${c.bold(wrap(line, 74, ' '))}`);
|
|
2986
|
+
} else if (fit.verdict === 'overlong-ttl') {
|
|
2987
|
+
console.log(
|
|
2988
|
+
` ${c.yellow('!')} ${c.bold(wrap(t.profile.ttlFitOverlong(name, fit.modelName, gap, formatUsd(fit.overpayUsd)), 74, ' '))}`,
|
|
2989
|
+
);
|
|
2990
|
+
} else if (fit.verdict === 'unsettled') {
|
|
2991
|
+
console.log(
|
|
2992
|
+
` ${c.dim(wrap(t.profile.ttlFitUnsettledGap(name, fit.modelName, gap), 74, ' '))}`,
|
|
2993
|
+
);
|
|
2994
|
+
} else {
|
|
2995
|
+
console.log(` ${c.dim(wrap(t.profile.ttlFitFits(name, fit.modelName, gap), 74, ' '))}`);
|
|
2996
|
+
}
|
|
2997
|
+
}
|
|
2998
|
+
if (report.total.cacheWriteTokens > 0 && report.cacheTtlFit.length === 0) {
|
|
2999
|
+
console.log(` ${c.dim(wrap(t.profile.ttlFitUnmeasured(), 74, ' '))}`);
|
|
3000
|
+
}
|
|
3001
|
+
|
|
3002
|
+
/**
|
|
3003
|
+
* Cache writes by conversations that never came back.
|
|
3004
|
+
*
|
|
3005
|
+
* Two sentences for the same tokens, and which one prints is decided by the
|
|
3006
|
+
* slice's own reads: with zero cache reads anywhere in the slice, nothing
|
|
3007
|
+
* read those writes — within the session, across sessions, at all — and the
|
|
3008
|
+
* ceiling collapses into a fact said loudly. With reads present, another
|
|
3009
|
+
* conversation sharing the prefix may have read them, the log cannot see
|
|
3010
|
+
* whose write a read hit, and the figure prints as the ceiling it is.
|
|
3011
|
+
*/
|
|
3012
|
+
const LEDGER_SHOWN = 3;
|
|
3013
|
+
if (report.singleTurnCacheWrites.length > 0) {
|
|
3014
|
+
const readsBySlice = new Map(
|
|
3015
|
+
report.byLabelAndModel.map((r) => [`${r.label}\n${r.model}`, r.breakdown.cacheReadTokens]),
|
|
3016
|
+
);
|
|
3017
|
+
for (const row of report.singleTurnCacheWrites.slice(0, LEDGER_SHOWN)) {
|
|
3018
|
+
const name = row.label === UNLABELLED ? t.profile.unlabelled() : row.label;
|
|
3019
|
+
const reads = readsBySlice.get(`${row.label}\n${row.model}`) ?? 0;
|
|
3020
|
+
if (reads === 0) {
|
|
3021
|
+
console.log(
|
|
3022
|
+
` ${c.yellow('!')} ${c.bold(wrap(t.profile.singleTurnConfirmed(name, row.modelName, n(row.singleTurnSessions), n(row.sessions), formatUsd(row.singleTurnWriteUsd)), 74, ' '))}`,
|
|
3023
|
+
);
|
|
3024
|
+
} else {
|
|
3025
|
+
console.log(
|
|
3026
|
+
` ${c.dim(wrap(t.profile.singleTurnCeiling(name, row.modelName, n(row.singleTurnSessions), n(row.sessions), formatUsd(row.singleTurnWriteUsd)), 74, ' '))}`,
|
|
3027
|
+
);
|
|
3028
|
+
}
|
|
3029
|
+
}
|
|
3030
|
+
}
|
|
3031
|
+
|
|
3032
|
+
/**
|
|
3033
|
+
* What one conversation costs — the question a total cannot answer, and the
|
|
3034
|
+
* one a per-seat price or a quota is set from. Median against p95, never a
|
|
3035
|
+
* mean: one runaway agent loop would drag a mean up and hide the ordinary
|
|
3036
|
+
* case, which is the figure somebody is actually pricing.
|
|
3037
|
+
*/
|
|
3038
|
+
for (const shape of report.sessionCosts.slice(0, 3)) {
|
|
3039
|
+
const name = shape.label === UNLABELLED ? t.profile.unlabelled() : shape.label;
|
|
3040
|
+
console.log();
|
|
3041
|
+
console.log(
|
|
3042
|
+
` ${c.dim(wrap(t.profile.sessionCost(name, shape.modelName, n(shape.sessions), formatUsd(shape.medianUsd), n(shape.medianTurns), formatUsd(shape.p95Usd), formatUsd(shape.maxUsd)), 74, ' '))}`,
|
|
3043
|
+
);
|
|
3044
|
+
/**
|
|
3045
|
+
* The tail, when there is one. A p95 far above the median is a shape a
|
|
3046
|
+
* quota can fix; a p95 beside it is a workload that is simply expensive,
|
|
3047
|
+
* and saying "hunt the tail" there would send somebody after nothing.
|
|
3048
|
+
* The threshold is in the sentence rather than hidden here.
|
|
3049
|
+
*/
|
|
3050
|
+
if (shape.medianUsd > 0 && shape.p95Usd > 10 * shape.medianUsd) {
|
|
3051
|
+
console.log(
|
|
3052
|
+
` ${c.yellow('!')} ${c.dim(wrap(t.profile.sessionCostTail((shape.p95Usd / shape.medianUsd).toFixed(0)), 74, ' '))}`,
|
|
3053
|
+
);
|
|
3054
|
+
}
|
|
3055
|
+
}
|
|
3056
|
+
|
|
3057
|
+
/**
|
|
3058
|
+
* A total that assumed a cache-write rate is a floor, and says so.
|
|
3059
|
+
*
|
|
3060
|
+
* Anthropic's 1-hour entry costs 2x input against the 5-minute entry's 1.25x. A
|
|
3061
|
+
* log carrying only the flat `cache_creation_input_tokens` cannot say which, so
|
|
3062
|
+
* the cheaper one is used — and the flattering direction is exactly the one this
|
|
3063
|
+
* tool refuses to take quietly.
|
|
3064
|
+
*/
|
|
3065
|
+
if (report.total.assumedWriteTtlCalls > 0) {
|
|
3066
|
+
console.log(
|
|
3067
|
+
` ${c.dim(wrap(t.profile.assumedWriteTtl(report.total.assumedWriteTtlCalls), 74, ' '))}`,
|
|
3068
|
+
);
|
|
3069
|
+
}
|
|
3070
|
+
|
|
3071
|
+
/**
|
|
3072
|
+
* The section this command is for, and the answer to the fairest complaint the
|
|
3073
|
+
* product has had: on a bill of twenty thousand, the rules recover two hundred.
|
|
3074
|
+
*
|
|
3075
|
+
* That figure is right — measured, three tokens out of three hundred and six on
|
|
3076
|
+
* an ordinary support prompt. The conclusion is not that the tool is worthless
|
|
3077
|
+
* but that it had been looking at the smallest line item. Which model a call
|
|
3078
|
+
* goes to moves 40% to 80%. The Batch API moves 50% flat. Both are priced here
|
|
3079
|
+
* from the reader's own tokens, at published rates, with no modelling in
|
|
3080
|
+
* between — and printed above the breakdowns, because a lever nobody scrolls to
|
|
3081
|
+
* is a lever nobody pulls.
|
|
3082
|
+
*
|
|
3083
|
+
* The ceiling on prompt shortening prints underneath them on purpose. A 1% win
|
|
3084
|
+
* reported without saying 1% of what is not information, and this repository
|
|
3085
|
+
* would rather say the uncomfortable number itself than let somebody else
|
|
3086
|
+
* discover it.
|
|
3087
|
+
*/
|
|
3088
|
+
const levers = billLevers(report, { catalogue: pricing });
|
|
3089
|
+
console.log();
|
|
3090
|
+
console.log(c.bold(t.profile.leversHeading()));
|
|
3091
|
+
/**
|
|
3092
|
+
* Every lever below describes a mixture when nothing carries a label.
|
|
3093
|
+
*
|
|
3094
|
+
* A 2,000-call classifier and a 400-call RAG pipeline merge into one slice, and
|
|
3095
|
+
* the section then offers a single route for two workloads that need different
|
|
3096
|
+
* answers — and `trazum route` would measure one prompt against a figure
|
|
3097
|
+
* covering both. The session case already tells the reader to add the field;
|
|
3098
|
+
* this one named the row `unlabelled` and said nothing, as though that were a
|
|
3099
|
+
* workload.
|
|
3100
|
+
*/
|
|
3101
|
+
const unlabelledOnly =
|
|
3102
|
+
report.byLabel.length === 1 && report.byLabel[0]!.label === UNLABELLED;
|
|
3103
|
+
if (unlabelledOnly && levers.slices.length > 0) {
|
|
3104
|
+
console.log(` ${c.yellow('!')} ${c.dim(wrap(t.profile.leversUnlabelled(), 74, ' '))}`);
|
|
3105
|
+
}
|
|
3106
|
+
if (levers.slices.length === 0) {
|
|
3107
|
+
console.log(` ${c.dim(wrap(t.profile.leversNone(), 74, ' '))}`);
|
|
3108
|
+
} else {
|
|
3109
|
+
for (const slice of levers.slices.slice(0, 5)) {
|
|
3110
|
+
const label = slice.label === UNLABELLED ? t.profile.unlabelled() : slice.label;
|
|
3111
|
+
console.log();
|
|
3112
|
+
/**
|
|
3113
|
+
* The headline is the **combined** figure, and the options underneath are
|
|
3114
|
+
* the ways to reach it — not rows to add up. Batching a routed call
|
|
3115
|
+
* discounts the cheaper model's price, so listing them separately printed
|
|
3116
|
+
* $12.60 and $10.50 against a slice that had spent $21.00: a saving larger
|
|
3117
|
+
* than the bill it came from, in the flattering direction.
|
|
3118
|
+
*/
|
|
3119
|
+
console.log(
|
|
3120
|
+
` ${c.green('→')} ${c.bold(wrap(t.profile.leverSlice(label, slice.modelName, formatUsd(slice.combinedUsd), pct(slice.shareOfBill)), 74, ' '))}`,
|
|
3121
|
+
);
|
|
3122
|
+
console.log(` ${c.dim(t.profile.leverCalls(t.profile.calls(slice.calls), formatUsd(slice.spentUsd)))}`);
|
|
3123
|
+
if (slice.route) {
|
|
3124
|
+
console.log(
|
|
3125
|
+
` ${c.dim('·')} ${c.dim(wrap(t.profile.leverRoute(slice.route.candidate.displayName, formatUsd(slice.route.savingUsd)), 74, ' '))}`,
|
|
3126
|
+
);
|
|
3127
|
+
}
|
|
3128
|
+
if (slice.batch) {
|
|
3129
|
+
console.log(
|
|
3130
|
+
` ${c.dim('·')} ${c.dim(wrap(t.profile.leverBatch(formatUsd(slice.batch.savingUsd)), 74, ' '))}`,
|
|
3131
|
+
);
|
|
3132
|
+
}
|
|
3133
|
+
// The arithmetic is exact and the quality question is untouched by it.
|
|
3134
|
+
// Naming the command is the difference between a saving and a gamble.
|
|
3135
|
+
if (slice.route) {
|
|
3136
|
+
console.log(
|
|
3137
|
+
` ${c.dim(wrap(t.profile.leverRouteVerify(slice.route.candidate.id), 74, ' '))}`,
|
|
3138
|
+
);
|
|
3139
|
+
}
|
|
3140
|
+
}
|
|
3141
|
+
}
|
|
3142
|
+
console.log();
|
|
3143
|
+
console.log(
|
|
3144
|
+
` ${c.dim(wrap(t.profile.leverPromptCeiling(formatUsd(levers.promptCeilingUsd), pct(levers.promptCeilingShare)), 74, ' '))}`,
|
|
3145
|
+
);
|
|
3146
|
+
|
|
3147
|
+
/**
|
|
3148
|
+
* `--what-if <model>`: these exact calls, at another model's rates.
|
|
3149
|
+
*
|
|
3150
|
+
* The levers above pick their own candidate; this answers the question the
|
|
3151
|
+
* reader arrived with. It is multiplication, not advice, and every part of
|
|
3152
|
+
* this section is built so it cannot be read as advice:
|
|
3153
|
+
*
|
|
3154
|
+
* - the caveat line prints **before** the figure, not after it;
|
|
3155
|
+
* - calls the target's context window could not have accepted are named as
|
|
3156
|
+
* impossible rather than priced as cheap, and their money is in none of
|
|
3157
|
+
* the totals;
|
|
3158
|
+
* - spend already on the target is stated separately, because a difference
|
|
3159
|
+
* computed over money that cannot move is a percentage of the wrong
|
|
3160
|
+
* denominator.
|
|
3161
|
+
*/
|
|
3162
|
+
if (whatIf !== null) {
|
|
3163
|
+
console.log();
|
|
3164
|
+
console.log(c.bold(t.profile.whatIfHeading(whatIf.target.displayName)));
|
|
3165
|
+
console.log(` ${c.dim(wrap(t.profile.whatIfAssumption(), 74, ' '))}`);
|
|
3166
|
+
console.log();
|
|
3167
|
+
if (whatIf.slices.length === 0) {
|
|
3168
|
+
console.log(` ${c.dim(wrap(t.profile.whatIfNothingToMove(), 74, ' '))}`);
|
|
3169
|
+
} else {
|
|
3170
|
+
const cheaper = whatIf.deltaUsd < 0;
|
|
3171
|
+
const line = t.profile.whatIfTotal(
|
|
3172
|
+
formatUsd(whatIf.currentUsd),
|
|
3173
|
+
formatUsd(whatIf.targetUsd),
|
|
3174
|
+
formatUsd(Math.abs(whatIf.deltaUsd)),
|
|
3175
|
+
);
|
|
3176
|
+
console.log(` ${cheaper ? c.green('→') : c.yellow('!')} ${c.bold(wrap(line, 74, ' '))}`);
|
|
3177
|
+
console.log(
|
|
3178
|
+
` ${c.dim(wrap(cheaper ? t.profile.whatIfCheaper() : t.profile.whatIfDearer(), 74, ' '))}`,
|
|
3179
|
+
);
|
|
3180
|
+
for (const slice of whatIf.slices.slice(0, 5)) {
|
|
3181
|
+
const label = slice.label === UNLABELLED ? t.profile.unlabelled() : slice.label;
|
|
3182
|
+
console.log(
|
|
3183
|
+
` ${c.dim('·')} ${c.dim(wrap(t.profile.whatIfSlice(label, slice.model, formatUsd(slice.currentUsd), formatUsd(slice.targetUsd)), 74, ' '))}`,
|
|
3184
|
+
);
|
|
3185
|
+
}
|
|
3186
|
+
}
|
|
3187
|
+
/**
|
|
3188
|
+
* The refusal, and it is loud. A call larger than the target's window is
|
|
3189
|
+
* not a cheaper call, and a comparison that priced it anyway would report
|
|
3190
|
+
* a saving for traffic that would have failed outright.
|
|
3191
|
+
*/
|
|
3192
|
+
for (const slice of whatIf.overContext.slice(0, 3)) {
|
|
3193
|
+
const label = slice.label === UNLABELLED ? t.profile.unlabelled() : slice.label;
|
|
3194
|
+
console.log(
|
|
3195
|
+
` ${c.yellow('!')} ${c.bold(wrap(t.profile.whatIfOverContext(label, n(slice.maxCallInputTokens), n(whatIf.target.contextWindow), formatUsd(slice.currentUsd)), 74, ' '))}`,
|
|
3196
|
+
);
|
|
3197
|
+
}
|
|
3198
|
+
// Money that is already there cannot move, and leaving it out of the
|
|
3199
|
+
// totals above is only honest if the reader is told it exists.
|
|
3200
|
+
if (whatIf.alreadyOnTarget.calls > 0) {
|
|
3201
|
+
console.log(
|
|
3202
|
+
` ${c.dim(wrap(t.profile.whatIfAlreadyThere(t.profile.calls(whatIf.alreadyOnTarget.calls), formatUsd(whatIf.alreadyOnTarget.usd)), 74, ' '))}`,
|
|
3203
|
+
);
|
|
3204
|
+
}
|
|
3205
|
+
// Models with no current price have no difference to state — their target
|
|
3206
|
+
// cost is knowable and the subtraction is not.
|
|
3207
|
+
if (whatIf.unpricedCalls > 0) {
|
|
3208
|
+
console.log(
|
|
3209
|
+
` ${c.dim(wrap(t.profile.whatIfUnpriced(t.profile.calls(whatIf.unpricedCalls), whatIf.unpricedModels.join(', ')), 74, ' '))}`,
|
|
3210
|
+
);
|
|
3211
|
+
}
|
|
3212
|
+
}
|
|
3213
|
+
|
|
3214
|
+
/**
|
|
3215
|
+
* What re-sending the conversation costs — the line nothing here could see.
|
|
3216
|
+
*
|
|
3217
|
+
* A chat or agent workload sends the whole conversation back on every turn, so
|
|
3218
|
+
* the input grows with the turn count and that growth is routinely the largest
|
|
3219
|
+
* item on the bill. A prompt file shows the system prompt and not the history; a
|
|
3220
|
+
* total shows the sum and not the shape.
|
|
3221
|
+
*
|
|
3222
|
+
* Reported as a **ceiling**, because part of the growth is the user's own new
|
|
3223
|
+
* messages and this reads counts rather than content, so it cannot separate the
|
|
3224
|
+
* two. Saying nothing because the exact split is unknowable would be worse: the
|
|
3225
|
+
* bound is exact, and the reader can act on it.
|
|
3226
|
+
*/
|
|
3227
|
+
if (report.conversations.length > 0) {
|
|
1886
3228
|
console.log();
|
|
1887
|
-
console.log(c.
|
|
3229
|
+
console.log(c.bold(t.profile.historyHeading()));
|
|
3230
|
+
for (const growth of report.conversations.slice(0, 3)) {
|
|
3231
|
+
const label = growth.label === UNLABELLED ? t.profile.unlabelled() : growth.label;
|
|
3232
|
+
console.log();
|
|
3233
|
+
console.log(
|
|
3234
|
+
` ${c.bold(wrap(t.profile.historyGrowth(label, growth.modelName, n(Math.round(growth.minTurnTokens)), n(Math.round(growth.maxTurnTokens)), n(growth.longestSession)), 74, ' '))}`,
|
|
3235
|
+
);
|
|
3236
|
+
console.log(
|
|
3237
|
+
` ${c.dim(wrap(t.profile.historyCeiling(formatUsd(growth.growthUsd), pct(growth.shareOfBill), formatUsd(growth.flatUsd), formatUsd(growth.inputUsd)), 74, ' '))}`,
|
|
3238
|
+
);
|
|
3239
|
+
}
|
|
3240
|
+
} else if (!report.hasSessions) {
|
|
3241
|
+
/**
|
|
3242
|
+
* Not the same as "no growth". A log without a session field cannot be asked
|
|
3243
|
+
* the question at all, and silence there would read as a clean bill of health
|
|
3244
|
+
* on the line most likely to be the biggest.
|
|
3245
|
+
*/
|
|
1888
3246
|
console.log();
|
|
1889
|
-
|
|
3247
|
+
console.log(c.bold(t.profile.historyHeading()));
|
|
3248
|
+
console.log(` ${c.dim(wrap(t.profile.historyNoSessions(), 74, ' '))}`);
|
|
1890
3249
|
}
|
|
1891
3250
|
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
3251
|
+
/**
|
|
3252
|
+
* Where the output spend concentrates — the actionable half of "output
|
|
3253
|
+
* dominates", which the headline above could only state as a total.
|
|
3254
|
+
*
|
|
3255
|
+
* Two bills with identical output spend want opposite responses. Six per cent
|
|
3256
|
+
* of calls holding half of it is a tail, and a tail has a cause worth a
|
|
3257
|
+
* morning; forty-five per cent is what evenly spread looks like, and the only
|
|
3258
|
+
* lever there is asking every answer to be shorter. The threshold between the
|
|
3259
|
+
* two is a quarter of the calls — far enough from both shapes that rounding
|
|
3260
|
+
* cannot flip the message, and stated here because it is a presentation choice,
|
|
3261
|
+
* not a measurement.
|
|
3262
|
+
*/
|
|
3263
|
+
if (report.outputShapes.length > 0) {
|
|
3264
|
+
console.log();
|
|
3265
|
+
console.log(c.bold(t.profile.outputShapeHeading()));
|
|
3266
|
+
for (const shape of report.outputShapes.slice(0, 3)) {
|
|
3267
|
+
const label = shape.label === UNLABELLED ? t.profile.unlabelled() : shape.label;
|
|
3268
|
+
const isTail = shape.heavyCallShare < 0.25;
|
|
3269
|
+
console.log();
|
|
3270
|
+
if (isTail) {
|
|
3271
|
+
console.log(
|
|
3272
|
+
` ${c.bold(wrap(t.profile.outputTail(label, shape.modelName, pct(shape.heavyCallShare), pct(shape.heavySpendShare), n(shape.aboveTokens), formatUsd(shape.outputUsd)), 74, ' '))}`,
|
|
3273
|
+
);
|
|
3274
|
+
console.log(` ${c.dim(wrap(t.profile.outputTailAdvice(), 74, ' '))}`);
|
|
3275
|
+
} else {
|
|
3276
|
+
console.log(
|
|
3277
|
+
` ${c.bold(wrap(t.profile.outputFlat(label, shape.modelName, pct(shape.heavyCallShare), pct(shape.heavySpendShare), formatUsd(shape.outputUsd)), 74, ' '))}`,
|
|
3278
|
+
);
|
|
3279
|
+
console.log(` ${c.dim(wrap(t.profile.outputFlatAdvice(), 74, ' '))}`);
|
|
3280
|
+
}
|
|
3281
|
+
/**
|
|
3282
|
+
* The ceilings a max_tokens cap actually wants, exact over the
|
|
3283
|
+
* histogram: every measured answer at or under the named number is
|
|
3284
|
+
* counted, none interpolated. Omitted when the covering bucket is the
|
|
3285
|
+
* open-ended last one, which has no ceiling to name honestly.
|
|
3286
|
+
*/
|
|
3287
|
+
if (shape.medianWithinTokens !== null && shape.p95WithinTokens !== null) {
|
|
3288
|
+
console.log(
|
|
3289
|
+
` ${c.dim(wrap(t.profile.outputPercentiles(n(shape.medianWithinTokens), n(shape.p95WithinTokens)), 74, ' '))}`,
|
|
3290
|
+
);
|
|
3291
|
+
}
|
|
3292
|
+
}
|
|
1900
3293
|
}
|
|
1901
3294
|
|
|
1902
3295
|
/**
|
|
1903
|
-
*
|
|
3296
|
+
* How big the calls themselves are — the other half of the bill.
|
|
1904
3297
|
*
|
|
1905
|
-
* The
|
|
1906
|
-
*
|
|
1907
|
-
*
|
|
1908
|
-
*
|
|
1909
|
-
*
|
|
3298
|
+
* The section above describes output; on a RAG or agent workload input is
|
|
3299
|
+
* most of the invoice, and a total could only ever say "input is 63% of
|
|
3300
|
+
* this bill", which nobody can act on. The actionable question is whether
|
|
3301
|
+
* the ordinary call is large or a few calls are enormous, and those two
|
|
3302
|
+
* shapes want opposite responses: a cap on something, or a shorter prompt.
|
|
1910
3303
|
*
|
|
1911
|
-
*
|
|
1912
|
-
*
|
|
3304
|
+
* Loud past **four times** the median — far enough from an even
|
|
3305
|
+
* distribution that a bucket boundary cannot flip the message, and stated
|
|
3306
|
+
* in the sentence rather than hidden here. Both figures are bucket
|
|
3307
|
+
* ceilings, so the ratio is coarse by construction and the copy says so.
|
|
1913
3308
|
*/
|
|
1914
|
-
if (report.
|
|
3309
|
+
if (report.inputShapes.length > 0) {
|
|
1915
3310
|
console.log();
|
|
1916
|
-
console.log(c.
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
|
|
3311
|
+
console.log(c.bold(t.profile.inputShapeHeading()));
|
|
3312
|
+
for (const shape of report.inputShapes.slice(0, 3)) {
|
|
3313
|
+
const label = shape.label === UNLABELLED ? t.profile.unlabelled() : shape.label;
|
|
3314
|
+
console.log();
|
|
3315
|
+
if (shape.medianWithinTokens === null || shape.p95WithinTokens === null || shape.p95OverMedian === null) {
|
|
3316
|
+
/**
|
|
3317
|
+
* The covering bucket is the open-ended last one, so there is no
|
|
3318
|
+
* ceiling to name. Said rather than skipped: a slice whose calls are
|
|
3319
|
+
* all above a million tokens is a finding, and silence would drop it.
|
|
3320
|
+
*/
|
|
3321
|
+
console.log(
|
|
3322
|
+
` ${c.bold(wrap(t.profile.inputHuge(label, shape.modelName, t.profile.calls(shape.calls), formatUsd(shape.inputUsd)), 74, ' '))}`,
|
|
3323
|
+
);
|
|
3324
|
+
continue;
|
|
3325
|
+
}
|
|
3326
|
+
const skewed = shape.p95OverMedian >= 4;
|
|
3327
|
+
const line = skewed
|
|
3328
|
+
? t.profile.inputSkewed(
|
|
3329
|
+
label,
|
|
3330
|
+
shape.modelName,
|
|
3331
|
+
n(shape.medianWithinTokens),
|
|
3332
|
+
n(shape.p95WithinTokens),
|
|
3333
|
+
shape.p95OverMedian.toFixed(1),
|
|
3334
|
+
formatUsd(shape.inputUsd),
|
|
3335
|
+
)
|
|
3336
|
+
: t.profile.inputEven(
|
|
3337
|
+
label,
|
|
3338
|
+
shape.modelName,
|
|
3339
|
+
n(shape.medianWithinTokens),
|
|
3340
|
+
n(shape.p95WithinTokens),
|
|
3341
|
+
formatUsd(shape.inputUsd),
|
|
3342
|
+
);
|
|
3343
|
+
console.log(` ${c.bold(wrap(line, 74, ' '))}`);
|
|
3344
|
+
console.log(
|
|
3345
|
+
` ${c.dim(wrap(skewed ? t.profile.inputSkewedAdvice() : t.profile.inputEvenAdvice(), 74, ' '))}`,
|
|
3346
|
+
);
|
|
3347
|
+
/**
|
|
3348
|
+
* What that size actually costs. A cache read is a tenth of input on
|
|
3349
|
+
* Anthropic, so a large slice reading almost everything from cache is a
|
|
3350
|
+
* very different bill from one paying full rate — and the token counts
|
|
3351
|
+
* alone cannot tell them apart.
|
|
3352
|
+
*/
|
|
3353
|
+
if (shape.cachedShare >= 0.5) {
|
|
3354
|
+
console.log(` ${c.dim(wrap(t.profile.inputMostlyCached(pct(shape.cachedShare)), 74, ' '))}`);
|
|
3355
|
+
} else if (shape.cachedShare < 0.1) {
|
|
3356
|
+
console.log(` ${c.dim(wrap(t.profile.inputFullRate(), 74, ' '))}`);
|
|
3357
|
+
}
|
|
3358
|
+
}
|
|
1937
3359
|
}
|
|
1938
3360
|
|
|
1939
3361
|
/**
|
|
1940
|
-
* The
|
|
3362
|
+
* The same request, sent again a moment later.
|
|
1941
3363
|
*
|
|
1942
|
-
*
|
|
1943
|
-
*
|
|
1944
|
-
*
|
|
3364
|
+
* A conversation's input grows with every turn, so two consecutive calls in
|
|
3365
|
+
* one conversation carrying the same size seconds apart is a thing going
|
|
3366
|
+
* wrong rather than a thing working — a retry after a timeout, an agent
|
|
3367
|
+
* step repeating, a loop. Loud, because the money bought nothing, and
|
|
3368
|
+
* hedged, because this reads counts and cannot see content: the sentence
|
|
3369
|
+
* says the pattern is *usually* a retry, never that it is one.
|
|
1945
3370
|
*/
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
|
|
1952
|
-
|
|
3371
|
+
if (report.repeatedTurns.length > 0) {
|
|
3372
|
+
console.log();
|
|
3373
|
+
console.log(c.bold(t.profile.repeatsHeading()));
|
|
3374
|
+
for (const row of report.repeatedTurns.slice(0, 3)) {
|
|
3375
|
+
const label = row.label === UNLABELLED ? t.profile.unlabelled() : row.label;
|
|
3376
|
+
console.log();
|
|
3377
|
+
console.log(
|
|
3378
|
+
` ${c.yellow('!')} ${c.bold(wrap(t.profile.repeatsFound(label, row.modelName, n(row.repeats), n(row.checkedCalls), n(Math.round(row.withinMs / 1000)), formatUsd(row.usd)), 74, ' '))}`,
|
|
3379
|
+
);
|
|
3380
|
+
console.log(` ${c.dim(wrap(t.profile.repeatsAdvice(), 74, ' '))}`);
|
|
3381
|
+
}
|
|
1953
3382
|
}
|
|
1954
3383
|
|
|
1955
|
-
const hitRate = cacheHitRate(report.total);
|
|
1956
|
-
console.log(
|
|
1957
|
-
hitRate === null
|
|
1958
|
-
? ` ${c.dim(wrap(t.profile.cacheNever(), 74, ' '))}`
|
|
1959
|
-
: ` ${c.dim(t.profile.cacheHit(pct(hitRate)))}`,
|
|
1960
|
-
);
|
|
1961
|
-
|
|
1962
3384
|
/**
|
|
1963
|
-
*
|
|
3385
|
+
* Output spend that bought answers cut off mid-generation — the one slice of
|
|
3386
|
+
* a bill that is waste without a counterpart. Paid in full, frequently
|
|
3387
|
+
* retried and billed again, and the truncated attempt bought nothing.
|
|
1964
3388
|
*
|
|
1965
|
-
*
|
|
1966
|
-
* log
|
|
1967
|
-
* the
|
|
1968
|
-
*
|
|
3389
|
+
* Three states, kept apart on purpose: waste found, none found on a log that
|
|
3390
|
+
* measured, and a log that never recorded a stop reason at all — which gets
|
|
3391
|
+
* the missing-field message, because silence there would read as a clean bill
|
|
3392
|
+
* of health on a question the log never asked.
|
|
1969
3393
|
*/
|
|
1970
|
-
if (report.total.
|
|
3394
|
+
if (report.total.truncatedCalls > 0 && report.total.outputUsd > 0) {
|
|
3395
|
+
console.log();
|
|
1971
3396
|
console.log(
|
|
1972
|
-
` ${c.
|
|
3397
|
+
` ${c.yellow('!')} ${c.bold(wrap(t.profile.truncatedWaste(t.profile.calls(report.total.truncatedCalls), formatUsd(report.total.truncatedOutputUsd), pct(report.total.truncatedOutputUsd / report.total.outputUsd)), 74, ' '))}`,
|
|
1973
3398
|
);
|
|
3399
|
+
/**
|
|
3400
|
+
* Which workloads are paying for it, and at what rate — the actionable
|
|
3401
|
+
* half the total hides. A 40% truncation rate is a max_tokens setting
|
|
3402
|
+
* that is simply wrong; 1% is a long tail, and the two call for opposite
|
|
3403
|
+
* responses.
|
|
3404
|
+
*
|
|
3405
|
+
* The rate is over calls that **recorded a stop reason**, never over all
|
|
3406
|
+
* calls: a workload that logs the field on half its traffic must not be
|
|
3407
|
+
* reported as though the unmeasured half completed. Both numbers print,
|
|
3408
|
+
* so the denominator is visible rather than implied.
|
|
3409
|
+
*/
|
|
3410
|
+
const truncatedLabels = report.byLabel
|
|
3411
|
+
.filter((row) => row.breakdown.truncatedCalls > 0)
|
|
3412
|
+
.sort((a, b) => b.breakdown.truncatedOutputUsd - a.breakdown.truncatedOutputUsd);
|
|
3413
|
+
if (truncatedLabels.length > 0 && report.byLabel.length > 1) {
|
|
3414
|
+
for (const row of truncatedLabels.slice(0, 3)) {
|
|
3415
|
+
const name = row.label === UNLABELLED ? t.profile.unlabelled() : row.label;
|
|
3416
|
+
console.log(
|
|
3417
|
+
` ${c.dim(wrap(t.profile.truncatedBy(name, n(row.breakdown.truncatedCalls), n(row.breakdown.stopReasonCalls), pct(row.breakdown.truncatedCalls / row.breakdown.stopReasonCalls), formatUsd(row.breakdown.truncatedOutputUsd)), 74, ' '))}`,
|
|
3418
|
+
);
|
|
3419
|
+
}
|
|
3420
|
+
}
|
|
3421
|
+
/**
|
|
3422
|
+
* The ceiling the completed answers actually needed, when the output
|
|
3423
|
+
* shapes measured it: "95% of the answers that finished fit within N
|
|
3424
|
+
* tokens" is the number a max_tokens cap wants, and it sits next to the
|
|
3425
|
+
* evidence that the current cap is too low. Measured on these calls,
|
|
3426
|
+
* promised for nothing.
|
|
3427
|
+
*/
|
|
3428
|
+
const ceiling = report.outputShapes.find((shape) => shape.p95WithinTokens !== null);
|
|
3429
|
+
if (ceiling !== undefined) {
|
|
3430
|
+
console.log(
|
|
3431
|
+
` ${c.dim(wrap(t.profile.truncatedCeiling(n(ceiling.p95WithinTokens!)), 74, ' '))}`,
|
|
3432
|
+
);
|
|
3433
|
+
}
|
|
3434
|
+
} else if (report.total.stopReasonCalls === 0) {
|
|
3435
|
+
console.log();
|
|
3436
|
+
console.log(` ${c.dim(wrap(t.profile.truncatedNotRecorded(), 74, ' '))}`);
|
|
3437
|
+
}
|
|
3438
|
+
|
|
3439
|
+
/**
|
|
3440
|
+
* This bill against the previous one — how spend actually gets out of hand.
|
|
3441
|
+
*
|
|
3442
|
+
* Nobody adds five thousand a month in one day; bills grow four percent a week
|
|
3443
|
+
* while every snapshot looks reasonable. This is the baseline gate the prompts
|
|
3444
|
+
* already had, applied to the money itself. **Positive means the bill grew**
|
|
3445
|
+
* (the diff convention), and every figure is between exactly these two files:
|
|
3446
|
+
* no period is assumed, so the call counts print beside the money for the
|
|
3447
|
+
* reader to judge comparability before judging the trend.
|
|
3448
|
+
*/
|
|
3449
|
+
if (previous !== null) {
|
|
3450
|
+
console.log();
|
|
3451
|
+
console.log(c.bold(t.profile.againstHeading()));
|
|
3452
|
+
if (previous.total.calls === 0) {
|
|
3453
|
+
console.log(` ${c.dim(wrap(t.profile.againstNothingPriced(), 74, ' '))}`);
|
|
3454
|
+
} else {
|
|
3455
|
+
const delta = report.total.totalUsd - previous.total.totalUsd;
|
|
3456
|
+
const growthPct =
|
|
3457
|
+
previous.total.totalUsd > 0
|
|
3458
|
+
? `${delta >= 0 ? '+' : ''}${((delta / previous.total.totalUsd) * 100).toFixed(1)}%`
|
|
3459
|
+
: '—';
|
|
3460
|
+
console.log(
|
|
3461
|
+
` ${c.bold(wrap(t.profile.againstTotals(formatUsd(previous.total.totalUsd), formatUsd(report.total.totalUsd), formatSignedUsd(delta), growthPct, t.profile.calls(previous.total.calls), t.profile.calls(report.total.calls)), 74, ' '))}`,
|
|
3462
|
+
);
|
|
3463
|
+
// Overlapping spans mean part of this "growth" is the same money on
|
|
3464
|
+
// both sides of the subtraction. Said after the figure it qualifies
|
|
3465
|
+
// and before the drivers built from it.
|
|
3466
|
+
if (againstOverlap !== null) {
|
|
3467
|
+
console.log(
|
|
3468
|
+
` ${c.yellow('!')} ${c.dim(wrap(t.profile.againstOverlap(dayOf(againstOverlap.fromMs), dayOf(againstOverlap.toMs)), 74, ' '))}`,
|
|
3469
|
+
);
|
|
3470
|
+
}
|
|
3471
|
+
|
|
3472
|
+
// Drivers: per-key contribution to the change, largest magnitude first,
|
|
3473
|
+
// computed once beside the gates so no rendering derives its own.
|
|
3474
|
+
const driverLine = (d: { key: string; was: number | null; now: number | null; delta: number }, shown: string): string =>
|
|
3475
|
+
d.was === null
|
|
3476
|
+
? t.profile.againstDriverNew(formatSignedUsd(d.delta), shown)
|
|
3477
|
+
: d.now === null
|
|
3478
|
+
? t.profile.againstDriverGone(formatSignedUsd(d.delta), shown)
|
|
3479
|
+
: t.profile.againstDriver(formatSignedUsd(d.delta), shown, formatUsd(d.was), formatUsd(d.now));
|
|
3480
|
+
|
|
3481
|
+
console.log();
|
|
3482
|
+
for (const d of labelDrivers.slice(0, 5)) {
|
|
3483
|
+
const line = driverLine(d, d.key === UNLABELLED ? t.profile.unlabelled() : d.key);
|
|
3484
|
+
console.log(` ${d.delta > 0 ? c.yellow(line) : c.dim(line)}`);
|
|
3485
|
+
}
|
|
3486
|
+
if (labelDrivers.length > 5) {
|
|
3487
|
+
console.log(` ${c.dim(t.profile.andMoreLabels(labelDrivers.length - 5))}`);
|
|
3488
|
+
}
|
|
3489
|
+
|
|
3490
|
+
/**
|
|
3491
|
+
* The same change, by model — where the mix moved. The label rows cannot
|
|
3492
|
+
* show it: a workload that kept its name and switched from Haiku to Opus
|
|
3493
|
+
* reads as "chat grew", and the reason is the model. Only printed when
|
|
3494
|
+
* more than one model is involved; with one model on both sides, this
|
|
3495
|
+
* section restates the totals line and says nothing new.
|
|
3496
|
+
*/
|
|
3497
|
+
const modelsInvolved = new Set([
|
|
3498
|
+
...previous.byModel.map((r) => r.model),
|
|
3499
|
+
...report.byModel.map((r) => r.model),
|
|
3500
|
+
]);
|
|
3501
|
+
if (modelDrivers.length > 0 && modelsInvolved.size > 1) {
|
|
3502
|
+
console.log();
|
|
3503
|
+
console.log(` ${c.dim(t.profile.againstByModel())}`);
|
|
3504
|
+
for (const d of modelDrivers.slice(0, 3)) {
|
|
3505
|
+
const line = driverLine(d, d.key);
|
|
3506
|
+
console.log(` ${d.delta > 0 ? c.yellow(line) : c.dim(line)}`);
|
|
3507
|
+
}
|
|
3508
|
+
}
|
|
3509
|
+
}
|
|
1974
3510
|
}
|
|
1975
3511
|
|
|
1976
3512
|
for (const [heading, rows] of [
|
|
@@ -1982,11 +3518,54 @@ async function commandProfile(args: Args, pricing: PricingCatalogue, t: CliMessa
|
|
|
1982
3518
|
console.log(c.bold(heading));
|
|
1983
3519
|
for (const [name, breakdown] of rows) {
|
|
1984
3520
|
const share = report.total.totalUsd > 0 ? breakdown.totalUsd / report.total.totalUsd : 0;
|
|
1985
|
-
console.log(` ${t.profile.row(name, formatUsd(breakdown.totalUsd), pct(share),
|
|
3521
|
+
console.log(` ${t.profile.row(name, formatUsd(breakdown.totalUsd), pct(share), t.profile.calls(breakdown.calls))}`);
|
|
3522
|
+
}
|
|
3523
|
+
}
|
|
3524
|
+
|
|
3525
|
+
/**
|
|
3526
|
+
* What this log cannot answer, and what would fix it.
|
|
3527
|
+
*
|
|
3528
|
+
* Every finding past the totals needs a field the format does not require,
|
|
3529
|
+
* and a reader who never adds them sees a report quietly missing half of
|
|
3530
|
+
* itself — with no way to tell "nothing to report" from "nothing recorded".
|
|
3531
|
+
* Named with counts rather than booleans: twelve labelled records out of
|
|
3532
|
+
* forty thousand is not a labelled log, and a boolean would call it one.
|
|
3533
|
+
*
|
|
3534
|
+
* Only fields that are actually missing are listed. A complete log gets no
|
|
3535
|
+
* section at all, because a paragraph of things that are fine is the
|
|
3536
|
+
* paragraph readers learn to skip.
|
|
3537
|
+
*/
|
|
3538
|
+
const coverage = report.fieldCoverage;
|
|
3539
|
+
if (coverage.parsed > 0) {
|
|
3540
|
+
const missing: string[] = [];
|
|
3541
|
+
const partial = (seen: number): string => `${n(seen)}/${n(coverage.parsed)}`;
|
|
3542
|
+
if (coverage.label < coverage.parsed) {
|
|
3543
|
+
missing.push(t.profile.needsLabel(partial(coverage.label)));
|
|
3544
|
+
}
|
|
3545
|
+
if (coverage.session < coverage.parsed) {
|
|
3546
|
+
missing.push(t.profile.needsSession(partial(coverage.session)));
|
|
3547
|
+
}
|
|
3548
|
+
if (coverage.ts < coverage.parsed) {
|
|
3549
|
+
missing.push(t.profile.needsTs(partial(coverage.ts)));
|
|
3550
|
+
}
|
|
3551
|
+
if (coverage.stopReason < coverage.parsed) {
|
|
3552
|
+
missing.push(t.profile.needsStopReason(partial(coverage.stopReason)));
|
|
3553
|
+
}
|
|
3554
|
+
if (coverage.cacheWrites > 0 && coverage.cacheTtl < coverage.cacheWrites) {
|
|
3555
|
+
missing.push(t.profile.needsCacheTtl(`${n(coverage.cacheTtl)}/${n(coverage.cacheWrites)}`));
|
|
3556
|
+
}
|
|
3557
|
+
if (missing.length > 0) {
|
|
3558
|
+
console.log();
|
|
3559
|
+
console.log(c.bold(t.profile.coverageHeading()));
|
|
3560
|
+
for (const line of missing) console.log(` ${c.dim(wrap(line, 74, ' '))}`);
|
|
1986
3561
|
}
|
|
1987
3562
|
}
|
|
1988
3563
|
|
|
1989
|
-
reportProfileGaps(report, t, n);
|
|
3564
|
+
reportProfileGaps(report, t, n, pricingStale);
|
|
3565
|
+
|
|
3566
|
+
await writeSideFiles();
|
|
3567
|
+
|
|
3568
|
+
applyGates();
|
|
1990
3569
|
}
|
|
1991
3570
|
|
|
1992
3571
|
/**
|
|
@@ -2000,7 +3579,20 @@ function reportProfileGaps(
|
|
|
2000
3579
|
report: ReturnType<typeof profileUsage>,
|
|
2001
3580
|
t: CliMessages,
|
|
2002
3581
|
n: (value: number) => string,
|
|
3582
|
+
stalePricing: { date: string; days: number } | null = null,
|
|
2003
3583
|
): void {
|
|
3584
|
+
/**
|
|
3585
|
+
* The one fact that silently invalidates every dollar above: a price table
|
|
3586
|
+
* the provider may have re-priced since. Loud, because unlike a skipped
|
|
3587
|
+
* line it does not name its own size — the error is exactly whatever the
|
|
3588
|
+
* provider changed, and only refreshing the table can say.
|
|
3589
|
+
*/
|
|
3590
|
+
if (stalePricing !== null) {
|
|
3591
|
+
console.log();
|
|
3592
|
+
console.log(
|
|
3593
|
+
` ${c.yellow('!')} ${c.dim(wrap(t.profile.pricesStale(stalePricing.date, stalePricing.days), 74, ' '))}`,
|
|
3594
|
+
);
|
|
3595
|
+
}
|
|
2004
3596
|
if (report.unpricedModels.length > 0) {
|
|
2005
3597
|
console.log();
|
|
2006
3598
|
console.log(
|
|
@@ -2015,6 +3607,150 @@ function reportProfileGaps(
|
|
|
2015
3607
|
}
|
|
2016
3608
|
}
|
|
2017
3609
|
|
|
3610
|
+
/**
|
|
3611
|
+
* `trazum route <log> --prompt-file <p> --cases <c>` — the loop the levers could
|
|
3612
|
+
* only point at.
|
|
3613
|
+
*
|
|
3614
|
+
* `profile` prices a route exactly and can say nothing whatever about whether the
|
|
3615
|
+
* cheaper model still does the job. So it printed a figure and a homework
|
|
3616
|
+
* assignment, and homework does not get done — the report said "$16.80 available,
|
|
3617
|
+
* go and test it" and the reader closed the terminal.
|
|
3618
|
+
*
|
|
3619
|
+
* This runs the test. Same prompt, two models, judged against **the expensive
|
|
3620
|
+
* model's own run-to-run variance** measured on the same cases in the same run. No
|
|
3621
|
+
* threshold anybody picked: the question is whether the cheaper model agrees with
|
|
3622
|
+
* the original more closely than the original agrees with itself.
|
|
3623
|
+
*
|
|
3624
|
+
* It costs three provider calls per case and says so before spending one of them,
|
|
3625
|
+
* exactly as `prune` does. A command that can spend somebody's money without
|
|
3626
|
+
* telling them first is a command they stop trusting.
|
|
3627
|
+
*/
|
|
3628
|
+
async function commandRoute(args: Args, pricing: PricingCatalogue, t: CliMessages): Promise<void> {
|
|
3629
|
+
const path = args.positional[0];
|
|
3630
|
+
if (path === undefined) {
|
|
3631
|
+
console.log();
|
|
3632
|
+
console.log(c.dim(wrap(t.route.noTarget(), 74, ' ')));
|
|
3633
|
+
console.log();
|
|
3634
|
+
return;
|
|
3635
|
+
}
|
|
3636
|
+
|
|
3637
|
+
const promptPath = stringFlag(args, 'prompt-file');
|
|
3638
|
+
const casesPath = stringFlag(args, 'cases');
|
|
3639
|
+
if (!promptPath || !casesPath) throw new Error(t.route.needsPrompt());
|
|
3640
|
+
|
|
3641
|
+
const report = profileUsage(await readFile(path, 'utf8'), { catalogue: pricing });
|
|
3642
|
+
const levers = billLevers(report, { catalogue: pricing });
|
|
3643
|
+
const wanted = stringFlag(args, 'label');
|
|
3644
|
+
/**
|
|
3645
|
+
* A `--label` nothing carries is a typo, and it gets the typo answer.
|
|
3646
|
+
*
|
|
3647
|
+
* Falling through to the generic "no route clears 1% of the bill: these calls
|
|
3648
|
+
* are already on the cheapest model of their family" asserted two falsehoods
|
|
3649
|
+
* at once when the log had a 60% route under a different name — a verdict
|
|
3650
|
+
* about calls the flag never selected.
|
|
3651
|
+
*/
|
|
3652
|
+
if (wanted !== undefined && !report.byLabel.some((r) => r.label === wanted)) {
|
|
3653
|
+
const available = report.byLabel
|
|
3654
|
+
.map((r) => (r.label === UNLABELLED ? t.profile.unlabelled() : r.label))
|
|
3655
|
+
.join(', ');
|
|
3656
|
+
console.log();
|
|
3657
|
+
console.log(c.dim(wrap(t.route.labelNotFound(wanted, available), 74, ' ')));
|
|
3658
|
+
console.log();
|
|
3659
|
+
return;
|
|
3660
|
+
}
|
|
3661
|
+
const slice = levers.slices.find(
|
|
3662
|
+
(s) => s.route !== null && (wanted === undefined || s.label === wanted),
|
|
3663
|
+
);
|
|
3664
|
+
if (!slice?.route) {
|
|
3665
|
+
console.log();
|
|
3666
|
+
console.log(c.dim(wrap(t.route.noRoute(), 74, ' ')));
|
|
3667
|
+
console.log();
|
|
3668
|
+
return;
|
|
3669
|
+
}
|
|
3670
|
+
|
|
3671
|
+
const prompt = await readFile(promptPath, 'utf8');
|
|
3672
|
+
const inputs = parseCases(await readFile(casesPath, 'utf8'));
|
|
3673
|
+
if (inputs.length === 0) throw new Error(t.errors.evalNoCases(casesPath));
|
|
3674
|
+
|
|
3675
|
+
const provider = providerFromEnv();
|
|
3676
|
+
if (!provider) throw new Error(t.errors.llmNotConfigured());
|
|
3677
|
+
/**
|
|
3678
|
+
* The candidate on the same endpoint and key, with the model swapped. Built
|
|
3679
|
+
* through the same factory rather than by hand so a provider that needs more
|
|
3680
|
+
* than a model id — a Bedrock region, a Vertex project — keeps whatever the
|
|
3681
|
+
* environment already gave it.
|
|
3682
|
+
*/
|
|
3683
|
+
const candidate = providerFromEnv({
|
|
3684
|
+
...process.env,
|
|
3685
|
+
TRAZUM_LLM_MODEL: slice.route.candidate.id,
|
|
3686
|
+
});
|
|
3687
|
+
if (!candidate) throw new Error(t.errors.llmNotConfigured());
|
|
3688
|
+
|
|
3689
|
+
const label = slice.label === UNLABELLED ? t.profile.unlabelled() : slice.label;
|
|
3690
|
+
const worth = formatUsd(slice.route.savingUsd);
|
|
3691
|
+
console.log();
|
|
3692
|
+
console.log(
|
|
3693
|
+
` ${c.bold(t.route.picked(label, slice.modelName, slice.route.candidate.displayName, worth, `${(slice.shareOfBill * 100).toFixed(1)}%`))}`,
|
|
3694
|
+
);
|
|
3695
|
+
/**
|
|
3696
|
+
* The money and the measurement have to describe the same calls.
|
|
3697
|
+
*
|
|
3698
|
+
* An unlabelled slice can hold a classifier and a RAG pipeline at once, and
|
|
3699
|
+
* this measures exactly one prompt. Attributing the verdict to a figure that
|
|
3700
|
+
* covers both is the fault this repository keeps finding in itself — a number
|
|
3701
|
+
* describing something other than what was measured. It cannot be detected from
|
|
3702
|
+
* counts, so it is stated rather than guessed at.
|
|
3703
|
+
*/
|
|
3704
|
+
if (slice.label === UNLABELLED) {
|
|
3705
|
+
console.log(` ${c.yellow('!')} ${c.dim(wrap(t.route.unlabelledSlice(), 74, ' '))}`);
|
|
3706
|
+
}
|
|
3707
|
+
console.log();
|
|
3708
|
+
console.log(
|
|
3709
|
+
` ${c.dim(wrap(t.route.willSpend(inputs.length * 3, provider.model, candidate.model), 74, ' '))}`,
|
|
3710
|
+
);
|
|
3711
|
+
|
|
3712
|
+
if (!boolFlag(args, 'yes')) {
|
|
3713
|
+
console.log(` ${c.dim(t.route.dryRun())}`);
|
|
3714
|
+
console.log();
|
|
3715
|
+
return;
|
|
3716
|
+
}
|
|
3717
|
+
|
|
3718
|
+
console.log(` ${c.dim(t.route.running(inputs.length))}`);
|
|
3719
|
+
// Same prompt on both sides. The axis under test is the model, and passing the
|
|
3720
|
+
// prompt twice is what says so at the call site.
|
|
3721
|
+
const result = await evaluate(prompt, prompt, inputs, provider, {
|
|
3722
|
+
candidateProvider: candidate,
|
|
3723
|
+
concurrency: numberFlag(args, 'concurrency', 3, t),
|
|
3724
|
+
});
|
|
3725
|
+
|
|
3726
|
+
if (boolFlag(args, 'json')) {
|
|
3727
|
+
console.log(JSON.stringify({ slice, evaluation: result }, null, 2));
|
|
3728
|
+
return;
|
|
3729
|
+
}
|
|
3730
|
+
|
|
3731
|
+
const asPct = (v: number): string => `${(v * 100).toFixed(0)}%`;
|
|
3732
|
+
console.log();
|
|
3733
|
+
console.log(
|
|
3734
|
+
` ${c.dim(wrap(t.route.agreement(asPct(result.crossAgreement), asPct(result.selfAgreement)), 74, ' '))}`,
|
|
3735
|
+
);
|
|
3736
|
+
console.log();
|
|
3737
|
+
if (result.verdict === 'inconclusive') {
|
|
3738
|
+
console.log(` ${c.bold(wrap(t.route.inconclusive(), 74, ' '))}`);
|
|
3739
|
+
} else if (result.verdict === 'diverges') {
|
|
3740
|
+
console.log(` ${c.yellow('!')} ${c.bold(wrap(t.route.diverges(worth), 74, ' '))}`);
|
|
3741
|
+
} else {
|
|
3742
|
+
console.log(` ${c.green('✓')} ${c.bold(wrap(t.route.holds(worth), 74, ' '))}`);
|
|
3743
|
+
}
|
|
3744
|
+
/**
|
|
3745
|
+
* Printed on every verdict including the good one. Agreement is not
|
|
3746
|
+
* correctness: this measures whether the answers moved, not whether they were
|
|
3747
|
+
* ever right, and a green tick that let somebody forget that would be the tool
|
|
3748
|
+
* overstating what it knows.
|
|
3749
|
+
*/
|
|
3750
|
+
console.log(` ${c.dim(wrap(t.route.yours(), 74, ' '))}`);
|
|
3751
|
+
console.log();
|
|
3752
|
+
}
|
|
3753
|
+
|
|
2018
3754
|
/**
|
|
2019
3755
|
* `trazum baseline <dir>` — record what the estate costs now.
|
|
2020
3756
|
*
|
|
@@ -3506,7 +5242,10 @@ async function main(): Promise<void> {
|
|
|
3506
5242
|
await commandBaseline(args, config, pricing, t, locale);
|
|
3507
5243
|
break;
|
|
3508
5244
|
case 'profile':
|
|
3509
|
-
await commandProfile(args, pricing, t);
|
|
5245
|
+
await commandProfile(args, config, pricing, t);
|
|
5246
|
+
break;
|
|
5247
|
+
case 'route':
|
|
5248
|
+
await commandRoute(args, pricing, t);
|
|
3510
5249
|
break;
|
|
3511
5250
|
case 'eval':
|
|
3512
5251
|
await commandEval(args, config, t, locale);
|