@trazum/cli 1.82.0 → 1.83.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/i18n/en.d.ts.map +1 -1
- package/dist/i18n/en.js +66 -1
- package/dist/i18n/en.js.map +1 -1
- package/dist/i18n/es.d.ts.map +1 -1
- package/dist/i18n/es.js +70 -1
- package/dist/i18n/es.js.map +1 -1
- package/dist/i18n/types.d.ts +69 -1
- package/dist/i18n/types.d.ts.map +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2012 -1486
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
- package/src/i18n/en.ts +84 -1
- package/src/i18n/es.ts +88 -1
- package/src/i18n/types.ts +71 -1
- package/src/index.ts +2714 -2091
package/dist/index.js
CHANGED
|
@@ -7,7 +7,7 @@ import { createInterface } from 'node:readline/promises';
|
|
|
7
7
|
import { dirname, join, resolve as resolvePath } from 'node:path';
|
|
8
8
|
import { fileURLToPath } from 'node:url';
|
|
9
9
|
import { gunzipSync } from 'node:zlib';
|
|
10
|
-
import { CONTRACT_NAMES, contractSchema, applyRewrites, BASELINE_FILENAME, BASELINE_VERSION, breaches, cacheableMinimum, analyzeCachePrefix, billLevers, bucketedCacheEconomics, bucketedProfile, buildHistory, buildPlan, connectorFor, CONNECTORS, normalizeAnthropicUsage, normalizeOpenAIUsage, bucketsFromRecords, evaluateWatch, firedKey, pruneRecords, recordsFromBuckets, storeInventory, storedReportFrom, verifyPlan, cacheEconomics, cacheHitRate, contextPressure, comparePrompts, compareToBaseline, computeSavings, bandGoverns, countTokensAnthropic, DEFAULT_USAGE, budgetPositions, conform, BREAK_EVEN_BAND, allocate, annualRecord, replayCommitment, coversTheTerm, runExperiment, qualityGate, semanticPassCost, verifySemanticProposals, SEMANTIC_SYSTEM_PROMPT, ladderPosition, validateLadder, outcomeReport, rankPerOutcome, FAILURE_POLICIES, detectFromSource, matchLocale, parsePlanDocument, waiverDay, waiverHistory, proposeInit, MIN_RATE_DAYS, parseConfig, coverageDrift, driversBetween, explainGateFailure, assignSources, fleetRollup, rollUp, heartbeats, ruleYield, labelCoverage, measuredUsage, gateMargin, GATE_MARGIN_TIGHT, estimateTokens, evaluate, extractPrompts, findExamples, formatBaseline, formatSignedUsd, formatUsd, getMessages, getModel, hasMarker, LOCALES, MAX_BASELINE_BYTES, matchGlob, MAX_INPUT_CHARS, moneyIsComparable, mostSpecificMatch, assemble, interview, nearestName, slot, SLOT_IDS, optimize, parseBaseline, PHRASE_LANGUAGES, detectTextLanguage, dictionaryStanding, languagesWithStanding, indexUsage, parseUsageLine, plannedCalls, claudeCodeRecords, ESTIMATE_ERROR_BAND_PCT, bandFor, foreignTokenizer, measuredForeignError, heliconeRecords, litellmRecords, otelRecords, ownRate, positionAt, positionReport, profilePrompt, profileToCsv, profileUsage, promptId, providerFromEnv, pruneExamples, refineWithLlm, rejectionText, reorderForCache, repriceProfile, switchAnalysis, reviewAgeDays, reviewExamples, RULES, sharedPrefixes, sharesOf, SOURCE_EXTENSIONS, suggestRewrites, toOtlpMetrics, toPromptfoo, TTL_1H_MS, UNLABELLED, withExactTokenCounts, } from '@trazum/core';
|
|
10
|
+
import { CONTRACT_NAMES, contractSchema, applyRewrites, BASELINE_FILENAME, BASELINE_VERSION, breaches, cacheableMinimum, analyzeCachePrefix, billLevers, bucketedCacheEconomics, bucketedProfile, buildHistory, buildPlan, connectorFor, CONNECTORS, normalizeAnthropicUsage, normalizeOpenAIUsage, bucketsFromRecords, evaluateWatch, firedKey, pruneRecords, recordsFromBuckets, storeInventory, storedReportFrom, verifyPlan, cacheEconomics, cacheHitRate, contextPressure, comparePrompts, compareToBaseline, computeSavings, bandGoverns, countTokensAnthropic, DEFAULT_USAGE, budgetPositions, conform, BREAK_EVEN_BAND, allocate, annualRecord, replayCommitment, coversTheTerm, runExperiment, qualityGate, semanticPassCost, verifySemanticProposals, SEMANTIC_SYSTEM_PROMPT, ladderPosition, validateLadder, outcomeReport, rankPerOutcome, FAILURE_POLICIES, detectFromSource, matchLocale, parsePlanDocument, waiverDay, waiverHistory, proposeInit, MIN_RATE_DAYS, parseConfig, coverageDrift, driversBetween, explainGateFailure, assignSources, fleetRollup, rollUp, heartbeats, ruleYield, labelCoverage, measuredUsage, gateMargin, GATE_MARGIN_TIGHT, estimateTokens, evaluate, extractPrompts, findExamples, formatBaseline, formatSignedUsd, formatUsd, receiptFrom, getMessages, getModel, hasMarker, LOCALES, MAX_BASELINE_BYTES, matchGlob, MAX_INPUT_CHARS, moneyIsComparable, mostSpecificMatch, assemble, interview, nearestName, slot, SLOT_IDS, optimize, parseBaseline, PHRASE_LANGUAGES, detectTextLanguage, dictionaryStanding, languagesWithStanding, indexUsage, parseUsageLine, plannedCalls, claudeCodeRecords, ESTIMATE_ERROR_BAND_PCT, bandFor, foreignTokenizer, measuredForeignError, heliconeRecords, langsmithRecords, litellmRecords, otelRecords, ownRate, positionAt, positionReport, PRICING_LAST_REVIEWED, PROVIDER_REVIEWED, STALE_PRICING_DAYS, profilePrompt, profileToCsv, profileUsage, promptId, providerFromEnv, pruneExamples, refineWithLlm, rejectionText, reorderForCache, repriceProfile, switchAnalysis, reviewAgeDays, reviewExamples, RULES, sharedPrefixes, sharesOf, isOffered, SOURCE_EXTENSIONS, suggestRewrites, toOtlpMetrics, toPromptfoo, TTL_1H_MS, UNLABELLED, withExactTokenCounts, } from '@trazum/core';
|
|
11
11
|
import { cacheDir, cacheStats, cachingProvider, clearCache } from './suggest-cache.js';
|
|
12
12
|
import { dayOf, formatGap, median, spanDays } from './time.js';
|
|
13
13
|
// Everything that reads the filesystem, on its own entry point so the web
|
|
@@ -362,7 +362,7 @@ const COMMAND_FLAGS = {
|
|
|
362
362
|
],
|
|
363
363
|
check: ['max-tokens', 'level', 'exact-tokens', 'markdown-out', 'baseline', 'files-from'],
|
|
364
364
|
baseline: ['model', 'calls', 'output-tokens', 'cache-hit-rate', 'batch', 'exact-tokens', 'out', 'o'],
|
|
365
|
-
profile: ['json', 'pricing', 'pricing-live', 'against', 'what-if', 'markdown-out', 'html-out', 'csv-out', 'csv-shape', 'max-usd', 'max-growth-usd', 'max-cache-loss-usd', 'max-day-usd', 'max-session-usd', 'label', 'since', 'until', 'dry-run', 'markdown-summary', 'by-source'],
|
|
365
|
+
profile: ['json', 'pricing', 'pricing-live', 'against', 'what-if', 'markdown-out', 'html-out', 'csv-out', 'csv-shape', 'max-usd', 'max-growth-usd', 'max-cache-loss-usd', 'max-day-usd', 'max-session-usd', 'label', 'since', 'until', 'dry-run', 'markdown-summary', 'by-source', 'allow-empty'],
|
|
366
366
|
plan: ['json', 'out', 'markdown-out', 'min-usd', 'pricing', 'pricing-live'],
|
|
367
367
|
verify: ['against', 'gate', 'json', 'markdown-out', 'pricing', 'pricing-live'],
|
|
368
368
|
history: ['store', 'json', 'markdown-out'],
|
|
@@ -381,10 +381,12 @@ const COMMAND_FLAGS = {
|
|
|
381
381
|
schema: [],
|
|
382
382
|
rollup: ['json', 'html-out'],
|
|
383
383
|
position: ['json', 'html-out', 'pricing', 'pricing-live'],
|
|
384
|
+
receipt: ['out', 'o', 'stamp', 'pricing', 'pricing-live'],
|
|
384
385
|
'from-claude-code': ['label', 'label-from-project', 'out', 'o', 'state'],
|
|
385
386
|
'from-otel': ['label-from-service', 'out', 'o'],
|
|
386
387
|
'from-litellm': ['out', 'o'],
|
|
387
388
|
'from-helicone': ['out', 'o'],
|
|
389
|
+
'from-langsmith': ['out', 'o'],
|
|
388
390
|
switch: ['to', 'migration-usd', 'cases'],
|
|
389
391
|
ownrate: ['gpu-usd-hour', 'tokens-per-second', 'utilization'],
|
|
390
392
|
pulse: ['json', 'max-stale-hours'],
|
|
@@ -769,7 +771,7 @@ measured = null) {
|
|
|
769
771
|
*
|
|
770
772
|
* `optimize` is the first command anybody runs, and it reports the smallest
|
|
771
773
|
* line item on the bill: measured, about 1% of a monthly figure. Everything
|
|
772
|
-
* that moves
|
|
774
|
+
* that moves 60% to 80% — which model the call goes to, the Batch API,
|
|
773
775
|
* caching, what re-sending the conversation costs — lives in `profile`, which
|
|
774
776
|
* needs a usage log a new reader does not have and has no reason to go looking
|
|
775
777
|
* for.
|
|
@@ -993,7 +995,23 @@ function defaultModelFor(provider, pricing) {
|
|
|
993
995
|
// not an edge one.
|
|
994
996
|
const RANK = { small: 0, mid: 1, large: 2, frontier: 3 };
|
|
995
997
|
const want = RANK[getModel(DEFAULT_USAGE.model).capability] ?? 2;
|
|
996
|
-
|
|
998
|
+
/**
|
|
999
|
+
* Offered first, and a retired model only when the provider has nothing else.
|
|
1000
|
+
*
|
|
1001
|
+
* `isOffered` is the rule everywhere a model is recommended, and this is not
|
|
1002
|
+
* quite that: nothing is being recommended here, a provider's call is being
|
|
1003
|
+
* priced. The distinction matters because three providers in this catalogue
|
|
1004
|
+
* have **only** retired ids today, and dropping them would return null and
|
|
1005
|
+
* fall through to the global default — printing "goes to deepseek / priced as
|
|
1006
|
+
* Claude Opus 5", which is wrong by an order of magnitude in the direction
|
|
1007
|
+
* this function exists to avoid.
|
|
1008
|
+
*
|
|
1009
|
+
* So a retired model's price is still that provider's price and is used as a
|
|
1010
|
+
* last resort. What the reader is not left to infer is that it is current:
|
|
1011
|
+
* `buildAdvisories` warns by name on a retired id, quoting the provider.
|
|
1012
|
+
*/
|
|
1013
|
+
const offered = pricing.models.filter((m) => m.provider === provider && isOffered(m));
|
|
1014
|
+
const candidates = offered.length > 0 ? offered : pricing.models.filter((m) => m.provider === provider);
|
|
997
1015
|
const best = candidates.reduce((chosen, m) => {
|
|
998
1016
|
if (chosen === null)
|
|
999
1017
|
return m;
|
|
@@ -2288,6 +2306,77 @@ async function commandFromClaudeCode(args, t) {
|
|
|
2288
2306
|
console.error(t.fromClaudeCode.labelled());
|
|
2289
2307
|
console.error(t.fromClaudeCode.skipped(otherLines, unparseable, withoutUsage));
|
|
2290
2308
|
}
|
|
2309
|
+
/**
|
|
2310
|
+
* `trazum receipt <usage.jsonl|dir>` — the bill's counts, with the provenance
|
|
2311
|
+
* of every figure attached.
|
|
2312
|
+
*
|
|
2313
|
+
* A profile answers a question on the terminal that ran it. The same figures
|
|
2314
|
+
* filed against an invoice, or read next month by somebody who was not there,
|
|
2315
|
+
* stop answering it: a dollar total with no provenance cannot tell a repricing
|
|
2316
|
+
* from a team whose spend moved. This writes the shape that keeps answering.
|
|
2317
|
+
*
|
|
2318
|
+
* **It sends nothing anywhere.** There is no endpoint, no key, no retry and no
|
|
2319
|
+
* queue: the document goes to a path you name or to standard output, and what
|
|
2320
|
+
* happens to it next is not this command's business. A command in this package
|
|
2321
|
+
* that phoned home would break the roadmap's first rule in the same release
|
|
2322
|
+
* that claims to be protecting it.
|
|
2323
|
+
*
|
|
2324
|
+
* **The document is undated unless you ask for a stamp**, and that is a
|
|
2325
|
+
* deliberate default rather than an omission. This product's first promise is
|
|
2326
|
+
* the same answer every time; a command that wrote the current clock into its
|
|
2327
|
+
* output would produce different bytes on every run, so it could not be
|
|
2328
|
+
* committed, diffed in a pull request, or compared against yesterday's. The
|
|
2329
|
+
* period the figures actually cover is already in the document, read from the
|
|
2330
|
+
* log's own clock. `--stamp` adds the emission time for whoever wants it, and
|
|
2331
|
+
* an undated receipt is valid: `receiptFrom` says so, and says why.
|
|
2332
|
+
*
|
|
2333
|
+
* The offline guard found this. The command stamped by default, two runs
|
|
2334
|
+
* disagreed, and the test that exists to catch a hidden network call caught a
|
|
2335
|
+
* hidden clock instead.
|
|
2336
|
+
*
|
|
2337
|
+
* The redaction property is not enforced here. It is a property of
|
|
2338
|
+
* `receiptFrom`, which takes a `UsageProfileReport` -- a shape with no field
|
|
2339
|
+
* that can hold prompt text -- and it is held by `receipt-redaction.test.js`
|
|
2340
|
+
* planting the 4 things that must never appear. This function only chooses the
|
|
2341
|
+
* log and the destination.
|
|
2342
|
+
*/
|
|
2343
|
+
async function commandReceipt(args, pricing, t) {
|
|
2344
|
+
const file = args.positional[0];
|
|
2345
|
+
if (file === undefined)
|
|
2346
|
+
throw new Error(t.receipt.noLog());
|
|
2347
|
+
const text = await readUsageLog(file, t);
|
|
2348
|
+
const report = profileUsage(text, { catalogue: pricing });
|
|
2349
|
+
const document = receiptFrom(report, pricing, boolFlag(args, 'stamp') ? { emittedAt: new Date() } : {});
|
|
2350
|
+
const json = JSON.stringify(document, null, 2);
|
|
2351
|
+
const out = stringFlag(args, 'out') ?? stringFlag(args, 'o');
|
|
2352
|
+
if (out !== undefined) {
|
|
2353
|
+
await writeFile(out, json + '\n', 'utf8');
|
|
2354
|
+
console.error(t.receipt.written(out, document.lines.length));
|
|
2355
|
+
}
|
|
2356
|
+
else {
|
|
2357
|
+
console.log(json);
|
|
2358
|
+
}
|
|
2359
|
+
/*
|
|
2360
|
+
* Everything below goes to stderr, so `trazum receipt log.jsonl > receipt.json`
|
|
2361
|
+
* writes a document and not a document with a summary stapled to the front.
|
|
2362
|
+
* The gaps are read off the document rather than recomputed, so what the
|
|
2363
|
+
* reader is told and what the file carries cannot drift apart.
|
|
2364
|
+
*/
|
|
2365
|
+
if (document.lines.length === 0) {
|
|
2366
|
+
console.error(t.receipt.nothingToBill());
|
|
2367
|
+
}
|
|
2368
|
+
else {
|
|
2369
|
+
console.error(t.receipt.summary(document.lines.length, formatUsd(document.total.usd)));
|
|
2370
|
+
}
|
|
2371
|
+
for (const gap of document.gaps) {
|
|
2372
|
+
if (gap.kind === 'unpriced')
|
|
2373
|
+
console.error(t.receipt.unpriced(gap.models.length, gap.calls));
|
|
2374
|
+
if (gap.kind === 'unread-lines')
|
|
2375
|
+
console.error(t.receipt.unread(gap.count));
|
|
2376
|
+
if (gap.kind === 'no-clock')
|
|
2377
|
+
console.error(t.receipt.noClock());
|
|
2378
|
+
}
|
|
2379
|
+
}
|
|
2291
2380
|
/**
|
|
2292
2381
|
* `trazum from-otel <file|dir>` — OpenTelemetry GenAI spans as a usage log.
|
|
2293
2382
|
*
|
|
@@ -2582,6 +2671,94 @@ async function commandFromLiteLlm(args, t) {
|
|
|
2582
2671
|
if (unparseable > 0)
|
|
2583
2672
|
console.error(t.fromLiteLlm.unparseable(unparseable));
|
|
2584
2673
|
}
|
|
2674
|
+
/**
|
|
2675
|
+
* `trazum from-langsmith <file|dir>` — a LangSmith run export as a usage log.
|
|
2676
|
+
*
|
|
2677
|
+
* The fifth converter, and the one where the unit is wrong before anything
|
|
2678
|
+
* else can be right: LangSmith records a **run**, and a trace is a tree of
|
|
2679
|
+
* them. The chain that wrapped a model call carries the same tokens as the
|
|
2680
|
+
* call, so summing the export bills them once per level. Only `run_type: llm`
|
|
2681
|
+
* is a call, and every run that is not one is counted out loud — skipping two
|
|
2682
|
+
* thirds of a file silently would look exactly like reading it.
|
|
2683
|
+
*
|
|
2684
|
+
* The model is refused rather than inferred. There is no model column; the
|
|
2685
|
+
* name lives in the metadata, and the obvious substitute is the run's own
|
|
2686
|
+
* `name`, which LangChain sets to the client class. Pricing a call by
|
|
2687
|
+
* `ChatAnthropic` would attribute a figure to something it does not describe.
|
|
2688
|
+
*
|
|
2689
|
+
* LangSmith's own cost is reported on its own line and never merged into
|
|
2690
|
+
* anything Trazum computes, the way `from-litellm` keeps the gateway's
|
|
2691
|
+
* arithmetic apart. Two price tables summed into one total is how a report
|
|
2692
|
+
* becomes quietly wrong.
|
|
2693
|
+
*/
|
|
2694
|
+
async function commandFromLangsmith(args, t) {
|
|
2695
|
+
const target = args.positional[0];
|
|
2696
|
+
if (target === undefined)
|
|
2697
|
+
throw new Error(t.fromLangsmith.noPath());
|
|
2698
|
+
let info;
|
|
2699
|
+
try {
|
|
2700
|
+
info = await stat(target);
|
|
2701
|
+
}
|
|
2702
|
+
catch {
|
|
2703
|
+
throw new Error(t.fromLangsmith.notFound(target));
|
|
2704
|
+
}
|
|
2705
|
+
const files = [];
|
|
2706
|
+
if (info.isDirectory()) {
|
|
2707
|
+
const entries = await readdir(target, { recursive: true, withFileTypes: true });
|
|
2708
|
+
for (const entry of entries) {
|
|
2709
|
+
if (entry.isFile() && /\.(json|jsonl|ndjson)$/i.test(entry.name)) {
|
|
2710
|
+
files.push(join(entry.parentPath, entry.name));
|
|
2711
|
+
}
|
|
2712
|
+
}
|
|
2713
|
+
files.sort();
|
|
2714
|
+
if (files.length === 0)
|
|
2715
|
+
throw new Error(t.fromLangsmith.noExports(target));
|
|
2716
|
+
}
|
|
2717
|
+
else {
|
|
2718
|
+
files.push(target);
|
|
2719
|
+
}
|
|
2720
|
+
const lines = [];
|
|
2721
|
+
let rows = 0;
|
|
2722
|
+
let notModelCalls = 0;
|
|
2723
|
+
let unnamedModel = 0;
|
|
2724
|
+
let noTokens = 0;
|
|
2725
|
+
let unparseable = 0;
|
|
2726
|
+
let reportedCostUsd = null;
|
|
2727
|
+
for (const file of files) {
|
|
2728
|
+
const conversion = langsmithRecords(await readFile(file, 'utf8'));
|
|
2729
|
+
for (const record of conversion.records)
|
|
2730
|
+
lines.push(JSON.stringify(record));
|
|
2731
|
+
rows += conversion.rows;
|
|
2732
|
+
notModelCalls += conversion.notModelCalls;
|
|
2733
|
+
unnamedModel += conversion.unnamedModel;
|
|
2734
|
+
noTokens += conversion.noTokens;
|
|
2735
|
+
unparseable += conversion.unparseable;
|
|
2736
|
+
if (conversion.reportedCostUsd !== null) {
|
|
2737
|
+
reportedCostUsd = (reportedCostUsd ?? 0) + conversion.reportedCostUsd;
|
|
2738
|
+
}
|
|
2739
|
+
}
|
|
2740
|
+
const out = stringFlag(args, 'out') ?? stringFlag(args, 'o');
|
|
2741
|
+
if (out !== undefined) {
|
|
2742
|
+
await writeFile(out, lines.join('\n') + (lines.length > 0 ? '\n' : ''), 'utf8');
|
|
2743
|
+
console.error(t.fromLangsmith.written(out));
|
|
2744
|
+
}
|
|
2745
|
+
else {
|
|
2746
|
+
for (const line of lines)
|
|
2747
|
+
console.log(line);
|
|
2748
|
+
}
|
|
2749
|
+
console.error(t.fromLangsmith.summary(files.length, rows));
|
|
2750
|
+
if (notModelCalls > 0)
|
|
2751
|
+
console.error(t.fromLangsmith.notModelCalls(notModelCalls));
|
|
2752
|
+
if (unnamedModel > 0)
|
|
2753
|
+
console.error(t.fromLangsmith.unnamedModel(unnamedModel));
|
|
2754
|
+
if (noTokens > 0)
|
|
2755
|
+
console.error(t.fromLangsmith.noTokens(noTokens));
|
|
2756
|
+
if (reportedCostUsd !== null) {
|
|
2757
|
+
console.error(t.fromLangsmith.reportedCost(formatUsd(reportedCostUsd)));
|
|
2758
|
+
}
|
|
2759
|
+
if (unparseable > 0)
|
|
2760
|
+
console.error(t.fromLangsmith.unparseable(unparseable));
|
|
2761
|
+
}
|
|
2585
2762
|
/**
|
|
2586
2763
|
* `trazum from-helicone <file|dir>` — a Helicone request export as a usage log.
|
|
2587
2764
|
*
|
|
@@ -4116,7 +4293,35 @@ function commandModels(t, pricing) {
|
|
|
4116
4293
|
const col = t.models.columns;
|
|
4117
4294
|
console.log();
|
|
4118
4295
|
console.log(sectionHeading(`${t.models.title()}${t.models.unit()}`));
|
|
4119
|
-
|
|
4296
|
+
const now = new Date();
|
|
4297
|
+
console.log(c.dim(t.models.reviewedOn(pricing.lastReviewed, reviewAgeDays(pricing.lastReviewed, now))));
|
|
4298
|
+
/**
|
|
4299
|
+
* The headline date is the oldest provider's, which is the honest answer to
|
|
4300
|
+
* "how old is this table" and the wrong answer to "how old is the price I am
|
|
4301
|
+
* being quoted". A reader pricing Claude calls was told two months when
|
|
4302
|
+
* their half had been checked that morning, so the providers say for
|
|
4303
|
+
* themselves — and only when they disagree, since one date repeated seven
|
|
4304
|
+
* times is noise.
|
|
4305
|
+
*
|
|
4306
|
+
* Suppressed under an overlay: `--pricing` and `--pricing-live` replace
|
|
4307
|
+
* prices with numbers whose provenance is the overlay's own date, and these
|
|
4308
|
+
* are the bundled table's.
|
|
4309
|
+
*/
|
|
4310
|
+
const dates = [...new Set(Object.values(PROVIDER_REVIEWED))];
|
|
4311
|
+
if (dates.length > 1 && pricing.lastReviewed === PRICING_LAST_REVIEWED) {
|
|
4312
|
+
const priced = new Set(pricing.models.map((m) => m.provider));
|
|
4313
|
+
/* Grouped by date: six providers repeating one date is noise, not provenance. */
|
|
4314
|
+
const byDate = new Map();
|
|
4315
|
+
for (const [provider, date] of Object.entries(PROVIDER_REVIEWED)) {
|
|
4316
|
+
if (!priced.has(provider))
|
|
4317
|
+
continue;
|
|
4318
|
+
byDate.set(date, [...(byDate.get(date) ?? []), provider]);
|
|
4319
|
+
}
|
|
4320
|
+
const rows = [...byDate.entries()]
|
|
4321
|
+
.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
|
|
4322
|
+
.map(([date, providers]) => t.models.reviewedGroup(date, reviewAgeDays(date, now), providers.join(', ')));
|
|
4323
|
+
console.log(c.dim(t.models.reviewedByProvider(rows.join(' · '))));
|
|
4324
|
+
}
|
|
4120
4325
|
console.log();
|
|
4121
4326
|
const rows = pricing.models.map((m) => ({
|
|
4122
4327
|
id: m.id,
|
|
@@ -5851,508 +6056,208 @@ async function commandVerify(args, pricing, t) {
|
|
|
5851
6056
|
}
|
|
5852
6057
|
}
|
|
5853
6058
|
/**
|
|
5854
|
-
*
|
|
5855
|
-
*
|
|
5856
|
-
*
|
|
5857
|
-
*
|
|
5858
|
-
*
|
|
5859
|
-
*
|
|
5860
|
-
*
|
|
6059
|
+
* `--by-source`: one report per service, plus the rollup, the fleet.
|
|
6060
|
+
*
|
|
6061
|
+
* A merged bill is right for one service and wrong for twelve: it hides which
|
|
6062
|
+
* service the money comes from, per-service budgets cannot exist, and the
|
|
6063
|
+
* findings a comparison between services could make are invisible. Files are
|
|
6064
|
+
* assigned to sources by the most specific matching glob from the config’s
|
|
6065
|
+
* `sources` block; a file matching no source is named loudly, because a log
|
|
6066
|
+
* that silently joined no report is spend missing from every bill.
|
|
6067
|
+
*
|
|
6068
|
+
* Lifted out of `commandProfile` unchanged. It was one of several complete
|
|
6069
|
+
* outputs sharing a 2,359-line body with the report they are alternatives to,
|
|
6070
|
+
* and it needs a fraction of what that body had in scope: naming the fraction
|
|
6071
|
+
* is most of the point of moving it.
|
|
5861
6072
|
*/
|
|
5862
|
-
async function
|
|
5863
|
-
const path = args.positional[0];
|
|
5864
|
-
if (path === undefined)
|
|
5865
|
-
throw new Error(t.plan.noTarget());
|
|
5866
|
-
const GZ = LOG_EXTENSIONS.map((ext) => `${ext}.gz`);
|
|
5867
|
-
const READABLE = [...LOG_EXTENSIONS, ...GZ];
|
|
5868
|
-
const target = await stat(path).catch(() => null);
|
|
5869
|
-
let files = [path];
|
|
5870
|
-
if (target?.isDirectory()) {
|
|
5871
|
-
const entries = await readdir(path, { withFileTypes: true });
|
|
5872
|
-
files = entries
|
|
5873
|
-
.filter((entry) => entry.isFile() && READABLE.some((ext) => entry.name.endsWith(ext)))
|
|
5874
|
-
.map((entry) => join(path, entry.name))
|
|
5875
|
-
.sort((a, b) => a.localeCompare(b));
|
|
5876
|
-
if (files.length === 0)
|
|
5877
|
-
throw new Error(t.profile.noLogsInDirectory(path, READABLE.join(', ')));
|
|
5878
|
-
}
|
|
5879
|
-
const texts = await Promise.all(files.map((file) => readUsageLog(file, t)));
|
|
5880
|
-
const raw = texts.map((text) => (text.endsWith('\n') ? text : `${text}\n`)).join('');
|
|
5881
|
-
const report = profileUsage(raw, { catalogue: pricing });
|
|
5882
|
-
if (report.total.calls === 0)
|
|
5883
|
-
throw new Error(t.plan.nothingPriced());
|
|
5884
|
-
const levers = billLevers(report, { catalogue: pricing });
|
|
5885
|
-
const plan = buildPlan(report, levers, pricing.lastReviewed);
|
|
5886
|
-
const minUsd = typeof args.flags.get('min-usd') === 'string' ? numberFlag(args, 'min-usd', 0, t) : 0;
|
|
5887
|
-
const actions = plan.actions.filter((a) => (a.savingUsd ?? a.stakeUsd ?? 0) >= minUsd);
|
|
5888
|
-
const filtered = plan.actions.length - actions.length;
|
|
5889
|
-
const droppedUsd = plan.actions
|
|
5890
|
-
.filter((a) => (a.savingUsd ?? a.stakeUsd ?? 0) < minUsd)
|
|
5891
|
-
.reduce((sum, a) => sum + (a.savingUsd ?? a.stakeUsd ?? 0), 0);
|
|
6073
|
+
async function profileBySource(args, config, logFiles, logTexts, pricing, window, t) {
|
|
5892
6074
|
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
5893
|
-
|
|
5894
|
-
|
|
5895
|
-
|
|
5896
|
-
|
|
5897
|
-
|
|
5898
|
-
|
|
5899
|
-
|
|
5900
|
-
|
|
5901
|
-
|
|
5902
|
-
|
|
5903
|
-
|
|
5904
|
-
|
|
5905
|
-
|
|
5906
|
-
|
|
5907
|
-
const
|
|
5908
|
-
|
|
5909
|
-
|
|
6075
|
+
const pct = (share) => `${(share * 100).toFixed(1)}%`;
|
|
6076
|
+
const { onlyLabel, sinceMs, untilMs } = window;
|
|
6077
|
+
/* The same join the caller makes, from the same texts: one log, all files. */
|
|
6078
|
+
const raw = logTexts.map((text) => (text.endsWith('\n') ? text : `${text}\n`)).join('');
|
|
6079
|
+
const sourceDefs = config.sources;
|
|
6080
|
+
if (sourceDefs === undefined || Object.keys(sourceDefs).length === 0) {
|
|
6081
|
+
throw new Error(t.profile.bySourceNeedsConfig());
|
|
6082
|
+
}
|
|
6083
|
+
const { bySource, unmatched } = assignSources(logFiles, sourceDefs);
|
|
6084
|
+
if (bySource.size === 0) {
|
|
6085
|
+
throw new Error(t.profile.bySourceNothingMatched(Object.keys(sourceDefs).join(', ')));
|
|
6086
|
+
}
|
|
6087
|
+
const textByFile = new Map(logFiles.map((file, i) => [file, logTexts[i]]));
|
|
6088
|
+
const fleetSources = [];
|
|
6089
|
+
const cacheDeltas = new Map();
|
|
6090
|
+
for (const [name, files] of [...bySource.entries()].sort((a, b) => a[0].localeCompare(b[0]))) {
|
|
6091
|
+
const text = files
|
|
6092
|
+
.map((file) => textByFile.get(file))
|
|
6093
|
+
.map((chunk) => (chunk.endsWith('\n') ? chunk : `${chunk}\n`))
|
|
6094
|
+
.join('');
|
|
6095
|
+
const sourceReport = profileUsage(text, { catalogue: pricing, label: onlyLabel, sinceMs, untilMs });
|
|
6096
|
+
fleetSources.push({ name, report: sourceReport });
|
|
6097
|
+
cacheDeltas.set(name, cacheEconomics(sourceReport.total).deltaUsd);
|
|
6098
|
+
}
|
|
6099
|
+
const aggregate = profileUsage(raw, { catalogue: pricing, label: onlyLabel, sinceMs, untilMs });
|
|
6100
|
+
const rollup = fleetRollup(fleetSources, {
|
|
6101
|
+
cacheDeltas,
|
|
6102
|
+
aggregateCacheDelta: cacheEconomics(aggregate.total).deltaUsd,
|
|
6103
|
+
});
|
|
6104
|
+
if (boolFlag(args, 'json')) {
|
|
6105
|
+
console.log(JSON.stringify({
|
|
6106
|
+
schemaVersion: 1,
|
|
6107
|
+
bySource: fleetSources.map((source) => ({ name: source.name, report: source.report })),
|
|
6108
|
+
rollup: {
|
|
6109
|
+
totalUsd: rollup.totalUsd,
|
|
6110
|
+
calls: rollup.calls,
|
|
6111
|
+
sources: rollup.sources,
|
|
6112
|
+
worst: rollup.worst,
|
|
6113
|
+
mismatchedSpans: rollup.mismatchedSpans,
|
|
6114
|
+
splitBrains: rollup.splitBrains,
|
|
6115
|
+
cacheUnderwater: rollup.cacheUnderwater,
|
|
6116
|
+
unmatchedFiles: unmatched,
|
|
6117
|
+
},
|
|
6118
|
+
}, (key, value) => (value instanceof Map ? undefined : value), 2));
|
|
5910
6119
|
}
|
|
5911
|
-
|
|
5912
|
-
|
|
5913
|
-
|
|
5914
|
-
|
|
5915
|
-
|
|
5916
|
-
if (plan.span === null) {
|
|
5917
|
-
lines.push('');
|
|
5918
|
-
lines.push(`_${t.plan.noClock()}_`);
|
|
6120
|
+
else {
|
|
6121
|
+
console.log(sectionHeading(t.profile.fleetHeading(n(rollup.sources.length), formatUsd(rollup.totalUsd), t.profile.calls(rollup.calls))));
|
|
6122
|
+
for (const row of rollup.sources) {
|
|
6123
|
+
const span = row.spanDays === null ? t.profile.fleetNoClock() : t.profile.fleetSpan(row.spanDays.toFixed(1));
|
|
6124
|
+
console.log(` ${t.profile.fleetRow(row.name, formatUsd(row.usd), pct(row.share), t.profile.calls(row.calls), span)}`);
|
|
5919
6125
|
}
|
|
5920
|
-
|
|
5921
|
-
|
|
5922
|
-
|
|
5923
|
-
? t.plan.projected(formatUsd(action.savingUsd))
|
|
5924
|
-
: t.plan.staked(formatUsd(action.stakeUsd ?? 0));
|
|
5925
|
-
lines.push('');
|
|
5926
|
-
lines.push(`### ${t.plan.action(action.kind, name, action.model)} — ${money}`);
|
|
5927
|
-
if (action.detail.routeTo !== undefined)
|
|
5928
|
-
lines.push(`- ${t.plan.routeTo(action.detail.routeTo.displayName)}`);
|
|
5929
|
-
for (const assumption of action.assumes)
|
|
5930
|
-
lines.push(`- ${t.plan.assume(assumption)}`);
|
|
5931
|
-
if (action.check !== null)
|
|
5932
|
-
lines.push(`- ${t.plan.check(action.check)}`);
|
|
6126
|
+
if (rollup.worst !== null && rollup.sources.length > 1) {
|
|
6127
|
+
console.log();
|
|
6128
|
+
console.log(` ${c.yellow('!')} ${c.bold(wrap(t.profile.fleetWorst(rollup.worst.name, formatUsd(rollup.worst.usd), pct(rollup.worst.share)), 74, ' '))}`);
|
|
5933
6129
|
}
|
|
5934
|
-
if (
|
|
5935
|
-
|
|
5936
|
-
|
|
6130
|
+
if (rollup.mismatchedSpans) {
|
|
6131
|
+
console.log(` ${c.dim(wrap(t.profile.fleetMismatchedSpans(), 74, ' '))}`);
|
|
6132
|
+
}
|
|
6133
|
+
for (const split of rollup.splitBrains.slice(0, 3)) {
|
|
6134
|
+
console.log();
|
|
6135
|
+
console.log(` ${c.yellow('!')} ${wrap(t.profile.fleetSplitBrain(split.label, split.sources.map((v) => `${v.name} → ${v.model} (${formatUsd(v.usd)})`).join(', ')), 74, ' ')}`);
|
|
6136
|
+
}
|
|
6137
|
+
for (const under of rollup.cacheUnderwater.slice(0, 3)) {
|
|
6138
|
+
console.log(` ${c.yellow('!')} ${wrap(t.profile.fleetCacheUnderwater(under.name, formatUsd(under.deltaUsd)), 74, ' ')}`);
|
|
6139
|
+
}
|
|
6140
|
+
for (const file of unmatched) {
|
|
6141
|
+
console.log(` ${c.yellow('!')} ${wrap(t.profile.fleetUnmatched(file), 74, ' ')}`);
|
|
5937
6142
|
}
|
|
5938
|
-
lines.push('');
|
|
5939
|
-
lines.push(`_${t.plan.footer()}_`);
|
|
5940
|
-
return lines.join('\n');
|
|
5941
|
-
});
|
|
5942
|
-
if (boolFlag(args, 'json')) {
|
|
5943
|
-
console.log(JSON.stringify(stamped, null, 2));
|
|
5944
|
-
return;
|
|
5945
|
-
}
|
|
5946
|
-
console.log(sectionHeading(t.plan.heading(n(actions.length), formatUsd(plan.totalUsd))));
|
|
5947
|
-
console.log(` ${wrap(t.plan.totals(formatUsd(stamped.projectedSavingUsd), formatUsd(stamped.measuredStakeUsd)), 74, ' ')}`);
|
|
5948
|
-
if (plan.span === null) {
|
|
5949
|
-
console.log(` ${c.dim(wrap(t.plan.noClock(), 74, ' '))}`);
|
|
5950
|
-
}
|
|
5951
|
-
for (const action of actions) {
|
|
5952
|
-
const name = action.label === UNLABELLED ? t.profile.unlabelled() : action.label;
|
|
5953
|
-
const money = action.savingUsd !== null
|
|
5954
|
-
? t.plan.projected(formatUsd(action.savingUsd))
|
|
5955
|
-
: t.plan.staked(formatUsd(action.stakeUsd ?? 0));
|
|
5956
6143
|
console.log();
|
|
5957
|
-
console.log(` ${c.
|
|
5958
|
-
|
|
5959
|
-
|
|
6144
|
+
console.log(` ${c.dim(wrap(t.profile.fleetFooter(), 74, ' '))}`);
|
|
6145
|
+
}
|
|
6146
|
+
/**
|
|
6147
|
+
* The per-source gates. Each budget judges its own service and the run
|
|
6148
|
+
* fails naming the service — a total that hides which source crossed its
|
|
6149
|
+
* line is the rendering this mode exists to end. Waivable per source
|
|
6150
|
+
* through `bySource:<name>`, under the same expiry discipline.
|
|
6151
|
+
*/
|
|
6152
|
+
const bySourceBudgets = config.spend?.bySource ?? {};
|
|
6153
|
+
for (const [name, limit] of Object.entries(bySourceBudgets)) {
|
|
6154
|
+
const found = fleetSources.find((source) => source.name === name);
|
|
6155
|
+
if (found === undefined) {
|
|
6156
|
+
console.error(c.dim(t.profile.fleetBudgetMissing(name)));
|
|
6157
|
+
continue;
|
|
5960
6158
|
}
|
|
5961
|
-
|
|
5962
|
-
|
|
6159
|
+
const usd = found.report.total.totalUsd;
|
|
6160
|
+
if (usd > limit) {
|
|
6161
|
+
console.error(c.red(t.profile.fleetBudgetFailed(name, formatUsd(usd), formatUsd(limit))));
|
|
6162
|
+
process.exitCode = 1;
|
|
5963
6163
|
}
|
|
5964
|
-
|
|
5965
|
-
console.
|
|
6164
|
+
else {
|
|
6165
|
+
console.error(c.dim(t.profile.fleetBudgetOk(name, formatUsd(usd), formatUsd(limit))));
|
|
5966
6166
|
}
|
|
5967
6167
|
}
|
|
5968
|
-
|
|
5969
|
-
|
|
5970
|
-
|
|
6168
|
+
}
|
|
6169
|
+
/**
|
|
6170
|
+
* `--dry-run`: what this log could and could not answer, and no bill.
|
|
6171
|
+
*
|
|
6172
|
+
* The question somebody has *before* wiring Trazum into CI is not "what did we
|
|
6173
|
+
* spend" but "will this log support the gates I want", and answering it with a
|
|
6174
|
+
* full report makes them read a bill to find out a field is missing. This path
|
|
6175
|
+
* states readiness per capability and produces no dollar figure at all, so
|
|
6176
|
+
* nothing here can be mistaken for spend. It also refuses to coexist with the
|
|
6177
|
+
* gates: a gate over a report that was never produced would exit green having
|
|
6178
|
+
* judged nothing.
|
|
6179
|
+
*/
|
|
6180
|
+
function profileDryRun(args, report, t) {
|
|
6181
|
+
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
6182
|
+
const pct = (share) => `${(share * 100).toFixed(1)}%`;
|
|
6183
|
+
const gateFlags = ['max-usd', 'max-growth-usd', 'max-cache-loss-usd', 'max-day-usd', 'max-session-usd'];
|
|
6184
|
+
if (gateFlags.some((flag) => typeof args.flags.get(flag) === 'string')) {
|
|
6185
|
+
throw new Error(t.profile.dryRunNoGates());
|
|
6186
|
+
}
|
|
6187
|
+
const cov = report.fieldCoverage;
|
|
6188
|
+
const parsed = cov.parsed;
|
|
6189
|
+
console.log(sectionHeading(t.profile.dryRunHeading()));
|
|
6190
|
+
console.log(` ${wrap(t.profile.dryRunParsed(n(parsed), n(report.skippedLines.length)), 74, ' ')}`);
|
|
6191
|
+
if (report.unpricedModels.length > 0) {
|
|
6192
|
+
console.log(` ${c.yellow('!')} ${wrap(t.profile.dryRunUnpriced(report.unpricedModels.join(', ')), 74, ' ')}`);
|
|
5971
6193
|
}
|
|
5972
6194
|
console.log();
|
|
5973
|
-
|
|
5974
|
-
|
|
5975
|
-
|
|
5976
|
-
|
|
5977
|
-
|
|
5978
|
-
|
|
5979
|
-
|
|
5980
|
-
|
|
5981
|
-
|
|
5982
|
-
|
|
5983
|
-
|
|
6195
|
+
const can = (ok, line) => {
|
|
6196
|
+
console.log(` ${ok ? c.green('✓') : c.yellow('✗')} ${wrap(line, 74, ' ')}`);
|
|
6197
|
+
};
|
|
6198
|
+
const share = (count) => (parsed > 0 ? pct(count / parsed) : '0%');
|
|
6199
|
+
can(parsed > 0, t.profile.dryRunTotals());
|
|
6200
|
+
can(cov.label > 0, t.profile.dryRunLabels(share(cov.label)));
|
|
6201
|
+
can(cov.ts > 0, t.profile.dryRunClock(share(cov.ts)));
|
|
6202
|
+
can(cov.session > 0, t.profile.dryRunSessions(share(cov.session)));
|
|
6203
|
+
can(cov.outcome > 0, t.profile.dryRunOutcomes(share(cov.outcome)));
|
|
6204
|
+
can(cov.stopReason > 0, t.profile.dryRunStopReason(share(cov.stopReason)));
|
|
6205
|
+
// "No cache traffic" is not a missing field: the split can only exist on
|
|
6206
|
+
// records that wrote, and a log that never wrote has nothing to record.
|
|
6207
|
+
if (cov.cacheWrites > 0) {
|
|
6208
|
+
can(cov.cacheTtl > 0, t.profile.dryRunCacheTtl(n(cov.cacheTtl), n(cov.cacheWrites)));
|
|
6209
|
+
}
|
|
6210
|
+
else {
|
|
6211
|
+
console.log(` ${c.dim('·')} ${wrap(t.profile.dryRunNoCacheTraffic(), 74, ' ')}`);
|
|
5984
6212
|
}
|
|
6213
|
+
console.log();
|
|
6214
|
+
console.log(` ${c.dim(wrap(t.profile.dryRunFooter(), 74, ' '))}`);
|
|
6215
|
+
if (parsed === 0)
|
|
6216
|
+
process.exitCode = 1;
|
|
6217
|
+
}
|
|
6218
|
+
/**
|
|
6219
|
+
* The money gates: run them, and say what they said.
|
|
6220
|
+
*
|
|
6221
|
+
* `check` gates tokens before the money is spent; these gate the spend itself,
|
|
6222
|
+
* from the provider’s own billed counts. No period is assumed — the budget
|
|
6223
|
+
* applies to exactly the log handed in, so a nightly job that profiles
|
|
6224
|
+
* yesterday’s log has a daily budget without Trazum ever guessing what a day is.
|
|
6225
|
+
*
|
|
6226
|
+
* ## Why this is one function and not five
|
|
6227
|
+
*
|
|
6228
|
+
* `waiverFor`, `waived`, the uses it collects, the verdict capture and the
|
|
6229
|
+
* flush to the waiver log were five closures inside `commandProfile`, sharing
|
|
6230
|
+
* its 2,359-line scope with the report they have nothing to do with. They are
|
|
6231
|
+
* one concern — policy — and the only two things the rest of the command needs
|
|
6232
|
+
* from them are whether a gate failed and what the gates said.
|
|
6233
|
+
*
|
|
6234
|
+
* ## Why it returns rather than assigns
|
|
6235
|
+
*
|
|
6236
|
+
* `gateFailed` and `gateVerdicts` were `let` and `const` in the enclosing
|
|
6237
|
+
* scope, written here and read three hundred lines away by the side-file
|
|
6238
|
+
* writer. That is an output channel nothing declares. They are the return
|
|
6239
|
+
* value now, and `failed` is still read off `process.exitCode` exactly as it
|
|
6240
|
+
* was — the gates set the exit code, and this reports what they set.
|
|
6241
|
+
*/
|
|
6242
|
+
async function runProfileGates(context) {
|
|
6243
|
+
const { args, config, configDir, report, previous, againstDelta, windowed, now, pricing, t } = context;
|
|
6244
|
+
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
6245
|
+
const pct = (share) => `${(share * 100).toFixed(1)}%`;
|
|
6246
|
+
/*
|
|
6247
|
+
What a failing gate points at. The enclosing command computes this too, a
|
|
6248
|
+
few hundred lines further down, and the gate block used to reach forward
|
|
6249
|
+
into that declaration — legal only because the closure that read it was
|
|
6250
|
+
called later. `billLevers` is pure, so the honest fix is to ask for it here.
|
|
6251
|
+
*/
|
|
6252
|
+
const levers = billLevers(report, { catalogue: pricing });
|
|
5985
6253
|
/**
|
|
5986
|
-
*
|
|
5987
|
-
*
|
|
5988
|
-
* Usage logs rotate: `logs/2026-08-01.jsonl`, `logs/2026-08-02.jsonl`, one
|
|
5989
|
-
* per day for a month. Making somebody `cat` them together before a profile
|
|
5990
|
-
* will read them is a setup cost that gets a tool skipped, and doing it for
|
|
5991
|
-
* them is a directory listing.
|
|
6254
|
+
* The money gates, armed by flags and applied on every output path.
|
|
5992
6255
|
*
|
|
5993
|
-
*
|
|
5994
|
-
*
|
|
5995
|
-
*
|
|
5996
|
-
*
|
|
5997
|
-
*
|
|
5998
|
-
*/
|
|
5999
|
-
/**
|
|
6000
|
-
* The same names, gzipped — which is what a rotated log actually looks like
|
|
6001
|
-
* a day after it rotates.
|
|
6002
|
-
*
|
|
6003
|
-
* `logrotate`, Docker's json-file driver and every cloud log export compress
|
|
6004
|
-
* yesterday's file, so a directory of a month's logs is one plain file and
|
|
6005
|
-
* twenty-nine `.gz` ones. Reading only the plain one and saying nothing
|
|
6006
|
-
* would report a month's bill from a day of it, in the flattering
|
|
6007
|
-
* direction, which is exactly the failure directory mode was added to
|
|
6008
|
-
* prevent.
|
|
6009
|
-
*/
|
|
6010
|
-
const GZ_EXTENSIONS = LOG_EXTENSIONS.map((ext) => `${ext}.gz`);
|
|
6011
|
-
const READABLE = [...LOG_EXTENSIONS, ...GZ_EXTENSIONS];
|
|
6012
|
-
const target = await stat(path).catch(() => null);
|
|
6013
|
-
let logFiles = [path];
|
|
6014
|
-
if (target?.isDirectory()) {
|
|
6015
|
-
/**
|
|
6016
|
-
* Recursive under `--by-source`, flat otherwise. The fleet's whole point
|
|
6017
|
-
* is one directory per service, so the walk must descend; the flat mode
|
|
6018
|
-
* keeps its long-standing behaviour because a directory of rotated logs
|
|
6019
|
-
* with an unrelated subfolder should not quietly absorb it.
|
|
6020
|
-
*/
|
|
6021
|
-
const bySourceMode = boolFlag(args, 'by-source');
|
|
6022
|
-
const entries = await readdir(path, { withFileTypes: true, recursive: bySourceMode });
|
|
6023
|
-
logFiles = entries
|
|
6024
|
-
.filter((entry) => entry.isFile() && READABLE.some((ext) => entry.name.endsWith(ext)))
|
|
6025
|
-
.map((entry) => join(entry.parentPath ?? path, entry.name))
|
|
6026
|
-
.sort((a, b) => a.localeCompare(b));
|
|
6027
|
-
if (logFiles.length === 0) {
|
|
6028
|
-
throw new Error(t.profile.noLogsInDirectory(path, READABLE.join(', ')));
|
|
6029
|
-
}
|
|
6030
|
-
}
|
|
6031
|
-
/**
|
|
6032
|
-
* Gzipped files are decompressed in memory; everything else is read as text.
|
|
6033
|
-
*
|
|
6034
|
-
* Decided by **extension**, not by sniffing the first two bytes: a file
|
|
6035
|
-
* named `.jsonl` whose contents happen to start with 0x1f8b is far more
|
|
6036
|
-
* likely to be a corrupt log than a mislabelled archive, and silently
|
|
6037
|
-
* treating it as one would turn a diagnosable error into an empty report.
|
|
6038
|
-
*
|
|
6039
|
-
* A `.gz` that will not decompress is an error naming the file. The
|
|
6040
|
-
* alternative — skipping it — is a total quietly missing a day, which is
|
|
6041
|
-
* the failure this repository refuses in every other place it can occur.
|
|
6042
|
-
*/
|
|
6043
|
-
const logTexts = await Promise.all(logFiles.map((file) => readUsageLog(file, t)));
|
|
6044
|
-
// A file that does not end in a newline would otherwise glue its last record
|
|
6045
|
-
// to the next file's first one, and both would be reported as unreadable.
|
|
6046
|
-
const raw = logTexts.map((text) => (text.endsWith('\n') ? text : `${text}\n`)).join('');
|
|
6047
|
-
/**
|
|
6048
|
-
* The drill-down. A label that matches nothing is an error naming the labels
|
|
6049
|
-
* that exist — the route command's rule, for the route command's reason: a
|
|
6050
|
-
* report over zero calls silently filtered would read as "this workload is
|
|
6051
|
-
* free".
|
|
6052
|
-
*/
|
|
6053
|
-
const onlyLabel = stringFlag(args, 'label');
|
|
6054
|
-
/**
|
|
6055
|
-
* The drill-down in time. `--since`/`--until` take a UTC day or a full
|
|
6056
|
-
* timestamp; a bare day means the whole of it — since its first instant,
|
|
6057
|
-
* until its last — because "--until 2026-08-14" excluding the named day is
|
|
6058
|
-
* a trap sprung on everyone who reads dates the way humans do. Internally
|
|
6059
|
-
* the window is half-open `[since, until)`, so two adjacent windows share
|
|
6060
|
-
* no record.
|
|
6061
|
-
*/
|
|
6062
|
-
const now = Date.now();
|
|
6063
|
-
const sinceWhen = parseWhen(args, 'since', false, t, now);
|
|
6064
|
-
const untilWhen = parseWhen(args, 'until', true, t, now);
|
|
6065
|
-
const relativeWindow = sinceWhen.relative || untilWhen.relative;
|
|
6066
|
-
const sinceMs = sinceWhen.ms;
|
|
6067
|
-
const untilMs = untilWhen.ms;
|
|
6068
|
-
if (sinceMs !== undefined && untilMs !== undefined && sinceMs >= untilMs) {
|
|
6069
|
-
throw new Error(t.profile.sinceAfterUntil());
|
|
6070
|
-
}
|
|
6071
|
-
const windowed = sinceMs !== undefined || untilMs !== undefined;
|
|
6072
|
-
/**
|
|
6073
|
-
* How old the price table behind every dollar below is. Stated only when it
|
|
6074
|
-
* is old enough to matter: `models` and `doctor` always print the date, but
|
|
6075
|
-
* a profile is read for its figures, and the one fact that silently
|
|
6076
|
-
* invalidates all of them is a table the provider has re-priced since.
|
|
6077
|
-
* The threshold is in the sentence, not hidden here.
|
|
6078
|
-
*/
|
|
6079
|
-
const STALE_PRICING_DAYS = 45;
|
|
6080
|
-
const pricingAgeDays = reviewAgeDays(pricing.lastReviewed, new Date());
|
|
6081
|
-
const pricingStale = pricingAgeDays !== null && pricingAgeDays > STALE_PRICING_DAYS
|
|
6082
|
-
? { date: pricing.lastReviewed, days: pricingAgeDays }
|
|
6083
|
-
: null;
|
|
6084
|
-
const report = profileUsage(raw, { catalogue: pricing, label: onlyLabel, sinceMs, untilMs });
|
|
6085
|
-
if (report.total.calls === 0 && report.unpriced.calls === 0) {
|
|
6086
|
-
if (onlyLabel !== undefined || windowed) {
|
|
6087
|
-
// Diagnose against the log without the failed filter, so the error can
|
|
6088
|
-
// name what does exist instead of describing an absence.
|
|
6089
|
-
const unfiltered = profileUsage(raw, { catalogue: pricing });
|
|
6090
|
-
if (unfiltered.total.calls > 0 || unfiltered.unpriced.calls > 0) {
|
|
6091
|
-
if (onlyLabel !== undefined && !unfiltered.byLabel.some((r) => r.label === onlyLabel)) {
|
|
6092
|
-
const available = unfiltered.byLabel
|
|
6093
|
-
.map((r) => (r.label === UNLABELLED ? t.profile.unlabelled() : r.label))
|
|
6094
|
-
.join(', ');
|
|
6095
|
-
throw new Error(t.route.labelNotFound(onlyLabel, available || '—'));
|
|
6096
|
-
}
|
|
6097
|
-
if (windowed) {
|
|
6098
|
-
/**
|
|
6099
|
-
* A window that matches nothing must not become a $0 report — under
|
|
6100
|
-
* `--max-usd` it would pass a budget gate over a period the log
|
|
6101
|
-
* simply does not cover, which is the flattering non-answer. The
|
|
6102
|
-
* error names what the log *does* cover, or says it has no clock at
|
|
6103
|
-
* all, so the fix is visible in the message.
|
|
6104
|
-
*/
|
|
6105
|
-
if (unfiltered.span === null)
|
|
6106
|
-
throw new Error(t.profile.windowNeedsClock());
|
|
6107
|
-
throw new Error(`${t.profile.windowMatchesNothing(dayOf(unfiltered.span.fromMs), dayOf(unfiltered.span.toMs))}${relativeWindow ? ` ${t.profile.windowRelativeEmpty()}` : ''}`);
|
|
6108
|
-
}
|
|
6109
|
-
}
|
|
6110
|
-
}
|
|
6111
|
-
}
|
|
6112
|
-
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
6113
|
-
const pct = (share) => `${(share * 100).toFixed(1)}%`;
|
|
6114
|
-
/**
|
|
6115
|
-
* `--by-source`: one report per service, plus the rollup — the fleet.
|
|
6116
|
-
*
|
|
6117
|
-
* A merged bill is right for one service and wrong for twelve: it hides
|
|
6118
|
-
* which service the money comes from, per-service budgets cannot exist,
|
|
6119
|
-
* and the findings a comparison between services could make are invisible.
|
|
6120
|
-
* Files are assigned to sources by the most specific matching glob from the
|
|
6121
|
-
* config's `sources` block; a file matching no source is named loudly,
|
|
6122
|
-
* because a log that silently joined no report is spend missing from every
|
|
6123
|
-
* bill.
|
|
6124
|
-
*/
|
|
6125
|
-
if (boolFlag(args, 'by-source')) {
|
|
6126
|
-
const sourceDefs = config.sources;
|
|
6127
|
-
if (sourceDefs === undefined || Object.keys(sourceDefs).length === 0) {
|
|
6128
|
-
throw new Error(t.profile.bySourceNeedsConfig());
|
|
6129
|
-
}
|
|
6130
|
-
const { bySource, unmatched } = assignSources(logFiles, sourceDefs);
|
|
6131
|
-
if (bySource.size === 0) {
|
|
6132
|
-
throw new Error(t.profile.bySourceNothingMatched(Object.keys(sourceDefs).join(', ')));
|
|
6133
|
-
}
|
|
6134
|
-
const textByFile = new Map(logFiles.map((file, i) => [file, logTexts[i]]));
|
|
6135
|
-
const fleetSources = [];
|
|
6136
|
-
const cacheDeltas = new Map();
|
|
6137
|
-
for (const [name, files] of [...bySource.entries()].sort((a, b) => a[0].localeCompare(b[0]))) {
|
|
6138
|
-
const text = files
|
|
6139
|
-
.map((file) => textByFile.get(file))
|
|
6140
|
-
.map((chunk) => (chunk.endsWith('\n') ? chunk : `${chunk}\n`))
|
|
6141
|
-
.join('');
|
|
6142
|
-
const sourceReport = profileUsage(text, { catalogue: pricing, label: onlyLabel, sinceMs, untilMs });
|
|
6143
|
-
fleetSources.push({ name, report: sourceReport });
|
|
6144
|
-
cacheDeltas.set(name, cacheEconomics(sourceReport.total).deltaUsd);
|
|
6145
|
-
}
|
|
6146
|
-
const aggregate = profileUsage(raw, { catalogue: pricing, label: onlyLabel, sinceMs, untilMs });
|
|
6147
|
-
const rollup = fleetRollup(fleetSources, {
|
|
6148
|
-
cacheDeltas,
|
|
6149
|
-
aggregateCacheDelta: cacheEconomics(aggregate.total).deltaUsd,
|
|
6150
|
-
});
|
|
6151
|
-
if (boolFlag(args, 'json')) {
|
|
6152
|
-
console.log(JSON.stringify({
|
|
6153
|
-
schemaVersion: 1,
|
|
6154
|
-
bySource: fleetSources.map((source) => ({ name: source.name, report: source.report })),
|
|
6155
|
-
rollup: {
|
|
6156
|
-
totalUsd: rollup.totalUsd,
|
|
6157
|
-
calls: rollup.calls,
|
|
6158
|
-
sources: rollup.sources,
|
|
6159
|
-
worst: rollup.worst,
|
|
6160
|
-
mismatchedSpans: rollup.mismatchedSpans,
|
|
6161
|
-
splitBrains: rollup.splitBrains,
|
|
6162
|
-
cacheUnderwater: rollup.cacheUnderwater,
|
|
6163
|
-
unmatchedFiles: unmatched,
|
|
6164
|
-
},
|
|
6165
|
-
}, (key, value) => (value instanceof Map ? undefined : value), 2));
|
|
6166
|
-
}
|
|
6167
|
-
else {
|
|
6168
|
-
console.log(sectionHeading(t.profile.fleetHeading(n(rollup.sources.length), formatUsd(rollup.totalUsd), t.profile.calls(rollup.calls))));
|
|
6169
|
-
for (const row of rollup.sources) {
|
|
6170
|
-
const span = row.spanDays === null ? t.profile.fleetNoClock() : t.profile.fleetSpan(row.spanDays.toFixed(1));
|
|
6171
|
-
console.log(` ${t.profile.fleetRow(row.name, formatUsd(row.usd), pct(row.share), t.profile.calls(row.calls), span)}`);
|
|
6172
|
-
}
|
|
6173
|
-
if (rollup.worst !== null && rollup.sources.length > 1) {
|
|
6174
|
-
console.log();
|
|
6175
|
-
console.log(` ${c.yellow('!')} ${c.bold(wrap(t.profile.fleetWorst(rollup.worst.name, formatUsd(rollup.worst.usd), pct(rollup.worst.share)), 74, ' '))}`);
|
|
6176
|
-
}
|
|
6177
|
-
if (rollup.mismatchedSpans) {
|
|
6178
|
-
console.log(` ${c.dim(wrap(t.profile.fleetMismatchedSpans(), 74, ' '))}`);
|
|
6179
|
-
}
|
|
6180
|
-
for (const split of rollup.splitBrains.slice(0, 3)) {
|
|
6181
|
-
console.log();
|
|
6182
|
-
console.log(` ${c.yellow('!')} ${wrap(t.profile.fleetSplitBrain(split.label, split.sources.map((v) => `${v.name} → ${v.model} (${formatUsd(v.usd)})`).join(', ')), 74, ' ')}`);
|
|
6183
|
-
}
|
|
6184
|
-
for (const under of rollup.cacheUnderwater.slice(0, 3)) {
|
|
6185
|
-
console.log(` ${c.yellow('!')} ${wrap(t.profile.fleetCacheUnderwater(under.name, formatUsd(under.deltaUsd)), 74, ' ')}`);
|
|
6186
|
-
}
|
|
6187
|
-
for (const file of unmatched) {
|
|
6188
|
-
console.log(` ${c.yellow('!')} ${wrap(t.profile.fleetUnmatched(file), 74, ' ')}`);
|
|
6189
|
-
}
|
|
6190
|
-
console.log();
|
|
6191
|
-
console.log(` ${c.dim(wrap(t.profile.fleetFooter(), 74, ' '))}`);
|
|
6192
|
-
}
|
|
6193
|
-
/**
|
|
6194
|
-
* The per-source gates. Each budget judges its own service and the run
|
|
6195
|
-
* fails naming the service — a total that hides which source crossed its
|
|
6196
|
-
* line is the rendering this mode exists to end. Waivable per source
|
|
6197
|
-
* through `bySource:<name>`, under the same expiry discipline.
|
|
6198
|
-
*/
|
|
6199
|
-
const bySourceBudgets = config.spend?.bySource ?? {};
|
|
6200
|
-
for (const [name, limit] of Object.entries(bySourceBudgets)) {
|
|
6201
|
-
const found = fleetSources.find((source) => source.name === name);
|
|
6202
|
-
if (found === undefined) {
|
|
6203
|
-
console.error(c.dim(t.profile.fleetBudgetMissing(name)));
|
|
6204
|
-
continue;
|
|
6205
|
-
}
|
|
6206
|
-
const usd = found.report.total.totalUsd;
|
|
6207
|
-
if (usd > limit) {
|
|
6208
|
-
console.error(c.red(t.profile.fleetBudgetFailed(name, formatUsd(usd), formatUsd(limit))));
|
|
6209
|
-
process.exitCode = 1;
|
|
6210
|
-
}
|
|
6211
|
-
else {
|
|
6212
|
-
console.error(c.dim(t.profile.fleetBudgetOk(name, formatUsd(usd), formatUsd(limit))));
|
|
6213
|
-
}
|
|
6214
|
-
}
|
|
6215
|
-
return;
|
|
6216
|
-
}
|
|
6217
|
-
/**
|
|
6218
|
-
* `--dry-run`: what this log could and could not answer, and no bill.
|
|
6219
|
-
*
|
|
6220
|
-
* The question somebody has *before* wiring Trazum into CI is not "what did
|
|
6221
|
-
* we spend" but "will this log support the gates I want" — and answering it
|
|
6222
|
-
* with a full report makes them read a bill to find out a field is missing.
|
|
6223
|
-
* This path states readiness per capability and produces no dollar figure
|
|
6224
|
-
* at all, so nothing here can be mistaken for spend. It also refuses to
|
|
6225
|
-
* coexist with the gates: a gate over a report that was never produced
|
|
6226
|
-
* would exit green having judged nothing.
|
|
6227
|
-
*/
|
|
6228
|
-
if (boolFlag(args, 'dry-run')) {
|
|
6229
|
-
const gateFlags = ['max-usd', 'max-growth-usd', 'max-cache-loss-usd', 'max-day-usd', 'max-session-usd'];
|
|
6230
|
-
if (gateFlags.some((flag) => typeof args.flags.get(flag) === 'string')) {
|
|
6231
|
-
throw new Error(t.profile.dryRunNoGates());
|
|
6232
|
-
}
|
|
6233
|
-
const cov = report.fieldCoverage;
|
|
6234
|
-
const parsed = cov.parsed;
|
|
6235
|
-
console.log(sectionHeading(t.profile.dryRunHeading()));
|
|
6236
|
-
console.log(` ${wrap(t.profile.dryRunParsed(n(parsed), n(report.skippedLines.length)), 74, ' ')}`);
|
|
6237
|
-
if (report.unpricedModels.length > 0) {
|
|
6238
|
-
console.log(` ${c.yellow('!')} ${wrap(t.profile.dryRunUnpriced(report.unpricedModels.join(', ')), 74, ' ')}`);
|
|
6239
|
-
}
|
|
6240
|
-
console.log();
|
|
6241
|
-
const can = (ok, line) => {
|
|
6242
|
-
console.log(` ${ok ? c.green('✓') : c.yellow('✗')} ${wrap(line, 74, ' ')}`);
|
|
6243
|
-
};
|
|
6244
|
-
const share = (count) => (parsed > 0 ? pct(count / parsed) : '0%');
|
|
6245
|
-
can(parsed > 0, t.profile.dryRunTotals());
|
|
6246
|
-
can(cov.label > 0, t.profile.dryRunLabels(share(cov.label)));
|
|
6247
|
-
can(cov.ts > 0, t.profile.dryRunClock(share(cov.ts)));
|
|
6248
|
-
can(cov.session > 0, t.profile.dryRunSessions(share(cov.session)));
|
|
6249
|
-
can(cov.outcome > 0, t.profile.dryRunOutcomes(share(cov.outcome)));
|
|
6250
|
-
can(cov.stopReason > 0, t.profile.dryRunStopReason(share(cov.stopReason)));
|
|
6251
|
-
// "No cache traffic" is not a missing field: the split can only exist on
|
|
6252
|
-
// records that wrote, and a log that never wrote has nothing to record.
|
|
6253
|
-
if (cov.cacheWrites > 0) {
|
|
6254
|
-
can(cov.cacheTtl > 0, t.profile.dryRunCacheTtl(n(cov.cacheTtl), n(cov.cacheWrites)));
|
|
6255
|
-
}
|
|
6256
|
-
else {
|
|
6257
|
-
console.log(` ${c.dim('·')} ${wrap(t.profile.dryRunNoCacheTraffic(), 74, ' ')}`);
|
|
6258
|
-
}
|
|
6259
|
-
console.log();
|
|
6260
|
-
console.log(` ${c.dim(wrap(t.profile.dryRunFooter(), 74, ' '))}`);
|
|
6261
|
-
if (parsed === 0)
|
|
6262
|
-
process.exitCode = 1;
|
|
6263
|
-
return;
|
|
6264
|
-
}
|
|
6265
|
-
/**
|
|
6266
|
-
* The previous log, loaded before the output paths split so the growth gate
|
|
6267
|
-
* exists under `--json` too — a CI step reads the JSON and trusts the exit
|
|
6268
|
-
* code, and a gate that only arms in the human rendering is a gate CI never
|
|
6269
|
-
* had.
|
|
6270
|
-
*/
|
|
6271
|
-
const againstPath = stringFlag(args, 'against');
|
|
6272
|
-
// The same filter on both sides: comparing one workload's bill against the
|
|
6273
|
-
// whole previous log would report every sibling workload as vanished savings.
|
|
6274
|
-
const previous = againstPath !== undefined
|
|
6275
|
-
// The same reader as the log itself, so `--against last-month.jsonl.gz`
|
|
6276
|
-
// works: a comparison that could only read one of the two formats would
|
|
6277
|
-
// be a flag that fails on exactly the rotated file it exists to read.
|
|
6278
|
-
? profileUsage(await readUsageLog(againstPath, t), {
|
|
6279
|
-
catalogue: pricing,
|
|
6280
|
-
label: onlyLabel,
|
|
6281
|
-
// The same window on both sides, for the same reason as the label:
|
|
6282
|
-
// a windowed bill against an unwindowed one compares a slice to a
|
|
6283
|
-
// whole and calls the difference growth.
|
|
6284
|
-
sinceMs,
|
|
6285
|
-
untilMs,
|
|
6286
|
-
})
|
|
6287
|
-
: null;
|
|
6288
|
-
const againstDelta = previous !== null && previous.total.calls > 0
|
|
6289
|
-
? report.total.totalUsd - previous.total.totalUsd
|
|
6290
|
-
: null;
|
|
6291
|
-
/**
|
|
6292
|
-
* The same tokens at another model's rates, computed before the output paths
|
|
6293
|
-
* split so `--json` carries it too.
|
|
6294
|
-
*
|
|
6295
|
-
* An unknown id **throws** rather than printing nothing. A flag that silently
|
|
6296
|
-
* does nothing is worse than a missing feature: the reader typed a question,
|
|
6297
|
-
* got a report with no answer in it, and has no way to tell a typo from a
|
|
6298
|
-
* model this comparison had nothing to say about.
|
|
6299
|
-
*/
|
|
6300
|
-
/**
|
|
6301
|
-
* The largest call against each model's window, computed once here so the
|
|
6302
|
-
* terminal, the JSON and the markdown state the same ratio — the
|
|
6303
|
-
* denominator lives in the (possibly overlaid) catalogue.
|
|
6304
|
-
*/
|
|
6305
|
-
const pressures = contextPressure(report, pricing);
|
|
6306
|
-
const whatIfModel = stringFlag(args, 'what-if');
|
|
6307
|
-
const whatIf = whatIfModel !== undefined ? repriceProfile(report, whatIfModel, pricing) : null;
|
|
6308
|
-
if (whatIfModel !== undefined && whatIf === null) {
|
|
6309
|
-
throw new Error(t.profile.whatIfUnknown(whatIfModel, pricing.models.map((m) => m.id).join(', ')));
|
|
6310
|
-
}
|
|
6311
|
-
/**
|
|
6312
|
-
* The drivers of the change, per label and per model, computed once here so
|
|
6313
|
-
* the terminal, the JSON and any future rendering describe the same change.
|
|
6314
|
-
* The model half answers the question the label half cannot: "the growth is
|
|
6315
|
-
* traffic moving from Haiku to Opus" is a fact about the mix, invisible in
|
|
6316
|
-
* per-workload rows whose names did not change.
|
|
6317
|
-
*/
|
|
6318
|
-
const labelDrivers = previous !== null && previous.total.calls > 0
|
|
6319
|
-
? driversBetween(previous.byLabel.map((r) => ({ key: r.label, usd: r.breakdown.totalUsd })), report.byLabel.map((r) => ({ key: r.label, usd: r.breakdown.totalUsd })))
|
|
6320
|
-
: [];
|
|
6321
|
-
const modelDrivers = previous !== null && previous.total.calls > 0
|
|
6322
|
-
? driversBetween(previous.byModel.map((r) => ({ key: r.model, usd: r.breakdown.totalUsd })), report.byModel.map((r) => ({ key: r.model, usd: r.breakdown.totalUsd })))
|
|
6323
|
-
: [];
|
|
6324
|
-
/**
|
|
6325
|
-
* Whether the two logs share any time at all. This comparison is meant for
|
|
6326
|
-
* disjoint periods or snapshots of different systems; when both spans are
|
|
6327
|
-
* known and intersect, the same calls may sit on both sides of the
|
|
6328
|
-
* subtraction and the "growth" is partly the same money counted twice.
|
|
6329
|
-
* Only decidable when both logs carry a clock — three states, as always:
|
|
6330
|
-
* warned, clear, or unknown, and unknown stays silent rather than clear.
|
|
6331
|
-
*/
|
|
6332
|
-
const againstOverlap = previous !== null &&
|
|
6333
|
-
previous.total.calls > 0 &&
|
|
6334
|
-
previous.span !== null &&
|
|
6335
|
-
report.span !== null &&
|
|
6336
|
-
Math.min(report.span.toMs, previous.span.toMs) >=
|
|
6337
|
-
Math.max(report.span.fromMs, previous.span.fromMs)
|
|
6338
|
-
? {
|
|
6339
|
-
fromMs: Math.max(report.span.fromMs, previous.span.fromMs),
|
|
6340
|
-
toMs: Math.min(report.span.toMs, previous.span.toMs),
|
|
6341
|
-
}
|
|
6342
|
-
: null;
|
|
6343
|
-
// A gate flag that silently does nothing is not an answer — same rule as
|
|
6344
|
-
// --apply-suggestions without --suggest.
|
|
6345
|
-
if (typeof args.flags.get('max-growth-usd') === 'string' && againstPath === undefined) {
|
|
6346
|
-
throw new Error(t.profile.maxGrowthNeedsAgainst());
|
|
6347
|
-
}
|
|
6348
|
-
/**
|
|
6349
|
-
* The money gates, armed by flags and applied on every output path.
|
|
6350
|
-
*
|
|
6351
|
-
* `check` gates tokens before the money is spent; these gate the spend
|
|
6352
|
-
* itself, from the provider's own billed counts. No period is assumed —
|
|
6353
|
-
* the budget applies to exactly the log handed in, so a nightly job that
|
|
6354
|
-
* profiles yesterday's log has a daily budget without Trazum ever
|
|
6355
|
-
* guessing what a day is.
|
|
6256
|
+
* `check` gates tokens before the money is spent; these gate the spend
|
|
6257
|
+
* itself, from the provider's own billed counts. No period is assumed —
|
|
6258
|
+
* the budget applies to exactly the log handed in, so a nightly job that
|
|
6259
|
+
* profiles yesterday's log has a daily budget without Trazum ever
|
|
6260
|
+
* guessing what a day is.
|
|
6356
6261
|
*/
|
|
6357
6262
|
/**
|
|
6358
6263
|
* The gate verdicts, kept so the markdown summary can carry them.
|
|
@@ -6741,461 +6646,908 @@ async function commandProfile(args, config, configDir, pricing, t) {
|
|
|
6741
6646
|
}
|
|
6742
6647
|
}
|
|
6743
6648
|
};
|
|
6744
|
-
|
|
6745
|
-
|
|
6746
|
-
|
|
6747
|
-
|
|
6748
|
-
|
|
6749
|
-
|
|
6750
|
-
|
|
6751
|
-
|
|
6752
|
-
|
|
6753
|
-
|
|
6754
|
-
|
|
6755
|
-
|
|
6756
|
-
|
|
6757
|
-
|
|
6758
|
-
|
|
6759
|
-
|
|
6760
|
-
|
|
6761
|
-
|
|
6762
|
-
|
|
6763
|
-
|
|
6764
|
-
|
|
6765
|
-
|
|
6766
|
-
|
|
6767
|
-
|
|
6768
|
-
|
|
6769
|
-
|
|
6770
|
-
|
|
6771
|
-
|
|
6772
|
-
|
|
6773
|
-
|
|
6774
|
-
|
|
6775
|
-
|
|
6776
|
-
|
|
6777
|
-
|
|
6778
|
-
|
|
6779
|
-
|
|
6780
|
-
/**
|
|
6781
|
-
* One input object for both doors, built once: the HTML file and the
|
|
6782
|
-
* Markdown file are two projections of the same figures, and building
|
|
6783
|
-
* the input twice is how two renderings of one bill start disagreeing.
|
|
6784
|
-
*/
|
|
6785
|
-
const renderInput = ({
|
|
6786
|
-
report,
|
|
6787
|
-
levers,
|
|
6788
|
-
cache,
|
|
6789
|
-
t,
|
|
6790
|
-
...(windowed
|
|
6791
|
-
? { window: { since: stringFlag(args, 'since') ?? '—', until: stringFlag(args, 'until') ?? '—' } }
|
|
6792
|
-
: {}),
|
|
6793
|
-
...(pricingStale !== null ? { stalePricing: pricingStale } : {}),
|
|
6794
|
-
// The verdict, where the person reading CI will see it. recordGates()
|
|
6795
|
-
// runs before the side files for exactly this.
|
|
6796
|
-
...(gateVerdicts.length > 0 ? { gates: { failed: gateFailed, lines: gateVerdicts } } : {}),
|
|
6797
|
-
// The short form, for a reader who is not in the terminal.
|
|
6798
|
-
...(boolFlag(args, 'markdown-summary') ? { summary: true } : {}),
|
|
6799
|
-
// The repricing, when --what-if was given: computed once above and
|
|
6800
|
-
// handed over, so the summary in a pull request cannot disagree
|
|
6801
|
-
// with the terminal about what a move would cost.
|
|
6802
|
-
...(whatIf !== null ? { whatIf } : {}),
|
|
6803
|
-
pressure: pressures,
|
|
6804
|
-
// The comparison, when there was one — the same figures and the same
|
|
6805
|
-
// drivers the terminal printed, never re-derived here.
|
|
6806
|
-
...(previous !== null
|
|
6807
|
-
? {
|
|
6808
|
-
against: {
|
|
6809
|
-
previousTotalUsd: previous.total.totalUsd,
|
|
6810
|
-
previousCalls: previous.total.calls,
|
|
6811
|
-
labelDrivers,
|
|
6812
|
-
modelDrivers: new Set([
|
|
6813
|
-
...previous.byModel.map((r) => r.model),
|
|
6814
|
-
...report.byModel.map((r) => r.model),
|
|
6815
|
-
]).size > 1
|
|
6816
|
-
? modelDrivers
|
|
6817
|
-
: [],
|
|
6818
|
-
overlap: againstOverlap !== null
|
|
6819
|
-
? { from: dayOf(againstOverlap.fromMs), to: dayOf(againstOverlap.toMs) }
|
|
6820
|
-
: null,
|
|
6821
|
-
nothingPriced: previous.total.calls === 0,
|
|
6822
|
-
},
|
|
6823
|
-
}
|
|
6824
|
-
: {}),
|
|
6825
|
-
});
|
|
6826
|
-
if (markdownOut !== undefined) {
|
|
6827
|
-
await writeFile(markdownOut, renderProfileMarkdown(renderInput), 'utf8');
|
|
6828
|
-
notice(c.dim(t.report.wroteTo(markdownOut)));
|
|
6829
|
-
}
|
|
6830
|
-
if (htmlOut !== undefined) {
|
|
6831
|
-
await writeFile(htmlOut, renderProfileHtml(renderInput), 'utf8');
|
|
6832
|
-
notice(c.dim(t.html.written(htmlOut)));
|
|
6833
|
-
}
|
|
6834
|
-
}
|
|
6649
|
+
recordGates();
|
|
6650
|
+
await recordWaiverUses();
|
|
6651
|
+
return { failed: gateFailed, verdicts: gateVerdicts };
|
|
6652
|
+
}
|
|
6653
|
+
/**
|
|
6654
|
+
* The side files the caller asked for.
|
|
6655
|
+
*
|
|
6656
|
+
* Written on **both** output paths: under `--json` the human rendering returns
|
|
6657
|
+
* early, and the first version of `--csv-out` therefore wrote nothing at all
|
|
6658
|
+
* there — a flag that silently did nothing, which is the fault this repository
|
|
6659
|
+
* keeps refusing elsewhere.
|
|
6660
|
+
*
|
|
6661
|
+
* Its inputs are named because it had eleven of them and declared none: it was
|
|
6662
|
+
* a closure over `commandProfile`’s scope, so what a CSV or a markdown summary
|
|
6663
|
+
* is built from could only be learned by reading the whole body it sat in.
|
|
6664
|
+
*/
|
|
6665
|
+
async function writeProfileSideFiles(context) {
|
|
6666
|
+
const { args, path, report, previous, pressures, labelDrivers, modelDrivers, againstOverlap, whatIf, pricingStale, windowed, gates, pricing, t, } = context;
|
|
6667
|
+
/**
|
|
6668
|
+
* Where the "wrote to" notice goes. Under `--json`, stdout carries the
|
|
6669
|
+
* report and nothing else — a status line there turns a parseable
|
|
6670
|
+
* document into a parse error, which is how a pipeline discovers the
|
|
6671
|
+
* feature. The gates already route their verdicts to stderr for the same
|
|
6672
|
+
* reason.
|
|
6673
|
+
*/
|
|
6674
|
+
const notice = boolFlag(args, 'json')
|
|
6675
|
+
? (message) => console.error(message)
|
|
6676
|
+
: (message) => console.log(message);
|
|
6677
|
+
/**
|
|
6678
|
+
* The same report as GitHub-flavoured markdown, for a job summary or a PR
|
|
6679
|
+
* comment. Written from the same message catalogue the terminal used, because
|
|
6680
|
+
* two renderings of one finding drift the moment they are worded twice.
|
|
6681
|
+
*/
|
|
6682
|
+
const markdownOut = stringFlag(args, 'markdown-out');
|
|
6683
|
+
const htmlOut = stringFlag(args, 'html-out');
|
|
6684
|
+
if (markdownOut !== undefined || htmlOut !== undefined) {
|
|
6835
6685
|
/**
|
|
6836
|
-
*
|
|
6837
|
-
*
|
|
6838
|
-
*
|
|
6839
|
-
*
|
|
6686
|
+
* Derived here, not borrowed from the terminal path. Under `--json`
|
|
6687
|
+
* that path never runs, and the outer `levers` this block used to
|
|
6688
|
+
* reach for was an uninitialised binding — `profile --json
|
|
6689
|
+
* --markdown-out` crashed with a ReferenceError from 1.59 until the
|
|
6690
|
+
* HTML door's test drove both flags together and found it. The flag
|
|
6691
|
+
* did not silently do nothing; it loudly did nothing, and no test had
|
|
6692
|
+
* ever asked.
|
|
6840
6693
|
*/
|
|
6841
|
-
const
|
|
6842
|
-
|
|
6843
|
-
/**
|
|
6844
|
-
* Which table the file holds. One row shape per file on purpose: a
|
|
6845
|
-
* spreadsheet that has to filter before it can sum is a spreadsheet
|
|
6846
|
-
* somebody sums wrong.
|
|
6847
|
-
*/
|
|
6848
|
-
const shape = stringFlag(args, 'csv-shape') ?? 'slice';
|
|
6849
|
-
if (shape !== 'slice' && shape !== 'day' && shape !== 'hour' && shape !== 'model-day') {
|
|
6850
|
-
throw new Error(t.profile.badCsvShape(shape));
|
|
6851
|
-
}
|
|
6852
|
-
await writeFile(csvOut, profileToCsv(report, { unlabelled: t.profile.unlabelled(), shape }), 'utf8');
|
|
6853
|
-
notice(c.dim(t.report.wroteTo(csvOut)));
|
|
6854
|
-
}
|
|
6855
|
-
};
|
|
6856
|
-
if (boolFlag(args, 'json')) {
|
|
6694
|
+
const levers = billLevers(report, { catalogue: pricing });
|
|
6695
|
+
const cache = cacheEconomics(report.total);
|
|
6857
6696
|
/**
|
|
6858
|
-
*
|
|
6859
|
-
*
|
|
6860
|
-
*
|
|
6861
|
-
* returns. The cache verdict is included because leaving a consumer to
|
|
6862
|
-
* re-derive it means two implementations of a sign convention where positive
|
|
6863
|
-
* means *worse*, and one of them will eventually get it backwards.
|
|
6864
|
-
*
|
|
6865
|
-
* `levers` is included because it was not, and that made the flagship
|
|
6866
|
-
* section terminal-only: "What would actually move this bill" — the reason
|
|
6867
|
-
* the command exists — was invisible to any pipeline, dashboard or CI step
|
|
6868
|
-
* reading the JSON. A finding the machine-readable output omits is a finding
|
|
6869
|
-
* the reader's tooling will never surface.
|
|
6697
|
+
* One input object for both doors, built once: the HTML file and the
|
|
6698
|
+
* Markdown file are two projections of the same figures, and building
|
|
6699
|
+
* the input twice is how two renderings of one bill start disagreeing.
|
|
6870
6700
|
*/
|
|
6871
|
-
|
|
6872
|
-
|
|
6873
|
-
|
|
6874
|
-
|
|
6875
|
-
|
|
6876
|
-
|
|
6877
|
-
|
|
6878
|
-
|
|
6879
|
-
|
|
6880
|
-
|
|
6881
|
-
|
|
6882
|
-
|
|
6883
|
-
|
|
6884
|
-
|
|
6885
|
-
|
|
6886
|
-
|
|
6887
|
-
|
|
6888
|
-
|
|
6889
|
-
|
|
6890
|
-
|
|
6891
|
-
|
|
6892
|
-
// The provenance of every dollar above: which price table, how old.
|
|
6893
|
-
pricing: { lastReviewed: pricing.lastReviewed, ageDays: pricingAgeDays },
|
|
6894
|
-
levers: billLevers(report, { catalogue: pricing }),
|
|
6895
|
-
// Present only when --against was passed: null delta means the
|
|
6896
|
-
// previous log had nothing priced, which is a different answer from
|
|
6897
|
-
// zero growth.
|
|
6701
|
+
const renderInput = ({
|
|
6702
|
+
report,
|
|
6703
|
+
levers,
|
|
6704
|
+
cache,
|
|
6705
|
+
t,
|
|
6706
|
+
...(windowed
|
|
6707
|
+
? { window: { since: stringFlag(args, 'since') ?? '—', until: stringFlag(args, 'until') ?? '—' } }
|
|
6708
|
+
: {}),
|
|
6709
|
+
...(pricingStale !== null ? { stalePricing: pricingStale } : {}),
|
|
6710
|
+
// The verdict, where the person reading CI will see it. recordGates()
|
|
6711
|
+
// runs before the side files for exactly this.
|
|
6712
|
+
...(gates.verdicts.length > 0 ? { gates: { failed: gates.failed, lines: gates.verdicts } } : {}),
|
|
6713
|
+
// The short form, for a reader who is not in the terminal.
|
|
6714
|
+
...(boolFlag(args, 'markdown-summary') ? { summary: true } : {}),
|
|
6715
|
+
// The repricing, when --what-if was given: computed once above and
|
|
6716
|
+
// handed over, so the summary in a pull request cannot disagree
|
|
6717
|
+
// with the terminal about what a move would cost.
|
|
6718
|
+
...(whatIf !== null ? { whatIf } : {}),
|
|
6719
|
+
pressure: pressures,
|
|
6720
|
+
// The comparison, when there was one — the same figures and the same
|
|
6721
|
+
// drivers the terminal printed, never re-derived here.
|
|
6898
6722
|
...(previous !== null
|
|
6899
6723
|
? {
|
|
6900
6724
|
against: {
|
|
6901
6725
|
previousTotalUsd: previous.total.totalUsd,
|
|
6902
|
-
|
|
6903
|
-
|
|
6904
|
-
|
|
6905
|
-
|
|
6906
|
-
|
|
6907
|
-
|
|
6726
|
+
previousCalls: previous.total.calls,
|
|
6727
|
+
labelDrivers,
|
|
6728
|
+
modelDrivers: new Set([
|
|
6729
|
+
...previous.byModel.map((r) => r.model),
|
|
6730
|
+
...report.byModel.map((r) => r.model),
|
|
6731
|
+
]).size > 1
|
|
6732
|
+
? modelDrivers
|
|
6733
|
+
: [],
|
|
6734
|
+
overlap: againstOverlap !== null
|
|
6735
|
+
? { from: dayOf(againstOverlap.fromMs), to: dayOf(againstOverlap.toMs) }
|
|
6736
|
+
: null,
|
|
6737
|
+
nothingPriced: previous.total.calls === 0,
|
|
6908
6738
|
},
|
|
6909
6739
|
}
|
|
6910
6740
|
: {}),
|
|
6911
|
-
|
|
6912
|
-
|
|
6913
|
-
|
|
6914
|
-
|
|
6915
|
-
// Present only when --what-if was passed. `sameTokensAssumed` rides
|
|
6916
|
-
// along inside it so a consumer cannot print the dollar figure
|
|
6917
|
-
// without the caveat being in the same object.
|
|
6918
|
-
...(whatIf !== null ? { whatIf } : {}),
|
|
6919
|
-
}, null, 2));
|
|
6920
|
-
recordGates();
|
|
6921
|
-
await recordWaiverUses();
|
|
6922
|
-
await writeSideFiles();
|
|
6923
|
-
return;
|
|
6924
|
-
}
|
|
6925
|
-
/**
|
|
6926
|
-
* Nothing priced means there is no report, not a report of zero.
|
|
6927
|
-
*
|
|
6928
|
-
* The guard was `total.calls === 0 && unpriced.calls === 0`, so a log whose every
|
|
6929
|
-
* model was unknown fell through and printed a full report built from a zeroed
|
|
6930
|
-
* total: `0 calls · $0`, four `$0 / 0.0%` rows, a meaningless "Input is 0.0% of
|
|
6931
|
-
* this bill", and — on a log containing a hundred thousand cache-read tokens —
|
|
6932
|
-
* the flatly false "Caching was never used on these calls".
|
|
6933
|
-
*
|
|
6934
|
-
* Two affirmatively wrong claims and a $0 headline for a real bill. The trailing
|
|
6935
|
-
* unpriced note was the only correct line on screen, and it was the quietest.
|
|
6936
|
-
*/
|
|
6937
|
-
if (report.total.calls === 0) {
|
|
6938
|
-
console.log();
|
|
6939
|
-
console.log(c.dim(report.unpriced.calls === 0 ? t.profile.empty() : t.profile.nothingPriced()));
|
|
6940
|
-
reportProfileGaps(report, t, n, pricingStale);
|
|
6941
|
-
return;
|
|
6942
|
-
}
|
|
6943
|
-
const shares = sharesOf(report.total);
|
|
6944
|
-
const parts = [
|
|
6945
|
-
[t.profile.partInput(), report.total.inputUsd, shares.input, report.total.inputTokens],
|
|
6946
|
-
[t.profile.partCacheRead(), report.total.cacheReadUsd, shares.cacheRead, report.total.cacheReadTokens],
|
|
6947
|
-
[t.profile.partCacheWrite(), report.total.cacheWriteUsd, shares.cacheWrite, report.total.cacheWriteTokens],
|
|
6948
|
-
[t.profile.partOutput(), report.total.outputUsd, shares.output, report.total.outputTokens],
|
|
6949
|
-
];
|
|
6950
|
-
console.log();
|
|
6951
|
-
console.log(sectionHeading(t.profile.heading()));
|
|
6952
|
-
console.log(` ${t.profile.spent(t.profile.calls(report.total.calls), formatUsd(report.total.totalUsd))}`);
|
|
6953
|
-
/**
|
|
6954
|
-
* The period, when the log carries a clock — stated, never extrapolated. A
|
|
6955
|
-
* span makes the reader's own monthly arithmetic valid; a per-month figure
|
|
6956
|
-
* printed from a partial month would be this tool doing the guessing it
|
|
6957
|
-
* exists to end. Partial coverage is said in the same breath, because a span
|
|
6958
|
-
* over a third of the calls silently presented as the log's period is a
|
|
6959
|
-
* figure attributed to something it does not describe.
|
|
6960
|
-
*/
|
|
6961
|
-
if (report.span !== null) {
|
|
6962
|
-
const totalParsed = report.total.calls + report.unpriced.calls;
|
|
6963
|
-
const partial = report.span.calls < totalParsed
|
|
6964
|
-
? ` ${t.profile.spanPartial(n(report.span.calls), n(totalParsed))}`
|
|
6965
|
-
: '';
|
|
6966
|
-
console.log(` ${c.dim(wrap(`${t.profile.spanLine(dayOf(report.span.fromMs), dayOf(report.span.toMs), spanDays(report.span.fromMs, report.span.toMs))}${partial}`, 74, ' '))}`);
|
|
6967
|
-
}
|
|
6968
|
-
// How many files this report covers, when it covers more than one: a total
|
|
6969
|
-
// over "the logs" that silently skipped one is wrong by an unknown amount.
|
|
6970
|
-
if (logFiles.length > 1) {
|
|
6971
|
-
console.log(` ${c.dim(wrap(t.profile.readFiles(logFiles.length, path), 74, ' '))}`);
|
|
6972
|
-
}
|
|
6973
|
-
/**
|
|
6974
|
-
* A doubled bill, said before anything is believed.
|
|
6975
|
-
*
|
|
6976
|
-
* Reading a directory of rotated logs makes double-counting easy — a log
|
|
6977
|
-
* exported twice, an overlapping export, a copy left in the folder — and
|
|
6978
|
-
* the total then reads high with nothing else able to see it. Only counted
|
|
6979
|
-
* over records with a clock, where an identical line is a claim worth
|
|
6980
|
-
* making. It states the count and the money and stops: whether it is a
|
|
6981
|
-
* double export or a genuinely busy millisecond is the reader's to know.
|
|
6982
|
-
*/
|
|
6983
|
-
if (report.duplicateLines.count > 0) {
|
|
6984
|
-
console.log(` ${c.yellow('!')} ${c.dim(wrap(t.profile.duplicateLines(report.duplicateLines.count, formatUsd(report.duplicateLines.usd)), 74, ' '))}`);
|
|
6985
|
-
}
|
|
6986
|
-
/**
|
|
6987
|
-
* The window, said before any figure is trusted as "the log": everything
|
|
6988
|
-
* below describes a slice, and a slice presented as the whole is a figure
|
|
6989
|
-
* attributed to something it does not describe. The undated count is loud —
|
|
6990
|
-
* those calls' spend is in the log and not in this report, so the window's
|
|
6991
|
-
* figures are a floor on the period, and only this line says so.
|
|
6992
|
-
*/
|
|
6993
|
-
if (report.timeWindow !== null) {
|
|
6994
|
-
console.log(` ${c.dim(wrap(t.profile.windowLine(stringFlag(args, 'since') ?? '—', stringFlag(args, 'until') ?? '—'), 74, ' '))}`);
|
|
6995
|
-
if (relativeWindow) {
|
|
6996
|
-
console.log(` ${c.dim(wrap(t.profile.windowRelative(), 74, ' '))}`);
|
|
6741
|
+
});
|
|
6742
|
+
if (markdownOut !== undefined) {
|
|
6743
|
+
await writeFile(markdownOut, renderProfileMarkdown(renderInput), 'utf8');
|
|
6744
|
+
notice(c.dim(t.report.wroteTo(markdownOut)));
|
|
6997
6745
|
}
|
|
6998
|
-
if (
|
|
6999
|
-
|
|
6746
|
+
if (htmlOut !== undefined) {
|
|
6747
|
+
await writeFile(htmlOut, renderProfileHtml(renderInput), 'utf8');
|
|
6748
|
+
notice(c.dim(t.html.written(htmlOut)));
|
|
7000
6749
|
}
|
|
7001
6750
|
}
|
|
7002
|
-
console.log();
|
|
7003
|
-
// Every part, including the zero ones. A row missing because it was zero reads
|
|
7004
|
-
// as a row somebody forgot, and "you are not caching at all" is a finding.
|
|
7005
|
-
for (const [name, usd, share, tokens] of parts) {
|
|
7006
|
-
console.log(` ${bar(share)} ${t.profile.part(name, formatUsd(usd), pct(share), n(tokens))}`);
|
|
7007
|
-
}
|
|
7008
|
-
/**
|
|
7009
|
-
* The line the command exists for: which part of the bill to argue with.
|
|
7010
|
-
*
|
|
7011
|
-
* When output is both the biggest part and over half, the two sentences say the
|
|
7012
|
-
* same thing and the second says more — so only the second prints. Reporting a
|
|
7013
|
-
* fact twice in adjacent lines reads as a bug, and it was one.
|
|
7014
|
-
*/
|
|
7015
|
-
const [biggestName, , biggestShare] = parts.reduce((a, b) => (b[1] > a[1] ? b : a));
|
|
7016
|
-
const outputDominates = shares.output > 0.5;
|
|
7017
|
-
console.log();
|
|
7018
|
-
if (outputDominates) {
|
|
7019
|
-
console.log(` ${c.bold(wrap(t.profile.outputDominates(pct(shares.output)), 74, ' '))}`);
|
|
7020
|
-
}
|
|
7021
|
-
else {
|
|
7022
|
-
console.log(` ${c.bold(t.profile.biggestPart(biggestName, pct(biggestShare)))}`);
|
|
7023
|
-
}
|
|
7024
6751
|
/**
|
|
7025
|
-
* The
|
|
7026
|
-
*
|
|
7027
|
-
*
|
|
7028
|
-
*
|
|
7029
|
-
* for opposite responses. Rendered against the **median** day — a mean would
|
|
7030
|
-
* let the spike inflate its own yardstick — and loud only when it clears twice
|
|
7031
|
-
* the median, a threshold stated in the sentence rather than hidden in code.
|
|
6752
|
+
* The same report as a spreadsheet, one row per label and model — the grain
|
|
6753
|
+
* a routing or budget decision is made at. Deliberately without a total
|
|
6754
|
+
* row: a total inside a data file is summed with the data and doubles every
|
|
6755
|
+
* figure downstream.
|
|
7032
6756
|
*/
|
|
7033
|
-
|
|
7034
|
-
|
|
7035
|
-
|
|
7036
|
-
|
|
7037
|
-
|
|
7038
|
-
|
|
7039
|
-
|
|
7040
|
-
|
|
7041
|
-
|
|
7042
|
-
|
|
7043
|
-
const text = wrap(`${line}${labelClause}`, 74, ' ');
|
|
7044
|
-
console.log(` ${loud ? c.yellow(text) : c.dim(text)}`);
|
|
6757
|
+
const csvOut = stringFlag(args, 'csv-out');
|
|
6758
|
+
if (csvOut !== undefined) {
|
|
6759
|
+
/**
|
|
6760
|
+
* Which table the file holds. One row shape per file on purpose: a
|
|
6761
|
+
* spreadsheet that has to filter before it can sum is a spreadsheet
|
|
6762
|
+
* somebody sums wrong.
|
|
6763
|
+
*/
|
|
6764
|
+
const shape = stringFlag(args, 'csv-shape') ?? 'slice';
|
|
6765
|
+
if (shape !== 'slice' && shape !== 'day' && shape !== 'hour' && shape !== 'model-day') {
|
|
6766
|
+
throw new Error(t.profile.badCsvShape(shape));
|
|
7045
6767
|
}
|
|
6768
|
+
await writeFile(csvOut, profileToCsv(report, { unlabelled: t.profile.unlabelled(), shape }), 'utf8');
|
|
6769
|
+
notice(c.dim(t.report.wroteTo(csvOut)));
|
|
7046
6770
|
}
|
|
6771
|
+
}
|
|
6772
|
+
/**
|
|
6773
|
+
* What this log recorded about outcomes, and what it cannot answer.
|
|
6774
|
+
*
|
|
6775
|
+
* Every finding past the totals needs a field the format does not require, and a
|
|
6776
|
+
* reader who never adds them sees a report quietly missing half of itself, with no
|
|
6777
|
+
* way to tell "nothing to report" from "nothing recorded". Named with counts rather
|
|
6778
|
+
* than booleans: twelve labelled records out of forty thousand is not a labelled
|
|
6779
|
+
* log, and a boolean would call it one.
|
|
6780
|
+
*
|
|
6781
|
+
* Outcomes print above coverage rather than below, and the two travel together here
|
|
6782
|
+
* for the same reason they do on screen: coverage is the sentence that qualifies
|
|
6783
|
+
* whatever the outcomes just claimed.
|
|
6784
|
+
*/
|
|
6785
|
+
function reportOutcomesAndCoverage(report, config, t) {
|
|
6786
|
+
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
6787
|
+
const pct = (share) => `${(share * 100).toFixed(1)}%`;
|
|
7047
6788
|
/**
|
|
7048
|
-
*
|
|
6789
|
+
* What this log cannot answer, and what would fix it.
|
|
7049
6790
|
*
|
|
7050
|
-
*
|
|
7051
|
-
*
|
|
7052
|
-
*
|
|
7053
|
-
*
|
|
7054
|
-
*
|
|
7055
|
-
* flat.
|
|
6791
|
+
* Every finding past the totals needs a field the format does not require,
|
|
6792
|
+
* and a reader who never adds them sees a report quietly missing half of
|
|
6793
|
+
* itself — with no way to tell "nothing to report" from "nothing recorded".
|
|
6794
|
+
* Named with counts rather than booleans: twelve labelled records out of
|
|
6795
|
+
* forty thousand is not a labelled log, and a boolean would call it one.
|
|
7056
6796
|
*
|
|
7057
|
-
*
|
|
7058
|
-
*
|
|
7059
|
-
*
|
|
7060
|
-
* section already prints is the one with money attached.
|
|
6797
|
+
* Only fields that are actually missing are listed. A complete log gets no
|
|
6798
|
+
* section at all, because a paragraph of things that are fine is the
|
|
6799
|
+
* paragraph readers learn to skip.
|
|
7061
6800
|
*/
|
|
7062
|
-
if (report.spendByHour.length >= 4 && report.total.totalUsd > 0) {
|
|
7063
|
-
const sorted = [...report.spendByHour].sort((a, b) => b.usd - a.usd);
|
|
7064
|
-
let covered = 0;
|
|
7065
|
-
let hoursForMost = 0;
|
|
7066
|
-
for (const hour of sorted) {
|
|
7067
|
-
covered += hour.usd;
|
|
7068
|
-
hoursForMost += 1;
|
|
7069
|
-
if (covered >= 0.8 * report.total.totalUsd)
|
|
7070
|
-
break;
|
|
7071
|
-
}
|
|
7072
|
-
const busiest = sorted
|
|
7073
|
-
.slice(0, hoursForMost)
|
|
7074
|
-
.map((hour) => hour.hour)
|
|
7075
|
-
.sort((a, b) => a - b)
|
|
7076
|
-
.map((hour) => `${String(hour).padStart(2, '0')}:00`)
|
|
7077
|
-
.join(', ');
|
|
7078
|
-
console.log();
|
|
7079
|
-
console.log(` ${c.dim(wrap(hoursForMost <= 8 ? t.profile.hoursConcentrated(n(hoursForMost), busiest) : t.profile.hoursFlat(n(hoursForMost)), 74, ' '))}`);
|
|
7080
|
-
}
|
|
7081
6801
|
/**
|
|
7082
|
-
*
|
|
6802
|
+
* Outcomes — the counterpart, where somebody recorded one.
|
|
7083
6803
|
*
|
|
7084
|
-
*
|
|
7085
|
-
*
|
|
7086
|
-
*
|
|
7087
|
-
*
|
|
7088
|
-
*
|
|
6804
|
+
* Printed above the coverage section rather than below it, because when this
|
|
6805
|
+
* section is present it is the most valuable thing on the page: every other
|
|
6806
|
+
* figure in this report is a cost, and this is the only one that says what
|
|
6807
|
+
* the money bought.
|
|
6808
|
+
*
|
|
6809
|
+
* Silent when nothing recorded an outcome. The coverage section below
|
|
6810
|
+
* already names the missing field and what it would unlock, and printing an
|
|
6811
|
+
* empty Outcomes heading above it would be the same sentence twice.
|
|
7089
6812
|
*/
|
|
7090
|
-
|
|
7091
|
-
|
|
7092
|
-
|
|
7093
|
-
|
|
6813
|
+
{
|
|
6814
|
+
const outcomes = outcomeReport(report.outcomeTally, config.outcomes ?? null);
|
|
6815
|
+
if (outcomes.coverage.recorded > 0) {
|
|
6816
|
+
console.log();
|
|
6817
|
+
console.log(sectionHeading(t.profile.outcomeHeading()));
|
|
6818
|
+
const col = t.profile.outcomeColumns;
|
|
6819
|
+
const rows = [...outcomes.slices, ...outcomes.undeclared].map((slice) => ({
|
|
6820
|
+
value: slice.value,
|
|
6821
|
+
verdict: slice.verdict === 'success'
|
|
6822
|
+
? t.profile.verdictSuccess()
|
|
6823
|
+
: slice.verdict === 'undeclared'
|
|
6824
|
+
? t.profile.verdictUndeclared()
|
|
6825
|
+
: t.profile.verdictOther(),
|
|
6826
|
+
calls: n(slice.calls),
|
|
6827
|
+
spend: formatUsd(slice.usd),
|
|
6828
|
+
}));
|
|
6829
|
+
const w = {
|
|
6830
|
+
value: Math.max(...rows.map((r) => r.value.length), col.outcome.length),
|
|
6831
|
+
verdict: Math.max(...rows.map((r) => r.verdict.length), 0),
|
|
6832
|
+
calls: Math.max(...rows.map((r) => r.calls.length), col.calls.length),
|
|
6833
|
+
spend: Math.max(...rows.map((r) => r.spend.length), col.spend.length),
|
|
6834
|
+
};
|
|
6835
|
+
console.log(c.dim(` ${col.outcome.padEnd(w.value)} ${''.padEnd(w.verdict)} ` +
|
|
6836
|
+
`${col.calls.padStart(w.calls)} ${col.spend.padStart(w.spend)}`));
|
|
6837
|
+
for (const row of rows) {
|
|
6838
|
+
const tint = row.verdict === t.profile.verdictUndeclared()
|
|
6839
|
+
? c.yellow
|
|
6840
|
+
: row.verdict === t.profile.verdictSuccess()
|
|
6841
|
+
? c.green
|
|
6842
|
+
: c.dim;
|
|
6843
|
+
console.log(` ${row.value.padEnd(w.value)} ${tint(row.verdict.padEnd(w.verdict))} ` +
|
|
6844
|
+
`${row.calls.padStart(w.calls)} ${row.spend.padStart(w.spend)}`);
|
|
6845
|
+
}
|
|
6846
|
+
console.log();
|
|
6847
|
+
if (outcomes.successShareOfRecordedUsd !== null) {
|
|
6848
|
+
const declaredUsd = outcomes.slices.reduce((sum, slice) => sum + slice.usd, 0);
|
|
6849
|
+
console.log(` ${wrap(t.profile.outcomeRate(pct(outcomes.successShareOfRecordedUsd), formatUsd(declaredUsd)), 74, ' ')}`);
|
|
6850
|
+
}
|
|
6851
|
+
else if (outcomes.noRate !== null) {
|
|
6852
|
+
console.log(` ${c.dim(wrap(t.profile.outcomeNoRate(outcomes.noRate), 74, ' '))}`);
|
|
6853
|
+
}
|
|
6854
|
+
// What the rate does not cover, every time it is printed. A rate over a
|
|
6855
|
+
// twelfth of the bill is a rate about a twelfth of the bill.
|
|
6856
|
+
if (outcomes.coverage.unrecordedUsd > 0 && report.total.totalUsd > 0) {
|
|
6857
|
+
console.log(` ${c.yellow('!')} ${wrap(t.profile.outcomeUnrecorded(pct(outcomes.coverage.unrecordedUsd / report.total.totalUsd), formatUsd(outcomes.coverage.unrecordedUsd)), 74, ' ')}`);
|
|
6858
|
+
}
|
|
6859
|
+
/**
|
|
6860
|
+
* What an outcome costs, per workload — the finding a total cannot make.
|
|
6861
|
+
*
|
|
6862
|
+
* Printed only when at least one workload has enough recorded outcomes
|
|
6863
|
+
* to say something, and every row that cannot state a figure says which
|
|
6864
|
+
* of the five reasons applies rather than showing a blank.
|
|
6865
|
+
*/
|
|
6866
|
+
const ranking = rankPerOutcome(report.outcomeTallyByLabel.map((entry) => ({
|
|
6867
|
+
key: entry.label,
|
|
6868
|
+
calls: entry.calls,
|
|
6869
|
+
totalUsd: entry.totalUsd,
|
|
6870
|
+
tally: entry.tally,
|
|
6871
|
+
})), config.outcomes ?? null);
|
|
6872
|
+
if (ranking.byCall.length > 0) {
|
|
6873
|
+
console.log();
|
|
6874
|
+
console.log(sectionHeading(t.profile.perOutcomeHeading()));
|
|
6875
|
+
const pcol = t.profile.perOutcomeColumns;
|
|
6876
|
+
const prows = ranking.byCall.map((slice) => ({
|
|
6877
|
+
key: slice.key,
|
|
6878
|
+
perCall: formatUsd(slice.usdPerCall),
|
|
6879
|
+
perOutcome: slice.per.usdPerSuccess !== null
|
|
6880
|
+
? formatUsd(slice.per.usdPerSuccess)
|
|
6881
|
+
: t.profile.perOutcomeWithheld(slice.per.withheld ?? 'no-vocabulary', n(slice.per.successes), pct(slice.per.coverage)),
|
|
6882
|
+
recorded: pct(slice.per.coverage),
|
|
6883
|
+
}));
|
|
6884
|
+
const pw = {
|
|
6885
|
+
key: Math.max(...prows.map((r) => r.key.length), pcol.workload.length),
|
|
6886
|
+
perCall: Math.max(...prows.map((r) => r.perCall.length), pcol.perCall.length),
|
|
6887
|
+
perOutcome: Math.max(...prows.map((r) => r.perOutcome.length), pcol.perOutcome.length),
|
|
6888
|
+
recorded: Math.max(...prows.map((r) => r.recorded.length), pcol.recorded.length),
|
|
6889
|
+
};
|
|
6890
|
+
console.log(c.dim(` ${pcol.workload.padEnd(pw.key)} ${pcol.perCall.padStart(pw.perCall)} ` +
|
|
6891
|
+
`${pcol.perOutcome.padStart(pw.perOutcome)} ${pcol.recorded.padStart(pw.recorded)}`));
|
|
6892
|
+
for (const row of prows) {
|
|
6893
|
+
console.log(` ${row.key.padEnd(pw.key)} ${row.perCall.padStart(pw.perCall)} ` +
|
|
6894
|
+
`${row.perOutcome.padStart(pw.perOutcome)} ${c.dim(row.recorded.padStart(pw.recorded))}`);
|
|
6895
|
+
}
|
|
6896
|
+
console.log();
|
|
6897
|
+
console.log(` ${c.dim(wrap(t.profile.perOutcomeNumerator(), 74, ' '))}`);
|
|
6898
|
+
// The disagreement between the two orders, which is itself the finding.
|
|
6899
|
+
if (ranking.disagreements.length > 0) {
|
|
6900
|
+
console.log();
|
|
6901
|
+
console.log(` ${c.dim(wrap(t.profile.perOutcomeBothOrders(), 74, ' '))}`);
|
|
6902
|
+
for (const d of ranking.disagreements) {
|
|
6903
|
+
console.log(` ${c.yellow('→')} ${wrap(t.profile.perOutcomeDisagreement(d.slice.key, n(d.callRank + 1), n(d.outcomeRank + 1)), 74, ' ')}`);
|
|
6904
|
+
}
|
|
6905
|
+
}
|
|
6906
|
+
}
|
|
6907
|
+
if (outcomes.undeclared.length > 0) {
|
|
6908
|
+
console.log(` ${c.yellow('!')} ${wrap(t.profile.outcomeUndeclared(outcomes.undeclared.map((s) => s.value).join(', ')), 74, ' ')}`);
|
|
6909
|
+
}
|
|
6910
|
+
}
|
|
7094
6911
|
}
|
|
7095
|
-
|
|
7096
|
-
|
|
6912
|
+
const coverage = report.fieldCoverage;
|
|
6913
|
+
if (coverage.parsed > 0) {
|
|
6914
|
+
const missing = [];
|
|
6915
|
+
const partial = (seen) => `${n(seen)}/${n(coverage.parsed)}`;
|
|
6916
|
+
if (coverage.label < coverage.parsed) {
|
|
6917
|
+
missing.push(t.profile.needsLabel(partial(coverage.label)));
|
|
6918
|
+
}
|
|
6919
|
+
if (coverage.session < coverage.parsed) {
|
|
6920
|
+
missing.push(t.profile.needsSession(partial(coverage.session)));
|
|
6921
|
+
}
|
|
6922
|
+
/**
|
|
6923
|
+
* Listed first among the missing when it is missing entirely, because it
|
|
6924
|
+
* is the one field that changes what every other figure here *means*. The
|
|
6925
|
+
* rest sharpen a cost; this one gives it a counterpart.
|
|
6926
|
+
*/
|
|
6927
|
+
if (coverage.outcome < coverage.parsed) {
|
|
6928
|
+
missing.push(t.profile.needsOutcome(partial(coverage.outcome)));
|
|
6929
|
+
}
|
|
6930
|
+
if (coverage.ts < coverage.parsed) {
|
|
6931
|
+
missing.push(t.profile.needsTs(partial(coverage.ts)));
|
|
6932
|
+
}
|
|
6933
|
+
if (coverage.stopReason < coverage.parsed) {
|
|
6934
|
+
missing.push(t.profile.needsStopReason(partial(coverage.stopReason)));
|
|
6935
|
+
}
|
|
6936
|
+
if (coverage.cacheWrites > 0 && coverage.cacheTtl < coverage.cacheWrites) {
|
|
6937
|
+
missing.push(t.profile.needsCacheTtl(`${n(coverage.cacheTtl)}/${n(coverage.cacheWrites)}`));
|
|
6938
|
+
}
|
|
6939
|
+
if (missing.length > 0) {
|
|
6940
|
+
console.log();
|
|
6941
|
+
console.log(sectionHeading(t.profile.coverageHeading()));
|
|
6942
|
+
for (const line of missing)
|
|
6943
|
+
console.log(` ${c.dim(wrap(line, 74, ' '))}`);
|
|
6944
|
+
}
|
|
6945
|
+
}
|
|
6946
|
+
}
|
|
6947
|
+
/**
|
|
6948
|
+
* This bill against the previous one: how spend actually gets out of hand.
|
|
6949
|
+
*
|
|
6950
|
+
* Nobody adds five thousand a month in one day; bills grow four percent a week
|
|
6951
|
+
* while every snapshot looks reasonable. This is the baseline gate the prompts
|
|
6952
|
+
* already had, applied to the money itself. **Positive means the bill grew** (the
|
|
6953
|
+
* diff convention), and every figure is between exactly these two files.
|
|
6954
|
+
*
|
|
6955
|
+
* Its inputs are stated because it had eight of them and declared none. Two are
|
|
6956
|
+
* drivers computed three hundred lines earlier and used nowhere else, which is a
|
|
6957
|
+
* thing you could only find out by reading both places.
|
|
6958
|
+
*/
|
|
6959
|
+
function reportAgainstPrevious(context) {
|
|
6960
|
+
const { report, previous, againstOverlap, labelDrivers, modelDrivers, gates, now, t } = context;
|
|
6961
|
+
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
6962
|
+
const pct = (share) => `${(share * 100).toFixed(1)}%`;
|
|
6963
|
+
/**
|
|
6964
|
+
* This bill against the previous one — how spend actually gets out of hand.
|
|
6965
|
+
*
|
|
6966
|
+
* Nobody adds five thousand a month in one day; bills grow four percent a week
|
|
6967
|
+
* while every snapshot looks reasonable. This is the baseline gate the prompts
|
|
6968
|
+
* already had, applied to the money itself. **Positive means the bill grew**
|
|
6969
|
+
* (the diff convention), and every figure is between exactly these two files:
|
|
6970
|
+
* no period is assumed, so the call counts print beside the money for the
|
|
6971
|
+
* reader to judge comparability before judging the trend.
|
|
6972
|
+
*/
|
|
6973
|
+
if (previous !== null) {
|
|
6974
|
+
console.log();
|
|
6975
|
+
console.log(sectionHeading(t.profile.againstHeading()));
|
|
6976
|
+
if (previous.total.calls === 0) {
|
|
6977
|
+
console.log(` ${c.dim(wrap(t.profile.againstNothingPriced(), 74, ' '))}`);
|
|
6978
|
+
}
|
|
6979
|
+
else {
|
|
6980
|
+
const delta = report.total.totalUsd - previous.total.totalUsd;
|
|
6981
|
+
const growthPct = previous.total.totalUsd > 0
|
|
6982
|
+
? `${delta >= 0 ? '+' : ''}${((delta / previous.total.totalUsd) * 100).toFixed(1)}%`
|
|
6983
|
+
: '—';
|
|
6984
|
+
console.log(` ${c.bold(wrap(t.profile.againstTotals(formatUsd(previous.total.totalUsd), formatUsd(report.total.totalUsd), formatSignedUsd(delta), growthPct, t.profile.calls(previous.total.calls), t.profile.calls(report.total.calls)), 74, ' '))}`);
|
|
6985
|
+
// Overlapping spans mean part of this "growth" is the same money on
|
|
6986
|
+
// both sides of the subtraction. Said after the figure it qualifies
|
|
6987
|
+
// and before the drivers built from it.
|
|
6988
|
+
if (againstOverlap !== null) {
|
|
6989
|
+
console.log(` ${c.yellow('!')} ${c.dim(wrap(t.profile.againstOverlap(dayOf(againstOverlap.fromMs), dayOf(againstOverlap.toMs)), 74, ' '))}`);
|
|
6990
|
+
}
|
|
6991
|
+
// Drivers: per-key contribution to the change, largest magnitude first,
|
|
6992
|
+
// computed once beside the gates so no rendering derives its own.
|
|
6993
|
+
const driverLine = (d, shown) => d.was === null
|
|
6994
|
+
? t.profile.againstDriverNew(formatSignedUsd(d.delta), shown)
|
|
6995
|
+
: d.now === null
|
|
6996
|
+
? t.profile.againstDriverGone(formatSignedUsd(d.delta), shown)
|
|
6997
|
+
: t.profile.againstDriver(formatSignedUsd(d.delta), shown, formatUsd(d.was), formatUsd(d.now));
|
|
6998
|
+
console.log();
|
|
6999
|
+
for (const d of labelDrivers.slice(0, 5)) {
|
|
7000
|
+
const line = driverLine(d, d.key === UNLABELLED ? t.profile.unlabelled() : d.key);
|
|
7001
|
+
console.log(` ${d.delta > 0 ? c.yellow(line) : c.dim(line)}`);
|
|
7002
|
+
}
|
|
7003
|
+
if (labelDrivers.length > 5) {
|
|
7004
|
+
console.log(` ${c.dim(t.profile.andMoreLabels(labelDrivers.length - 5))}`);
|
|
7005
|
+
}
|
|
7006
|
+
/**
|
|
7007
|
+
* The same change, by model — where the mix moved. The label rows cannot
|
|
7008
|
+
* show it: a workload that kept its name and switched from Haiku to Opus
|
|
7009
|
+
* reads as "chat grew", and the reason is the model. Only printed when
|
|
7010
|
+
* more than one model is involved; with one model on both sides, this
|
|
7011
|
+
* section restates the totals line and says nothing new.
|
|
7012
|
+
*/
|
|
7013
|
+
const modelsInvolved = new Set([
|
|
7014
|
+
...previous.byModel.map((r) => r.model),
|
|
7015
|
+
...report.byModel.map((r) => r.model),
|
|
7016
|
+
]);
|
|
7017
|
+
if (modelDrivers.length > 0 && modelsInvolved.size > 1) {
|
|
7018
|
+
console.log();
|
|
7019
|
+
console.log(` ${c.dim(t.profile.againstByModel())}`);
|
|
7020
|
+
for (const d of modelDrivers.slice(0, 3)) {
|
|
7021
|
+
const line = driverLine(d, d.key);
|
|
7022
|
+
console.log(` ${d.delta > 0 ? c.yellow(line) : c.dim(line)}`);
|
|
7023
|
+
}
|
|
7024
|
+
}
|
|
7025
|
+
/**
|
|
7026
|
+
* What the comparison stopped being able to see.
|
|
7027
|
+
*
|
|
7028
|
+
* Every figure above is dollars, and dollars cannot tell a finding that
|
|
7029
|
+
* was fixed from a finding whose field the log stopped recording — both
|
|
7030
|
+
* are silence. This is the only section that can, so it is loud: a
|
|
7031
|
+
* collapse in coverage invalidates whichever findings depended on it,
|
|
7032
|
+
* and reading the drop as good news is the specific mistake it exists
|
|
7033
|
+
* to prevent.
|
|
7034
|
+
*/
|
|
7035
|
+
const drifts = coverageDrift(previous.fieldCoverage, report.fieldCoverage);
|
|
7036
|
+
if (drifts.length > 0) {
|
|
7037
|
+
console.log();
|
|
7038
|
+
for (const drift of drifts) {
|
|
7039
|
+
const line = t.profile.coverageDrift(t.profile.coverageField(drift.field), pct(drift.was), pct(drift.now));
|
|
7040
|
+
console.log(drift.delta < 0
|
|
7041
|
+
? ` ${c.yellow('!')} ${c.bold(wrap(line, 74, ' '))}`
|
|
7042
|
+
: ` ${c.dim(wrap(line, 74, ' '))}`);
|
|
7043
|
+
/**
|
|
7044
|
+
* Which findings went with it, named. "Some findings are silent" is
|
|
7045
|
+
* not something a reader can act on; knowing that conversation
|
|
7046
|
+
* growth and the cache-TTL fit are now silence rather than absence
|
|
7047
|
+
* tells them exactly which sections of this report to distrust.
|
|
7048
|
+
*/
|
|
7049
|
+
if (drift.delta < 0) {
|
|
7050
|
+
const silenced = t.profile.coverageSilenced(drift.field);
|
|
7051
|
+
if (silenced !== '')
|
|
7052
|
+
console.log(` ${c.dim(wrap(silenced, 72, ' '))}`);
|
|
7053
|
+
}
|
|
7054
|
+
}
|
|
7055
|
+
if (drifts.some((d) => d.delta < 0)) {
|
|
7056
|
+
console.log(` ${c.dim(wrap(t.profile.coverageDriftWhy(), 74, ' '))}`);
|
|
7057
|
+
}
|
|
7058
|
+
}
|
|
7059
|
+
}
|
|
7060
|
+
}
|
|
7061
|
+
}
|
|
7062
|
+
/**
|
|
7063
|
+
* The same request, sent again a moment later.
|
|
7064
|
+
*
|
|
7065
|
+
* A conversation's input grows with every turn, so two calls of the same size
|
|
7066
|
+
* seconds apart are not a conversation: they are a retry, a duplicated step, a
|
|
7067
|
+
* loop. Loud, because the money bought nothing, and hedged, because this reads
|
|
7068
|
+
* counts and cannot see content.
|
|
7069
|
+
*/
|
|
7070
|
+
function reportRepeatedCalls(report, t) {
|
|
7071
|
+
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
7072
|
+
const pct = (share) => `${(share * 100).toFixed(1)}%`;
|
|
7073
|
+
/**
|
|
7074
|
+
* The same request, sent again a moment later.
|
|
7075
|
+
*
|
|
7076
|
+
* A conversation's input grows with every turn, so two consecutive calls in
|
|
7077
|
+
* one conversation carrying the same size seconds apart is a thing going
|
|
7078
|
+
* wrong rather than a thing working — a retry after a timeout, an agent
|
|
7079
|
+
* step repeating, a loop. Loud, because the money bought nothing, and
|
|
7080
|
+
* hedged, because this reads counts and cannot see content: the sentence
|
|
7081
|
+
* says the pattern is *usually* a retry, never that it is one.
|
|
7082
|
+
*/
|
|
7083
|
+
if (report.repeatedTurns.length > 0) {
|
|
7084
|
+
console.log();
|
|
7085
|
+
console.log(sectionHeading(t.profile.repeatsHeading()));
|
|
7086
|
+
for (const row of report.repeatedTurns.slice(0, 3)) {
|
|
7087
|
+
const label = row.label === UNLABELLED ? t.profile.unlabelled() : row.label;
|
|
7088
|
+
console.log();
|
|
7089
|
+
console.log(` ${c.yellow('!')} ${c.bold(wrap(t.profile.repeatsFound(label, row.modelName, n(row.repeats), n(row.checkedCalls), n(Math.round(row.withinMs / 1000)), formatUsd(row.usd)), 74, ' '))}`);
|
|
7090
|
+
console.log(` ${c.dim(wrap(t.profile.repeatsAdvice(), 74, ' '))}`);
|
|
7091
|
+
}
|
|
7097
7092
|
}
|
|
7098
7093
|
/**
|
|
7099
|
-
*
|
|
7100
|
-
*
|
|
7094
|
+
* Output spend that bought answers cut off mid-generation — the one slice of
|
|
7095
|
+
* a bill that is waste without a counterpart. Paid in full, frequently
|
|
7096
|
+
* retried and billed again, and the truncated attempt bought nothing.
|
|
7097
|
+
*
|
|
7098
|
+
* Three states, kept apart on purpose: waste found, none found on a log that
|
|
7099
|
+
* measured, and a log that never recorded a stop reason at all — which gets
|
|
7100
|
+
* the missing-field message, because silence there would read as a clean bill
|
|
7101
|
+
* of health on a question the log never asked.
|
|
7102
|
+
*/
|
|
7103
|
+
if (report.total.truncatedCalls > 0 && report.total.outputUsd > 0) {
|
|
7104
|
+
console.log();
|
|
7105
|
+
console.log(` ${c.yellow('!')} ${c.bold(wrap(t.profile.truncatedWaste(t.profile.calls(report.total.truncatedCalls), formatUsd(report.total.truncatedOutputUsd), pct(report.total.truncatedOutputUsd / report.total.outputUsd)), 74, ' '))}`);
|
|
7106
|
+
/**
|
|
7107
|
+
* Which workloads are paying for it, and at what rate — the actionable
|
|
7108
|
+
* half the total hides. A 40% truncation rate is a max_tokens setting
|
|
7109
|
+
* that is simply wrong; 1% is a long tail, and the two call for opposite
|
|
7110
|
+
* responses.
|
|
7111
|
+
*
|
|
7112
|
+
* The rate is over calls that **recorded a stop reason**, never over all
|
|
7113
|
+
* calls: a workload that logs the field on half its traffic must not be
|
|
7114
|
+
* reported as though the unmeasured half completed. Both numbers print,
|
|
7115
|
+
* so the denominator is visible rather than implied.
|
|
7116
|
+
*/
|
|
7117
|
+
const truncatedLabels = report.byLabel
|
|
7118
|
+
.filter((row) => row.breakdown.truncatedCalls > 0)
|
|
7119
|
+
.sort((a, b) => b.breakdown.truncatedOutputUsd - a.breakdown.truncatedOutputUsd);
|
|
7120
|
+
if (truncatedLabels.length > 0 && report.byLabel.length > 1) {
|
|
7121
|
+
for (const row of truncatedLabels.slice(0, 3)) {
|
|
7122
|
+
const name = row.label === UNLABELLED ? t.profile.unlabelled() : row.label;
|
|
7123
|
+
console.log(` ${c.dim(wrap(t.profile.truncatedBy(name, n(row.breakdown.truncatedCalls), n(row.breakdown.stopReasonCalls), pct(row.breakdown.truncatedCalls / row.breakdown.stopReasonCalls), formatUsd(row.breakdown.truncatedOutputUsd)), 74, ' '))}`);
|
|
7124
|
+
}
|
|
7125
|
+
}
|
|
7126
|
+
/**
|
|
7127
|
+
* The ceiling the completed answers actually needed, when the output
|
|
7128
|
+
* shapes measured it: "95% of the answers that finished fit within N
|
|
7129
|
+
* tokens" is the number a max_tokens cap wants, and it sits next to the
|
|
7130
|
+
* evidence that the current cap is too low. Measured on these calls,
|
|
7131
|
+
* promised for nothing.
|
|
7132
|
+
*/
|
|
7133
|
+
const ceiling = report.outputShapes.find((shape) => shape.p95WithinTokens !== null);
|
|
7134
|
+
if (ceiling !== undefined) {
|
|
7135
|
+
console.log(` ${c.dim(wrap(t.profile.truncatedCeiling(n(ceiling.p95WithinTokens)), 74, ' '))}`);
|
|
7136
|
+
}
|
|
7137
|
+
/**
|
|
7138
|
+
* The "billed again" half, measured. The sentence above has always said
|
|
7139
|
+
* truncated answers are frequently retried; this is the count and the
|
|
7140
|
+
* money, when the log can carry it — a pattern, never a certainty, and
|
|
7141
|
+
* attributed to the truncated call's slice, where the ceiling that
|
|
7142
|
+
* caused it lives.
|
|
7143
|
+
*/
|
|
7144
|
+
for (const row of report.truncationRetries.slice(0, 3)) {
|
|
7145
|
+
const name = row.label === UNLABELLED ? t.profile.unlabelled() : row.label;
|
|
7146
|
+
console.log(` ${c.yellow('!')} ${c.bold(wrap(t.profile.truncationRetryLine(name, row.modelName, n(row.retried), n(row.truncatedCalls), n(Math.round(row.withinMs / 1000)), formatUsd(row.wastedUsd), formatUsd(row.retryUsd)), 74, ' '))}`);
|
|
7147
|
+
}
|
|
7148
|
+
if (report.truncationRetries.length > 0) {
|
|
7149
|
+
console.log(` ${c.dim(wrap(t.profile.truncationRetryNote(), 74, ' '))}`);
|
|
7150
|
+
}
|
|
7151
|
+
}
|
|
7152
|
+
else if (report.total.stopReasonCalls === 0) {
|
|
7153
|
+
console.log();
|
|
7154
|
+
console.log(` ${c.dim(wrap(t.profile.truncatedNotRecorded(), 74, ' '))}`);
|
|
7155
|
+
}
|
|
7156
|
+
}
|
|
7157
|
+
/**
|
|
7158
|
+
* The mix moving inside one log: the drift `--against` needs two files to see.
|
|
7159
|
+
*
|
|
7160
|
+
* Spoken only past fifteen points, and hedged in the copy while the data states
|
|
7161
|
+
* exact shares, because a mix that moved is a question rather than a finding.
|
|
7162
|
+
*/
|
|
7163
|
+
function reportMixDrift(report, t) {
|
|
7164
|
+
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
7165
|
+
const pct = (share) => `${(share * 100).toFixed(1)}%`;
|
|
7166
|
+
/**
|
|
7167
|
+
* The mix moving inside one log — the drift `--against` needs a second log
|
|
7168
|
+
* to see. Spoken only past fifteen points, a presentation threshold stated
|
|
7169
|
+
* in the copy; the data states exact shares and the JSON carries them all.
|
|
7170
|
+
*/
|
|
7171
|
+
if (report.modelMixDrift !== null) {
|
|
7172
|
+
const moved = report.modelMixDrift.models.filter((m) => Math.abs(m.lastShare - m.firstShare) >= 0.15);
|
|
7173
|
+
if (moved.length > 0) {
|
|
7174
|
+
console.log();
|
|
7175
|
+
console.log(sectionHeading(t.profile.mixDriftHeading()));
|
|
7176
|
+
for (const m of moved.slice(0, 3)) {
|
|
7177
|
+
console.log();
|
|
7178
|
+
console.log(` ${c.yellow('!')} ${c.bold(wrap(t.profile.mixDriftLine(m.model, pct(m.firstShare), pct(m.lastShare), n(report.modelMixDrift.firstDays), n(report.modelMixDrift.lastDays), formatUsd(m.lastUsd)), 74, ' '))}`);
|
|
7179
|
+
}
|
|
7180
|
+
console.log(` ${c.dim(wrap(t.profile.mixDriftNote(), 74, ' '))}`);
|
|
7181
|
+
}
|
|
7182
|
+
}
|
|
7183
|
+
}
|
|
7184
|
+
/**
|
|
7185
|
+
* How close the largest call is to the model's ceiling.
|
|
7186
|
+
*
|
|
7187
|
+
* A window is a wall, not a budget: the call that hits it fails rather than costing
|
|
7188
|
+
* more, so this is a different warning from every dollar above it and is kept a
|
|
7189
|
+
* different shape.
|
|
7190
|
+
*/
|
|
7191
|
+
function reportContextPressure(pressures, t) {
|
|
7192
|
+
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
7193
|
+
const pct = (share) => `${(share * 100).toFixed(1)}%`;
|
|
7194
|
+
/**
|
|
7195
|
+
* How close the largest call is to the model's ceiling.
|
|
7196
|
+
*
|
|
7197
|
+
* The failure a bill cannot show: input grows turn by turn or document by
|
|
7198
|
+
* document, costs nothing extra to grow, and then one call crosses the
|
|
7199
|
+
* context window and the API refuses it. Loud from 85% — close enough
|
|
7200
|
+
* that the next retrieval bump or long conversation plausibly crosses —
|
|
7201
|
+
* and quiet above half, so the reader sees it coming either way. No
|
|
7202
|
+
* prediction of *when*: a straight line through two points is a guess
|
|
7203
|
+
* wearing arithmetic's clothes.
|
|
7204
|
+
*/
|
|
7205
|
+
{
|
|
7206
|
+
if (pressures.length > 0) {
|
|
7207
|
+
console.log();
|
|
7208
|
+
console.log(sectionHeading(t.profile.pressureHeading()));
|
|
7209
|
+
for (const row of pressures.slice(0, 3)) {
|
|
7210
|
+
const label = row.label === UNLABELLED ? t.profile.unlabelled() : row.label;
|
|
7211
|
+
const line = t.profile.pressureLine(label, row.modelName, n(row.maxCallInputTokens), n(row.contextWindow), pct(row.share));
|
|
7212
|
+
console.log();
|
|
7213
|
+
if (row.share >= 0.85) {
|
|
7214
|
+
console.log(` ${c.yellow('!')} ${c.bold(wrap(line, 74, ' '))}`);
|
|
7215
|
+
console.log(` ${c.dim(wrap(t.profile.pressureAdvice(), 74, ' '))}`);
|
|
7216
|
+
}
|
|
7217
|
+
else {
|
|
7218
|
+
console.log(` ${c.dim(wrap(line, 74, ' '))}`);
|
|
7219
|
+
}
|
|
7220
|
+
}
|
|
7221
|
+
}
|
|
7222
|
+
}
|
|
7223
|
+
}
|
|
7224
|
+
/**
|
|
7225
|
+
* How big the calls themselves are, which is the other half of the bill.
|
|
7226
|
+
*
|
|
7227
|
+
* The totals say what was spent; this says whether it went on many small calls or a
|
|
7228
|
+
* few enormous ones, and those two bills are fixed by different things.
|
|
7229
|
+
*/
|
|
7230
|
+
function reportInputShape(report, t) {
|
|
7231
|
+
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
7232
|
+
const pct = (share) => `${(share * 100).toFixed(1)}%`;
|
|
7233
|
+
/**
|
|
7234
|
+
* How big the calls themselves are — the other half of the bill.
|
|
7235
|
+
*
|
|
7236
|
+
* The section above describes output; on a RAG or agent workload input is
|
|
7237
|
+
* most of the invoice, and a total could only ever say "input is 63% of
|
|
7238
|
+
* this bill", which nobody can act on. The actionable question is whether
|
|
7239
|
+
* the ordinary call is large or a few calls are enormous, and those two
|
|
7240
|
+
* shapes want opposite responses: a cap on something, or a shorter prompt.
|
|
7241
|
+
*
|
|
7242
|
+
* Loud past **four times** the median — far enough from an even
|
|
7243
|
+
* distribution that a bucket boundary cannot flip the message, and stated
|
|
7244
|
+
* in the sentence rather than hidden here. Both figures are bucket
|
|
7245
|
+
* ceilings, so the ratio is coarse by construction and the copy says so.
|
|
7246
|
+
*/
|
|
7247
|
+
if (report.inputShapes.length > 0) {
|
|
7248
|
+
console.log();
|
|
7249
|
+
console.log(sectionHeading(t.profile.inputShapeHeading()));
|
|
7250
|
+
for (const shape of report.inputShapes.slice(0, 3)) {
|
|
7251
|
+
const label = shape.label === UNLABELLED ? t.profile.unlabelled() : shape.label;
|
|
7252
|
+
console.log();
|
|
7253
|
+
if (shape.medianWithinTokens === null || shape.p95WithinTokens === null || shape.p95OverMedian === null) {
|
|
7254
|
+
/**
|
|
7255
|
+
* The covering bucket is the open-ended last one, so there is no
|
|
7256
|
+
* ceiling to name. Said rather than skipped: a slice whose calls are
|
|
7257
|
+
* all above a million tokens is a finding, and silence would drop it.
|
|
7258
|
+
*/
|
|
7259
|
+
console.log(` ${c.bold(wrap(t.profile.inputHuge(label, shape.modelName, t.profile.calls(shape.calls), formatUsd(shape.inputUsd)), 74, ' '))}`);
|
|
7260
|
+
continue;
|
|
7261
|
+
}
|
|
7262
|
+
const skewed = shape.p95OverMedian >= 4;
|
|
7263
|
+
const line = skewed
|
|
7264
|
+
? t.profile.inputSkewed(label, shape.modelName, n(shape.medianWithinTokens), n(shape.p95WithinTokens), shape.p95OverMedian.toFixed(1), formatUsd(shape.inputUsd))
|
|
7265
|
+
: t.profile.inputEven(label, shape.modelName, n(shape.medianWithinTokens), n(shape.p95WithinTokens), formatUsd(shape.inputUsd));
|
|
7266
|
+
console.log(` ${c.bold(wrap(line, 74, ' '))}`);
|
|
7267
|
+
console.log(` ${c.dim(wrap(skewed ? t.profile.inputSkewedAdvice() : t.profile.inputEvenAdvice(), 74, ' '))}`);
|
|
7268
|
+
/**
|
|
7269
|
+
* What that size actually costs. A cache read is a tenth of input on
|
|
7270
|
+
* Anthropic, so a large slice reading almost everything from cache is a
|
|
7271
|
+
* very different bill from one paying full rate — and the token counts
|
|
7272
|
+
* alone cannot tell them apart.
|
|
7273
|
+
*/
|
|
7274
|
+
if (shape.cachedShare >= 0.5) {
|
|
7275
|
+
console.log(` ${c.dim(wrap(t.profile.inputMostlyCached(pct(shape.cachedShare)), 74, ' '))}`);
|
|
7276
|
+
}
|
|
7277
|
+
else if (shape.cachedShare < 0.1) {
|
|
7278
|
+
console.log(` ${c.dim(wrap(t.profile.inputFullRate(), 74, ' '))}`);
|
|
7279
|
+
}
|
|
7280
|
+
}
|
|
7281
|
+
}
|
|
7282
|
+
}
|
|
7283
|
+
/**
|
|
7284
|
+
* Where the output spend concentrates.
|
|
7285
|
+
*
|
|
7286
|
+
* Output is usually most of the bill, so the shape of it is usually the actionable
|
|
7287
|
+
* half of the report rather than a footnote to the totals.
|
|
7288
|
+
*/
|
|
7289
|
+
function reportOutputShape(report, t) {
|
|
7290
|
+
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
7291
|
+
const pct = (share) => `${(share * 100).toFixed(1)}%`;
|
|
7292
|
+
/**
|
|
7293
|
+
* Where the output spend concentrates — the actionable half of "output
|
|
7294
|
+
* dominates", which the headline above could only state as a total.
|
|
7101
7295
|
*
|
|
7102
|
-
*
|
|
7103
|
-
*
|
|
7104
|
-
*
|
|
7105
|
-
*
|
|
7106
|
-
*
|
|
7107
|
-
*
|
|
7108
|
-
*
|
|
7296
|
+
* Two bills with identical output spend want opposite responses. Six per cent
|
|
7297
|
+
* of calls holding half of it is a tail, and a tail has a cause worth a
|
|
7298
|
+
* morning; forty-five per cent is what evenly spread looks like, and the only
|
|
7299
|
+
* lever there is asking every answer to be shorter. The threshold between the
|
|
7300
|
+
* two is a quarter of the calls — far enough from both shapes that rounding
|
|
7301
|
+
* cannot flip the message, and stated here because it is a presentation choice,
|
|
7302
|
+
* not a measurement.
|
|
7109
7303
|
*/
|
|
7304
|
+
if (report.outputShapes.length > 0) {
|
|
7305
|
+
console.log();
|
|
7306
|
+
console.log(sectionHeading(t.profile.outputShapeHeading()));
|
|
7307
|
+
for (const shape of report.outputShapes.slice(0, 3)) {
|
|
7308
|
+
const label = shape.label === UNLABELLED ? t.profile.unlabelled() : shape.label;
|
|
7309
|
+
const isTail = shape.heavyCallShare < 0.25;
|
|
7310
|
+
console.log();
|
|
7311
|
+
if (isTail) {
|
|
7312
|
+
console.log(` ${c.bold(wrap(t.profile.outputTail(label, shape.modelName, pct(shape.heavyCallShare), pct(shape.heavySpendShare), n(shape.aboveTokens), formatUsd(shape.outputUsd)), 74, ' '))}`);
|
|
7313
|
+
console.log(` ${c.dim(wrap(t.profile.outputTailAdvice(), 74, ' '))}`);
|
|
7314
|
+
}
|
|
7315
|
+
else {
|
|
7316
|
+
console.log(` ${c.bold(wrap(t.profile.outputFlat(label, shape.modelName, pct(shape.heavyCallShare), pct(shape.heavySpendShare), formatUsd(shape.outputUsd)), 74, ' '))}`);
|
|
7317
|
+
console.log(` ${c.dim(wrap(t.profile.outputFlatAdvice(), 74, ' '))}`);
|
|
7318
|
+
}
|
|
7319
|
+
/**
|
|
7320
|
+
* The ceilings a max_tokens cap actually wants, exact over the
|
|
7321
|
+
* histogram: every measured answer at or under the named number is
|
|
7322
|
+
* counted, none interpolated. Omitted when the covering bucket is the
|
|
7323
|
+
* open-ended last one, which has no ceiling to name honestly.
|
|
7324
|
+
*/
|
|
7325
|
+
if (shape.medianWithinTokens !== null && shape.p95WithinTokens !== null) {
|
|
7326
|
+
console.log(` ${c.dim(wrap(t.profile.outputPercentiles(n(shape.medianWithinTokens), n(shape.p95WithinTokens)), 74, ' '))}`);
|
|
7327
|
+
}
|
|
7328
|
+
}
|
|
7329
|
+
}
|
|
7330
|
+
}
|
|
7331
|
+
/**
|
|
7332
|
+
* What re-sending the conversation costs.
|
|
7333
|
+
*
|
|
7334
|
+
* A chat or agent workload sends the whole conversation every turn, so the history
|
|
7335
|
+
* is paid for again on each call. It is one of the four levers with real money on
|
|
7336
|
+
* it and the one least likely to be noticed, because nothing in the bill is
|
|
7337
|
+
* labelled "history".
|
|
7338
|
+
*/
|
|
7339
|
+
function reportConversationHistory(report, t) {
|
|
7340
|
+
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
7341
|
+
const pct = (share) => `${(share * 100).toFixed(1)}%`;
|
|
7110
7342
|
/**
|
|
7111
|
-
*
|
|
7343
|
+
* What re-sending the conversation costs — the line nothing here could see.
|
|
7112
7344
|
*
|
|
7113
|
-
*
|
|
7114
|
-
*
|
|
7115
|
-
* on
|
|
7116
|
-
* the
|
|
7117
|
-
*/
|
|
7118
|
-
const lostLabels = report.byLabel
|
|
7119
|
-
.map((r) => ({ row: r, cache: cacheEconomics(r.breakdown) }))
|
|
7120
|
-
.filter((r) => r.cache.verdict === 'lost-money')
|
|
7121
|
-
.sort((a, b) => b.cache.deltaUsd - a.cache.deltaUsd);
|
|
7122
|
-
const NAMED = 3;
|
|
7123
|
-
const nameOf = (row) => row.label === UNLABELLED ? t.profile.unlabelled() : row.label;
|
|
7124
|
-
/**
|
|
7125
|
-
* The names, with the ones that did not fit **counted rather than dropped**.
|
|
7345
|
+
* A chat or agent workload sends the whole conversation back on every turn, so
|
|
7346
|
+
* the input grows with the turn count and that growth is routinely the largest
|
|
7347
|
+
* item on the bill. A prompt file shows the system prompt and not the history; a
|
|
7348
|
+
* total shows the sum and not the shape.
|
|
7126
7349
|
*
|
|
7127
|
-
*
|
|
7128
|
-
*
|
|
7129
|
-
*
|
|
7130
|
-
*
|
|
7131
|
-
* catching itself at, and `reportProfileGaps` already had the pattern.
|
|
7350
|
+
* Reported as a **ceiling**, because part of the growth is the user's own new
|
|
7351
|
+
* messages and this reads counts rather than content, so it cannot separate the
|
|
7352
|
+
* two. Saying nothing because the exact split is unknowable would be worse: the
|
|
7353
|
+
* bound is exact, and the reader can act on it.
|
|
7132
7354
|
*/
|
|
7133
|
-
|
|
7134
|
-
|
|
7135
|
-
|
|
7136
|
-
|
|
7137
|
-
|
|
7138
|
-
|
|
7139
|
-
|
|
7140
|
-
|
|
7355
|
+
if (report.conversations.length > 0) {
|
|
7356
|
+
console.log();
|
|
7357
|
+
console.log(sectionHeading(t.profile.historyHeading()));
|
|
7358
|
+
for (const growth of report.conversations.slice(0, 3)) {
|
|
7359
|
+
const label = growth.label === UNLABELLED ? t.profile.unlabelled() : growth.label;
|
|
7360
|
+
console.log();
|
|
7361
|
+
console.log(` ${c.bold(wrap(t.profile.historyGrowth(label, growth.modelName, n(Math.round(growth.minTurnTokens)), n(Math.round(growth.maxTurnTokens)), n(growth.longestSession)), 74, ' '))}`);
|
|
7362
|
+
console.log(` ${c.dim(wrap(t.profile.historyCeiling(formatUsd(growth.growthUsd), pct(growth.shareOfBill), formatUsd(growth.flatUsd), formatUsd(growth.inputUsd)), 74, ' '))}`);
|
|
7363
|
+
}
|
|
7364
|
+
}
|
|
7365
|
+
else if (!report.hasSessions) {
|
|
7366
|
+
/**
|
|
7367
|
+
* Not the same as "no growth". A log without a session field cannot be asked
|
|
7368
|
+
* the question at all, and silence there would read as a clean bill of health
|
|
7369
|
+
* on the line most likely to be the biggest.
|
|
7370
|
+
*/
|
|
7371
|
+
console.log();
|
|
7372
|
+
console.log(sectionHeading(t.profile.historyHeading()));
|
|
7373
|
+
console.log(` ${c.dim(wrap(t.profile.historyNoSessions(), 74, ' '))}`);
|
|
7374
|
+
}
|
|
7375
|
+
}
|
|
7376
|
+
/**
|
|
7377
|
+
* `--what-if <model>`: these exact calls, at another model's prices.
|
|
7378
|
+
*
|
|
7379
|
+
* Priced from the tokens that were actually billed rather than from an estimate,
|
|
7380
|
+
* which is the only version of this question worth answering: a migration priced
|
|
7381
|
+
* off a guess is a forecast, and this product does not forecast.
|
|
7382
|
+
*/
|
|
7383
|
+
function reportWhatIf(whatIf, report, t) {
|
|
7384
|
+
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
7385
|
+
const pct = (share) => `${(share * 100).toFixed(1)}%`;
|
|
7141
7386
|
/**
|
|
7142
|
-
*
|
|
7387
|
+
* `--what-if <model>`: these exact calls, at another model's rates.
|
|
7143
7388
|
*
|
|
7144
|
-
*
|
|
7145
|
-
*
|
|
7146
|
-
*
|
|
7147
|
-
*
|
|
7148
|
-
*
|
|
7389
|
+
* The levers above pick their own candidate; this answers the question the
|
|
7390
|
+
* reader arrived with. It is multiplication, not advice, and every part of
|
|
7391
|
+
* this section is built so it cannot be read as advice:
|
|
7392
|
+
*
|
|
7393
|
+
* - the caveat line prints **before** the figure, not after it;
|
|
7394
|
+
* - calls the target's context window could not have accepted are named as
|
|
7395
|
+
* impossible rather than priced as cheap, and their money is in none of
|
|
7396
|
+
* the totals;
|
|
7397
|
+
* - spend already on the target is stated separately, because a difference
|
|
7398
|
+
* computed over money that cannot move is a percentage of the wrong
|
|
7399
|
+
* denominator.
|
|
7149
7400
|
*/
|
|
7150
|
-
|
|
7151
|
-
|
|
7152
|
-
console.log(
|
|
7153
|
-
|
|
7154
|
-
|
|
7155
|
-
|
|
7156
|
-
|
|
7157
|
-
if (lostLabels.length > 0 && report.byLabel.length > 1) {
|
|
7158
|
-
console.log(` ${c.dim(wrap(t.profile.cacheLostBy(namedLosers), 74, ' '))}`);
|
|
7401
|
+
if (whatIf !== null) {
|
|
7402
|
+
console.log();
|
|
7403
|
+
console.log(sectionHeading(t.profile.whatIfHeading(whatIf.target.displayName)));
|
|
7404
|
+
console.log(` ${c.dim(wrap(t.profile.whatIfAssumption(), 74, ' '))}`);
|
|
7405
|
+
console.log();
|
|
7406
|
+
if (whatIf.slices.length === 0) {
|
|
7407
|
+
console.log(` ${c.dim(wrap(t.profile.whatIfNothingToMove(), 74, ' '))}`);
|
|
7159
7408
|
}
|
|
7160
|
-
|
|
7161
|
-
|
|
7162
|
-
|
|
7163
|
-
console.log(` ${c.
|
|
7409
|
+
else {
|
|
7410
|
+
const cheaper = whatIf.deltaUsd < 0;
|
|
7411
|
+
const line = t.profile.whatIfTotal(formatUsd(whatIf.currentUsd), formatUsd(whatIf.targetUsd), formatUsd(Math.abs(whatIf.deltaUsd)));
|
|
7412
|
+
console.log(` ${cheaper ? c.green('→') : c.yellow('!')} ${c.bold(wrap(line, 74, ' '))}`);
|
|
7413
|
+
console.log(` ${c.dim(wrap(cheaper ? t.profile.whatIfCheaper() : t.profile.whatIfDearer(), 74, ' '))}`);
|
|
7414
|
+
/**
|
|
7415
|
+
* The other half of the whole decision: the same move with the target's
|
|
7416
|
+
* Batch API on top. Computed against the target's rates, never by
|
|
7417
|
+
* adding two savings, and hedged the only honest way — whether these
|
|
7418
|
+
* calls can wait is not in the log.
|
|
7419
|
+
*/
|
|
7420
|
+
if (whatIf.batchOnTarget !== null) {
|
|
7421
|
+
console.log(` ${c.dim(wrap(t.profile.whatIfBatchOnTarget(formatUsd(whatIf.batchOnTarget.targetUsd), formatUsd(whatIf.targetUsd)), 74, ' '))}`);
|
|
7422
|
+
}
|
|
7423
|
+
for (const slice of whatIf.slices.slice(0, 5)) {
|
|
7424
|
+
const label = slice.label === UNLABELLED ? t.profile.unlabelled() : slice.label;
|
|
7425
|
+
console.log(` ${c.dim('·')} ${c.dim(wrap(t.profile.whatIfSlice(label, slice.model, formatUsd(slice.currentUsd), formatUsd(slice.targetUsd)), 74, ' '))}`);
|
|
7426
|
+
/**
|
|
7427
|
+
* Cache traffic the target could not grant, said loudly and beside
|
|
7428
|
+
* the figure it corrects. The standard row prices cache entries the
|
|
7429
|
+
* target's minimum would refuse to create — an error that flatters
|
|
7430
|
+
* the move, which is the direction this repository refuses.
|
|
7431
|
+
*/
|
|
7432
|
+
if (slice.cacheBeyondTarget !== null) {
|
|
7433
|
+
console.log(` ${c.yellow('!')} ${c.dim(wrap(t.profile.whatIfCacheBeyond(n(slice.maxCallInputTokens), n(slice.cacheBeyondTarget.minTokens), formatUsd(slice.cacheBeyondTarget.noCacheUsd)), 74, ' '))}`);
|
|
7434
|
+
}
|
|
7435
|
+
}
|
|
7164
7436
|
}
|
|
7165
|
-
|
|
7166
|
-
|
|
7437
|
+
/**
|
|
7438
|
+
* The refusal, and it is loud. A call larger than the target's window is
|
|
7439
|
+
* not a cheaper call, and a comparison that priced it anyway would report
|
|
7440
|
+
* a saving for traffic that would have failed outright.
|
|
7441
|
+
*/
|
|
7442
|
+
for (const slice of whatIf.overContext.slice(0, 3)) {
|
|
7443
|
+
const label = slice.label === UNLABELLED ? t.profile.unlabelled() : slice.label;
|
|
7444
|
+
console.log(` ${c.yellow('!')} ${c.bold(wrap(t.profile.whatIfOverContext(label, n(slice.maxCallInputTokens), n(whatIf.target.contextWindow), formatUsd(slice.currentUsd)), 74, ' '))}`);
|
|
7445
|
+
}
|
|
7446
|
+
// Money that is already there cannot move, and leaving it out of the
|
|
7447
|
+
// totals above is only honest if the reader is told it exists.
|
|
7448
|
+
if (whatIf.alreadyOnTarget.calls > 0) {
|
|
7449
|
+
console.log(` ${c.dim(wrap(t.profile.whatIfAlreadyThere(t.profile.calls(whatIf.alreadyOnTarget.calls), formatUsd(whatIf.alreadyOnTarget.usd)), 74, ' '))}`);
|
|
7450
|
+
}
|
|
7451
|
+
// Models with no current price have no difference to state — their target
|
|
7452
|
+
// cost is knowable and the subtraction is not.
|
|
7453
|
+
if (whatIf.unpricedCalls > 0) {
|
|
7454
|
+
console.log(` ${c.dim(wrap(t.profile.whatIfUnpriced(t.profile.calls(whatIf.unpricedCalls), whatIf.unpricedModels.join(', ')), 74, ' '))}`);
|
|
7167
7455
|
}
|
|
7168
7456
|
}
|
|
7457
|
+
}
|
|
7458
|
+
/**
|
|
7459
|
+
* The section this command exists for: what would actually move this bill.
|
|
7460
|
+
*
|
|
7461
|
+
* Shortening the prompt is the smallest line item there is. Which model a call goes
|
|
7462
|
+
* to, the Batch API, caching and re-sent conversation history are the levers with
|
|
7463
|
+
* real money on them, and all four are priced here from what the provider charged
|
|
7464
|
+
* rather than from what a file would cost.
|
|
7465
|
+
*/
|
|
7466
|
+
function reportLevers(report, pricing, t) {
|
|
7467
|
+
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
7468
|
+
const pct = (share) => `${(share * 100).toFixed(1)}%`;
|
|
7169
7469
|
/**
|
|
7170
|
-
*
|
|
7470
|
+
* The section this command is for, and the answer to the fairest complaint the
|
|
7471
|
+
* product has had: on a bill of twenty thousand, the rules recover two hundred.
|
|
7171
7472
|
*
|
|
7172
|
-
*
|
|
7173
|
-
*
|
|
7174
|
-
*
|
|
7473
|
+
* That figure is right — measured, three tokens out of three hundred and six on
|
|
7474
|
+
* an ordinary support prompt. The conclusion is not that the tool is worthless
|
|
7475
|
+
* but that it had been looking at the smallest line item. Which model a call
|
|
7476
|
+
* goes to moves 60% to 80%. The Batch API moves 50% flat. Both are priced here
|
|
7477
|
+
* from the reader's own tokens, at published rates, with no modelling in
|
|
7478
|
+
* between — and printed above the breakdowns, because a lever nobody scrolls to
|
|
7479
|
+
* is a lever nobody pulls.
|
|
7175
7480
|
*
|
|
7176
|
-
* The
|
|
7177
|
-
*
|
|
7178
|
-
*
|
|
7179
|
-
*
|
|
7481
|
+
* The ceiling on prompt shortening prints underneath them on purpose. A 1% win
|
|
7482
|
+
* reported without saying 1% of what is not information, and this repository
|
|
7483
|
+
* would rather say the uncomfortable number itself than let somebody else
|
|
7484
|
+
* discover it.
|
|
7180
7485
|
*/
|
|
7181
|
-
|
|
7182
|
-
|
|
7183
|
-
|
|
7486
|
+
const levers = billLevers(report, { catalogue: pricing });
|
|
7487
|
+
console.log();
|
|
7488
|
+
console.log(sectionHeading(t.profile.leversHeading()));
|
|
7184
7489
|
/**
|
|
7185
|
-
*
|
|
7490
|
+
* Every lever below describes a mixture when nothing carries a label.
|
|
7186
7491
|
*
|
|
7187
|
-
*
|
|
7188
|
-
*
|
|
7189
|
-
*
|
|
7190
|
-
*
|
|
7191
|
-
*
|
|
7492
|
+
* A 2,000-call classifier and a 400-call RAG pipeline merge into one slice, and
|
|
7493
|
+
* the section then offers a single route for two workloads that need different
|
|
7494
|
+
* answers — and `trazum route` would measure one prompt against a figure
|
|
7495
|
+
* covering both. The session case already tells the reader to add the field;
|
|
7496
|
+
* this one named the row `unlabelled` and said nothing, as though that were a
|
|
7497
|
+
* workload.
|
|
7192
7498
|
*/
|
|
7193
|
-
const
|
|
7194
|
-
|
|
7195
|
-
.
|
|
7196
|
-
|
|
7197
|
-
|
|
7499
|
+
const unlabelledOnly = report.byLabel.length === 1 && report.byLabel[0].label === UNLABELLED;
|
|
7500
|
+
if (unlabelledOnly && levers.slices.length > 0) {
|
|
7501
|
+
console.log(` ${c.yellow('!')} ${c.dim(wrap(t.profile.leversUnlabelled(), 74, ' '))}`);
|
|
7502
|
+
}
|
|
7503
|
+
if (levers.slices.length === 0) {
|
|
7504
|
+
console.log(` ${c.dim(wrap(t.profile.leversNone(), 74, ' '))}`);
|
|
7505
|
+
}
|
|
7506
|
+
else {
|
|
7507
|
+
for (const slice of levers.slices.slice(0, 5)) {
|
|
7508
|
+
const label = slice.label === UNLABELLED ? t.profile.unlabelled() : slice.label;
|
|
7509
|
+
console.log();
|
|
7510
|
+
/**
|
|
7511
|
+
* The headline is the **combined** figure, and the options underneath are
|
|
7512
|
+
* the ways to reach it — not rows to add up. Batching a routed call
|
|
7513
|
+
* discounts the cheaper model's price, so listing them separately printed
|
|
7514
|
+
* $12.60 and $10.50 against a slice that had spent $21.00: a saving larger
|
|
7515
|
+
* than the bill it came from, in the flattering direction.
|
|
7516
|
+
*/
|
|
7517
|
+
console.log(` ${c.green('→')} ${c.bold(wrap(t.profile.leverSlice(label, slice.modelName, formatUsd(slice.combinedUsd), pct(slice.shareOfBill)), 74, ' '))}`);
|
|
7518
|
+
console.log(` ${c.dim(t.profile.leverCalls(t.profile.calls(slice.calls), formatUsd(slice.spentUsd)))}`);
|
|
7519
|
+
if (slice.route) {
|
|
7520
|
+
console.log(` ${c.dim('·')} ${c.dim(wrap(t.profile.leverRoute(slice.route.candidate.displayName, formatUsd(slice.route.savingUsd)), 74, ' '))}`);
|
|
7521
|
+
}
|
|
7522
|
+
if (slice.batch) {
|
|
7523
|
+
console.log(` ${c.dim('·')} ${c.dim(wrap(t.profile.leverBatch(formatUsd(slice.batch.savingUsd)), 74, ' '))}`);
|
|
7524
|
+
}
|
|
7525
|
+
// The arithmetic is exact and the quality question is untouched by it.
|
|
7526
|
+
// Naming the command is the difference between a saving and a gamble.
|
|
7527
|
+
if (slice.route) {
|
|
7528
|
+
console.log(` ${c.dim(wrap(t.profile.leverRouteVerify(slice.route.candidate.id), 74, ' '))}`);
|
|
7529
|
+
}
|
|
7530
|
+
}
|
|
7198
7531
|
}
|
|
7532
|
+
console.log();
|
|
7533
|
+
console.log(` ${c.dim(wrap(t.profile.leverPromptCeiling(formatUsd(levers.promptCeilingUsd), pct(levers.promptCeilingShare)), 74, ' '))}`);
|
|
7534
|
+
}
|
|
7535
|
+
/**
|
|
7536
|
+
* What the labels, the budgets and the cache TTLs say about this bill.
|
|
7537
|
+
*
|
|
7538
|
+
* Why a label's caching is failing, read from the prompt file itself: the log
|
|
7539
|
+
* carries counts, so `profile` can say *that* caching loses money on a label and
|
|
7540
|
+
* nothing more, and `labels` in the config maps a label to the file it sends.
|
|
7541
|
+
* Every sentence carries "as it is today", because the file is whatever the
|
|
7542
|
+
* repository holds now and may not be what produced the log.
|
|
7543
|
+
*
|
|
7544
|
+
* Then the budgets, the TTL fits, the single-turn writes and the shape of the
|
|
7545
|
+
* sessions: two hundred lines of one topic, what the configuration and the
|
|
7546
|
+
* conversation did to the money.
|
|
7547
|
+
*/
|
|
7548
|
+
async function reportLabelsBudgetsAndCache(report, config, pricing, t) {
|
|
7549
|
+
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
7550
|
+
const pct = (share) => `${(share * 100).toFixed(1)}%`;
|
|
7199
7551
|
/**
|
|
7200
7552
|
* Why, read from the prompt file itself — the loop `profile` could not close.
|
|
7201
7553
|
*
|
|
@@ -7389,664 +7741,794 @@ async function commandProfile(args, config, configDir, pricing, t) {
|
|
|
7389
7741
|
if (report.total.assumedWriteTtlCalls > 0) {
|
|
7390
7742
|
console.log(` ${c.dim(wrap(t.profile.assumedWriteTtl(report.total.assumedWriteTtlCalls), 74, ' '))}`);
|
|
7391
7743
|
}
|
|
7744
|
+
}
|
|
7745
|
+
/**
|
|
7746
|
+
* What caching did to this bill, and what it is about to do.
|
|
7747
|
+
*
|
|
7748
|
+
* The hit rate, the money the cache made or lost, the labels bleeding on it, and
|
|
7749
|
+
* the TTL question the hit rate cannot answer. One topic, and it needs the report
|
|
7750
|
+
* and the messages: it was a hundred and thirty lines inside a function with
|
|
7751
|
+
* twenty-three values in scope, and the only way to learn it used two of them was
|
|
7752
|
+
* to read it.
|
|
7753
|
+
*/
|
|
7754
|
+
function reportCacheVerdict(report, t) {
|
|
7755
|
+
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
7756
|
+
const pct = (share) => `${(share * 100).toFixed(1)}%`;
|
|
7392
7757
|
/**
|
|
7393
|
-
* The
|
|
7394
|
-
* product has had: on a bill of twenty thousand, the rules recover two hundred.
|
|
7758
|
+
* The hit rate, and then the question the hit rate does not answer.
|
|
7395
7759
|
*
|
|
7396
|
-
*
|
|
7397
|
-
*
|
|
7398
|
-
*
|
|
7399
|
-
*
|
|
7400
|
-
*
|
|
7401
|
-
|
|
7402
|
-
|
|
7760
|
+
* `cacheNever()` is keyed off the **verdict**, not off a null hit rate. Those
|
|
7761
|
+
* two came apart on a log whose calls were entirely cache writes with no plain
|
|
7762
|
+
* input: the rate is undefined there — zero reads over zero attempts — while
|
|
7763
|
+
* caching was plainly in use, and the old branch printed "caching was never
|
|
7764
|
+
* used" over a bill made of cache writes.
|
|
7765
|
+
*/
|
|
7766
|
+
const cache = cacheEconomics(report.total);
|
|
7767
|
+
const hitRate = cacheHitRate(report.total);
|
|
7768
|
+
if (cache.verdict === 'not-attempted') {
|
|
7769
|
+
console.log(` ${c.dim(wrap(t.profile.cacheNever(), 74, ' '))}`);
|
|
7770
|
+
}
|
|
7771
|
+
else if (hitRate !== null) {
|
|
7772
|
+
console.log(` ${c.dim(t.profile.cacheHit(pct(hitRate)))}`);
|
|
7773
|
+
}
|
|
7774
|
+
/**
|
|
7775
|
+
* Whether the caching was worth doing — the one finding here that can
|
|
7776
|
+
* contradict the advice Trazum gives everywhere else.
|
|
7403
7777
|
*
|
|
7404
|
-
*
|
|
7405
|
-
*
|
|
7406
|
-
*
|
|
7407
|
-
*
|
|
7778
|
+
* A cache write costs 1.25x plain input on Anthropic and 2x at the one-hour
|
|
7779
|
+
* TTL, so a prefix rebuilt faster than it is reused is billed at a premium and
|
|
7780
|
+
* returns nothing: that workload is cheaper with caching switched off. The
|
|
7781
|
+
* counterfactual is exact rather than a projection — caching changes the
|
|
7782
|
+
* multiplier on a token, never the token — so this is the one place in `profile`
|
|
7783
|
+
* where a comparison against what-might-have-been is arithmetic instead of a
|
|
7784
|
+
* guess about a prompt nobody wrote.
|
|
7408
7785
|
*/
|
|
7409
|
-
const levers = billLevers(report, { catalogue: pricing });
|
|
7410
|
-
console.log();
|
|
7411
|
-
console.log(sectionHeading(t.profile.leversHeading()));
|
|
7412
7786
|
/**
|
|
7413
|
-
*
|
|
7787
|
+
* The losing labels, **ranked by what caching cost them** and not by bill size.
|
|
7414
7788
|
*
|
|
7415
|
-
*
|
|
7416
|
-
* the
|
|
7417
|
-
*
|
|
7418
|
-
*
|
|
7419
|
-
* this one named the row `unlabelled` and said nothing, as though that were a
|
|
7420
|
-
* workload.
|
|
7789
|
+
* `byLabel` arrives sorted by total spend, which is the right order for the
|
|
7790
|
+
* table above and the wrong one here: the worst cache in an estate usually sits
|
|
7791
|
+
* on a small workload, so taking the first three off a spend-ordered list meant
|
|
7792
|
+
* the biggest loser could be the one that went unnamed.
|
|
7421
7793
|
*/
|
|
7422
|
-
const
|
|
7423
|
-
|
|
7424
|
-
|
|
7794
|
+
const lostLabels = report.byLabel
|
|
7795
|
+
.map((r) => ({ row: r, cache: cacheEconomics(r.breakdown) }))
|
|
7796
|
+
.filter((r) => r.cache.verdict === 'lost-money')
|
|
7797
|
+
.sort((a, b) => b.cache.deltaUsd - a.cache.deltaUsd);
|
|
7798
|
+
const NAMED = 3;
|
|
7799
|
+
const nameOf = (row) => row.label === UNLABELLED ? t.profile.unlabelled() : row.label;
|
|
7800
|
+
/**
|
|
7801
|
+
* The names, with the ones that did not fit **counted rather than dropped**.
|
|
7802
|
+
*
|
|
7803
|
+
* The first version sliced to three silently while the money beside it was
|
|
7804
|
+
* summed over every loser — so four bleeding labels printed three names and a
|
|
7805
|
+
* figure that charged them with a fourth label's loss. Truncating is fine;
|
|
7806
|
+
* truncating without saying so is the flattering omission this repository keeps
|
|
7807
|
+
* catching itself at, and `reportProfileGaps` already had the pattern.
|
|
7808
|
+
*/
|
|
7809
|
+
const listNames = (rows) => {
|
|
7810
|
+
const names = rows.slice(0, NAMED).map((r) => nameOf(r.row)).join(', ');
|
|
7811
|
+
return rows.length <= NAMED
|
|
7812
|
+
? names
|
|
7813
|
+
: `${names} ${t.profile.andMoreLabels(rows.length - NAMED)}`;
|
|
7814
|
+
};
|
|
7815
|
+
const namedLosers = listNames(lostLabels);
|
|
7816
|
+
const bleeding = lostLabels.reduce((sum, r) => sum + r.cache.deltaUsd, 0);
|
|
7817
|
+
/**
|
|
7818
|
+
* Whether the log can settle the question at all.
|
|
7819
|
+
*
|
|
7820
|
+
* Decided before anything prints, because it governs whether the confident
|
|
7821
|
+
* sentence prints — not merely whether a caveat follows it. The first attempt
|
|
7822
|
+
* added the caveat and left the assertion above it, so the reader met `Caching
|
|
7823
|
+
* took $0.1000 off this bill` and only afterwards learned it might be a $3.65
|
|
7824
|
+
* loss. A finding a later line retracts is still a finding somebody acted on.
|
|
7825
|
+
*/
|
|
7826
|
+
const unsettled = cache.worstCaseVerdict !== cache.verdict && report.total.assumedWriteTtlCalls > 0;
|
|
7827
|
+
if (unsettled) {
|
|
7828
|
+
console.log(` ${c.yellow('!')} ${c.bold(wrap(t.profile.cacheTtlUnsettled(report.total.assumedWriteTtlCalls, formatUsd(-cache.deltaUsd), formatUsd(cache.worstCaseDeltaUsd)), 74, ' '))}`);
|
|
7829
|
+
}
|
|
7830
|
+
else if (cache.verdict === 'lost-money') {
|
|
7831
|
+
console.log(` ${c.yellow('!')} ${c.bold(wrap(t.profile.cacheLost(formatUsd(cache.deltaUsd), n(report.total.cacheWriteTokens), n(report.total.cacheReadTokens)), 74, ' '))}`);
|
|
7832
|
+
// Only when it narrows the search. One label is the total again, said twice.
|
|
7833
|
+
if (lostLabels.length > 0 && report.byLabel.length > 1) {
|
|
7834
|
+
console.log(` ${c.dim(wrap(t.profile.cacheLostBy(namedLosers), 74, ' '))}`);
|
|
7835
|
+
}
|
|
7836
|
+
}
|
|
7837
|
+
else {
|
|
7838
|
+
if (cache.verdict === 'paid-off') {
|
|
7839
|
+
console.log(` ${c.dim(wrap(t.profile.cachePaidOff(formatUsd(-cache.deltaUsd)), 74, ' '))}`);
|
|
7840
|
+
}
|
|
7841
|
+
else if (cache.verdict === 'no-difference') {
|
|
7842
|
+
console.log(` ${c.dim(wrap(t.profile.cacheNoDifference(), 74, ' '))}`);
|
|
7843
|
+
}
|
|
7844
|
+
}
|
|
7845
|
+
/**
|
|
7846
|
+
* A workload bleeding underneath a total that does not report a loss.
|
|
7847
|
+
*
|
|
7848
|
+
* The case the aggregate is actively hiding, so it prints as a warning: a cache
|
|
7849
|
+
* paying for itself on one label and losing on another nets out to a comfortable
|
|
7850
|
+
* number, and nothing else on screen would say otherwise.
|
|
7851
|
+
*
|
|
7852
|
+
* The sentence deliberately does not restate the total's verdict. It used to
|
|
7853
|
+
* open "Caching pays off overall", which this position cannot claim — it also
|
|
7854
|
+
* runs under `no-difference`, where the line immediately above has just said the
|
|
7855
|
+
* opposite, and under `unsettled`, where there is no verdict to report at all.
|
|
7856
|
+
*/
|
|
7857
|
+
if (lostLabels.length > 0 && cache.verdict !== 'lost-money') {
|
|
7858
|
+
console.log(` ${c.yellow('!')} ${c.dim(wrap(t.profile.cacheLostHidden(formatUsd(bleeding), namedLosers), 74, ' '))}`);
|
|
7859
|
+
}
|
|
7860
|
+
/**
|
|
7861
|
+
* A label that loses money only if its unstated TTL was the long one.
|
|
7862
|
+
*
|
|
7863
|
+
* The same ambiguity one level down, and it hides better here: a total whose
|
|
7864
|
+
* TTLs are mostly recorded reads as settled while one workload inside it is
|
|
7865
|
+
* entirely unstated. Listed apart from the confirmed losers because it is a
|
|
7866
|
+
* different claim — this one is conditional, and merging the two would make
|
|
7867
|
+
* every name in either list mean less.
|
|
7868
|
+
*/
|
|
7869
|
+
const maybeLostLabels = report.byLabel
|
|
7870
|
+
.map((r) => ({ row: r, cache: cacheEconomics(r.breakdown) }))
|
|
7871
|
+
.filter((r) => r.cache.verdict !== 'lost-money' && r.cache.worstCaseVerdict === 'lost-money');
|
|
7872
|
+
if (maybeLostLabels.length > 0) {
|
|
7873
|
+
console.log(` ${c.dim(wrap(t.profile.cacheTtlUnsettledLabels(listNames(maybeLostLabels)), 74, ' '))}`);
|
|
7874
|
+
}
|
|
7875
|
+
}
|
|
7876
|
+
/**
|
|
7877
|
+
* `trazum plan <log>` — not a list of findings, a ranked plan of what to do.
|
|
7878
|
+
*
|
|
7879
|
+
* The composition (route and batch on one slice never summed) happens in
|
|
7880
|
+
* core's `buildPlan`; this command owns the I/O and the rendering. The plan
|
|
7881
|
+
* saves as a dated JSON file on request, which is what makes verifying it
|
|
7882
|
+
* against a later log possible at all — a prediction nobody wrote down is a
|
|
7883
|
+
* prediction nobody can be held to.
|
|
7884
|
+
*/
|
|
7885
|
+
async function commandPlan(args, pricing, t) {
|
|
7886
|
+
const path = args.positional[0];
|
|
7887
|
+
if (path === undefined)
|
|
7888
|
+
throw new Error(t.plan.noTarget());
|
|
7889
|
+
const GZ = LOG_EXTENSIONS.map((ext) => `${ext}.gz`);
|
|
7890
|
+
const READABLE = [...LOG_EXTENSIONS, ...GZ];
|
|
7891
|
+
const target = await stat(path).catch(() => null);
|
|
7892
|
+
let files = [path];
|
|
7893
|
+
if (target?.isDirectory()) {
|
|
7894
|
+
const entries = await readdir(path, { withFileTypes: true });
|
|
7895
|
+
files = entries
|
|
7896
|
+
.filter((entry) => entry.isFile() && READABLE.some((ext) => entry.name.endsWith(ext)))
|
|
7897
|
+
.map((entry) => join(path, entry.name))
|
|
7898
|
+
.sort((a, b) => a.localeCompare(b));
|
|
7899
|
+
if (files.length === 0)
|
|
7900
|
+
throw new Error(t.profile.noLogsInDirectory(path, READABLE.join(', ')));
|
|
7901
|
+
}
|
|
7902
|
+
const texts = await Promise.all(files.map((file) => readUsageLog(file, t)));
|
|
7903
|
+
const raw = texts.map((text) => (text.endsWith('\n') ? text : `${text}\n`)).join('');
|
|
7904
|
+
const report = profileUsage(raw, { catalogue: pricing });
|
|
7905
|
+
if (report.total.calls === 0)
|
|
7906
|
+
throw new Error(t.plan.nothingPriced());
|
|
7907
|
+
const levers = billLevers(report, { catalogue: pricing });
|
|
7908
|
+
const plan = buildPlan(report, levers, pricing.lastReviewed);
|
|
7909
|
+
const minUsd = typeof args.flags.get('min-usd') === 'string' ? numberFlag(args, 'min-usd', 0, t) : 0;
|
|
7910
|
+
const actions = plan.actions.filter((a) => (a.savingUsd ?? a.stakeUsd ?? 0) >= minUsd);
|
|
7911
|
+
const filtered = plan.actions.length - actions.length;
|
|
7912
|
+
const droppedUsd = plan.actions
|
|
7913
|
+
.filter((a) => (a.savingUsd ?? a.stakeUsd ?? 0) < minUsd)
|
|
7914
|
+
.reduce((sum, a) => sum + (a.savingUsd ?? a.stakeUsd ?? 0), 0);
|
|
7915
|
+
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
7916
|
+
/**
|
|
7917
|
+
* The document's totals cover the actions the document holds — a filtered
|
|
7918
|
+
* plan whose totals still counted the filtered actions would be a file
|
|
7919
|
+
* that contradicts itself, and 1.39's verify would hold it to money it
|
|
7920
|
+
* cannot see. What --min-usd dropped is stated with its worth, never
|
|
7921
|
+
* silently.
|
|
7922
|
+
*/
|
|
7923
|
+
const stamped = {
|
|
7924
|
+
...plan,
|
|
7925
|
+
actions,
|
|
7926
|
+
projectedSavingUsd: actions.reduce((sum, a) => sum + (a.savingUsd ?? 0), 0),
|
|
7927
|
+
measuredStakeUsd: actions.reduce((sum, a) => sum + (a.stakeUsd ?? 0), 0),
|
|
7928
|
+
createdAt: new Date().toISOString(),
|
|
7929
|
+
};
|
|
7930
|
+
const outPath = stringFlag(args, 'out');
|
|
7931
|
+
if (outPath !== undefined) {
|
|
7932
|
+
await writeFile(outPath, `${JSON.stringify(stamped, null, 2)}\n`);
|
|
7933
|
+
}
|
|
7934
|
+
await writeMarkdown(args, () => {
|
|
7935
|
+
const lines = [];
|
|
7936
|
+
lines.push(`## ${t.plan.heading(n(actions.length), formatUsd(plan.totalUsd))}`);
|
|
7937
|
+
lines.push('');
|
|
7938
|
+
lines.push(t.plan.totals(formatUsd(stamped.projectedSavingUsd), formatUsd(stamped.measuredStakeUsd)));
|
|
7939
|
+
if (plan.span === null) {
|
|
7940
|
+
lines.push('');
|
|
7941
|
+
lines.push(`_${t.plan.noClock()}_`);
|
|
7942
|
+
}
|
|
7943
|
+
for (const action of actions) {
|
|
7944
|
+
const name = action.label === UNLABELLED ? t.profile.unlabelled() : action.label;
|
|
7945
|
+
const money = action.savingUsd !== null
|
|
7946
|
+
? t.plan.projected(formatUsd(action.savingUsd))
|
|
7947
|
+
: t.plan.staked(formatUsd(action.stakeUsd ?? 0));
|
|
7948
|
+
lines.push('');
|
|
7949
|
+
lines.push(`### ${t.plan.action(action.kind, name, action.model)} — ${money}`);
|
|
7950
|
+
if (action.detail.routeTo !== undefined)
|
|
7951
|
+
lines.push(`- ${t.plan.routeTo(action.detail.routeTo.displayName)}`);
|
|
7952
|
+
for (const assumption of action.assumes)
|
|
7953
|
+
lines.push(`- ${t.plan.assume(assumption)}`);
|
|
7954
|
+
if (action.check !== null)
|
|
7955
|
+
lines.push(`- ${t.plan.check(action.check)}`);
|
|
7956
|
+
}
|
|
7957
|
+
if (filtered > 0) {
|
|
7958
|
+
lines.push('');
|
|
7959
|
+
lines.push(`_${t.plan.filtered(n(filtered), formatUsd(minUsd), formatUsd(droppedUsd))}_`);
|
|
7960
|
+
}
|
|
7961
|
+
lines.push('');
|
|
7962
|
+
lines.push(`_${t.plan.footer()}_`);
|
|
7963
|
+
return lines.join('\n');
|
|
7964
|
+
});
|
|
7965
|
+
if (boolFlag(args, 'json')) {
|
|
7966
|
+
console.log(JSON.stringify(stamped, null, 2));
|
|
7967
|
+
return;
|
|
7425
7968
|
}
|
|
7426
|
-
|
|
7427
|
-
|
|
7969
|
+
console.log(sectionHeading(t.plan.heading(n(actions.length), formatUsd(plan.totalUsd))));
|
|
7970
|
+
console.log(` ${wrap(t.plan.totals(formatUsd(stamped.projectedSavingUsd), formatUsd(stamped.measuredStakeUsd)), 74, ' ')}`);
|
|
7971
|
+
if (plan.span === null) {
|
|
7972
|
+
console.log(` ${c.dim(wrap(t.plan.noClock(), 74, ' '))}`);
|
|
7428
7973
|
}
|
|
7429
|
-
|
|
7430
|
-
|
|
7431
|
-
|
|
7432
|
-
|
|
7433
|
-
|
|
7434
|
-
|
|
7435
|
-
|
|
7436
|
-
|
|
7437
|
-
|
|
7438
|
-
|
|
7439
|
-
|
|
7440
|
-
console.log(`
|
|
7441
|
-
|
|
7442
|
-
|
|
7443
|
-
|
|
7444
|
-
}
|
|
7445
|
-
if (slice.batch) {
|
|
7446
|
-
console.log(` ${c.dim('·')} ${c.dim(wrap(t.profile.leverBatch(formatUsd(slice.batch.savingUsd)), 74, ' '))}`);
|
|
7447
|
-
}
|
|
7448
|
-
// The arithmetic is exact and the quality question is untouched by it.
|
|
7449
|
-
// Naming the command is the difference between a saving and a gamble.
|
|
7450
|
-
if (slice.route) {
|
|
7451
|
-
console.log(` ${c.dim(wrap(t.profile.leverRouteVerify(slice.route.candidate.id), 74, ' '))}`);
|
|
7452
|
-
}
|
|
7974
|
+
for (const action of actions) {
|
|
7975
|
+
const name = action.label === UNLABELLED ? t.profile.unlabelled() : action.label;
|
|
7976
|
+
const money = action.savingUsd !== null
|
|
7977
|
+
? t.plan.projected(formatUsd(action.savingUsd))
|
|
7978
|
+
: t.plan.staked(formatUsd(action.stakeUsd ?? 0));
|
|
7979
|
+
console.log();
|
|
7980
|
+
console.log(` ${c.green('→')} ${c.bold(t.plan.action(action.kind, name, action.model))} ${money}`);
|
|
7981
|
+
if (action.detail.routeTo !== undefined) {
|
|
7982
|
+
console.log(` ${c.dim(t.plan.routeTo(action.detail.routeTo.displayName))}`);
|
|
7983
|
+
}
|
|
7984
|
+
for (const assumption of action.assumes) {
|
|
7985
|
+
console.log(` ${c.yellow('?')} ${c.dim(wrap(t.plan.assume(assumption), 72, ' '))}`);
|
|
7986
|
+
}
|
|
7987
|
+
if (action.check !== null) {
|
|
7988
|
+
console.log(` ${c.dim(wrap(t.plan.check(action.check), 72, ' '))}`);
|
|
7453
7989
|
}
|
|
7454
7990
|
}
|
|
7991
|
+
if (filtered > 0) {
|
|
7992
|
+
console.log();
|
|
7993
|
+
console.log(` ${c.dim(wrap(t.plan.filtered(n(filtered), formatUsd(minUsd), formatUsd(droppedUsd)), 74, ' '))}`);
|
|
7994
|
+
}
|
|
7455
7995
|
console.log();
|
|
7456
|
-
console.log(` ${c.dim(wrap(t.
|
|
7996
|
+
console.log(` ${c.dim(wrap(t.plan.footer(), 74, ' '))}`);
|
|
7997
|
+
if (outPath !== undefined)
|
|
7998
|
+
console.log(c.dim(wrap(t.plan.wrote(outPath), 74, '')));
|
|
7999
|
+
}
|
|
8000
|
+
async function commandProfile(args, config, configDir, pricing, t) {
|
|
8001
|
+
const path = args.positional[0];
|
|
8002
|
+
if (path === undefined) {
|
|
8003
|
+
console.log();
|
|
8004
|
+
console.log(c.dim(wrap(t.profile.noTarget(), 74, ' ')));
|
|
8005
|
+
console.log();
|
|
8006
|
+
return;
|
|
8007
|
+
}
|
|
7457
8008
|
/**
|
|
7458
|
-
*
|
|
8009
|
+
* A log, or a directory of them.
|
|
7459
8010
|
*
|
|
7460
|
-
*
|
|
7461
|
-
*
|
|
7462
|
-
*
|
|
8011
|
+
* Usage logs rotate: `logs/2026-08-01.jsonl`, `logs/2026-08-02.jsonl`, one
|
|
8012
|
+
* per day for a month. Making somebody `cat` them together before a profile
|
|
8013
|
+
* will read them is a setup cost that gets a tool skipped, and doing it for
|
|
8014
|
+
* them is a directory listing.
|
|
7463
8015
|
*
|
|
7464
|
-
*
|
|
7465
|
-
*
|
|
7466
|
-
*
|
|
7467
|
-
*
|
|
7468
|
-
*
|
|
7469
|
-
* computed over money that cannot move is a percentage of the wrong
|
|
7470
|
-
* denominator.
|
|
8016
|
+
* Files are read in name order — which for dated names is time order — and
|
|
8017
|
+
* how many were read is stated, because a report over "the logs" that
|
|
8018
|
+
* silently skipped one is a total that is wrong by an unknown amount. A
|
|
8019
|
+
* directory holding nothing readable is an error naming what it looked for,
|
|
8020
|
+
* not an empty report.
|
|
7471
8021
|
*/
|
|
7472
|
-
|
|
7473
|
-
|
|
7474
|
-
|
|
7475
|
-
|
|
7476
|
-
|
|
7477
|
-
|
|
7478
|
-
|
|
7479
|
-
|
|
7480
|
-
|
|
7481
|
-
|
|
7482
|
-
|
|
7483
|
-
|
|
7484
|
-
|
|
7485
|
-
|
|
7486
|
-
|
|
7487
|
-
|
|
7488
|
-
* adding two savings, and hedged the only honest way — whether these
|
|
7489
|
-
* calls can wait is not in the log.
|
|
7490
|
-
*/
|
|
7491
|
-
if (whatIf.batchOnTarget !== null) {
|
|
7492
|
-
console.log(` ${c.dim(wrap(t.profile.whatIfBatchOnTarget(formatUsd(whatIf.batchOnTarget.targetUsd), formatUsd(whatIf.targetUsd)), 74, ' '))}`);
|
|
7493
|
-
}
|
|
7494
|
-
for (const slice of whatIf.slices.slice(0, 5)) {
|
|
7495
|
-
const label = slice.label === UNLABELLED ? t.profile.unlabelled() : slice.label;
|
|
7496
|
-
console.log(` ${c.dim('·')} ${c.dim(wrap(t.profile.whatIfSlice(label, slice.model, formatUsd(slice.currentUsd), formatUsd(slice.targetUsd)), 74, ' '))}`);
|
|
7497
|
-
/**
|
|
7498
|
-
* Cache traffic the target could not grant, said loudly and beside
|
|
7499
|
-
* the figure it corrects. The standard row prices cache entries the
|
|
7500
|
-
* target's minimum would refuse to create — an error that flatters
|
|
7501
|
-
* the move, which is the direction this repository refuses.
|
|
7502
|
-
*/
|
|
7503
|
-
if (slice.cacheBeyondTarget !== null) {
|
|
7504
|
-
console.log(` ${c.yellow('!')} ${c.dim(wrap(t.profile.whatIfCacheBeyond(n(slice.maxCallInputTokens), n(slice.cacheBeyondTarget.minTokens), formatUsd(slice.cacheBeyondTarget.noCacheUsd)), 74, ' '))}`);
|
|
7505
|
-
}
|
|
7506
|
-
}
|
|
7507
|
-
}
|
|
8022
|
+
/**
|
|
8023
|
+
* The same names, gzipped — which is what a rotated log actually looks like
|
|
8024
|
+
* a day after it rotates.
|
|
8025
|
+
*
|
|
8026
|
+
* `logrotate`, Docker's json-file driver and every cloud log export compress
|
|
8027
|
+
* yesterday's file, so a directory of a month's logs is one plain file and
|
|
8028
|
+
* twenty-nine `.gz` ones. Reading only the plain one and saying nothing
|
|
8029
|
+
* would report a month's bill from a day of it, in the flattering
|
|
8030
|
+
* direction, which is exactly the failure directory mode was added to
|
|
8031
|
+
* prevent.
|
|
8032
|
+
*/
|
|
8033
|
+
const GZ_EXTENSIONS = LOG_EXTENSIONS.map((ext) => `${ext}.gz`);
|
|
8034
|
+
const READABLE = [...LOG_EXTENSIONS, ...GZ_EXTENSIONS];
|
|
8035
|
+
const target = await stat(path).catch(() => null);
|
|
8036
|
+
let logFiles = [path];
|
|
8037
|
+
if (target?.isDirectory()) {
|
|
7508
8038
|
/**
|
|
7509
|
-
*
|
|
7510
|
-
*
|
|
7511
|
-
*
|
|
8039
|
+
* Recursive under `--by-source`, flat otherwise. The fleet's whole point
|
|
8040
|
+
* is one directory per service, so the walk must descend; the flat mode
|
|
8041
|
+
* keeps its long-standing behaviour because a directory of rotated logs
|
|
8042
|
+
* with an unrelated subfolder should not quietly absorb it.
|
|
7512
8043
|
*/
|
|
7513
|
-
|
|
7514
|
-
|
|
7515
|
-
|
|
7516
|
-
|
|
7517
|
-
|
|
7518
|
-
|
|
7519
|
-
if (
|
|
7520
|
-
|
|
7521
|
-
}
|
|
7522
|
-
// Models with no current price have no difference to state — their target
|
|
7523
|
-
// cost is knowable and the subtraction is not.
|
|
7524
|
-
if (whatIf.unpricedCalls > 0) {
|
|
7525
|
-
console.log(` ${c.dim(wrap(t.profile.whatIfUnpriced(t.profile.calls(whatIf.unpricedCalls), whatIf.unpricedModels.join(', ')), 74, ' '))}`);
|
|
8044
|
+
const bySourceMode = boolFlag(args, 'by-source');
|
|
8045
|
+
const entries = await readdir(path, { withFileTypes: true, recursive: bySourceMode });
|
|
8046
|
+
logFiles = entries
|
|
8047
|
+
.filter((entry) => entry.isFile() && READABLE.some((ext) => entry.name.endsWith(ext)))
|
|
8048
|
+
.map((entry) => join(entry.parentPath ?? path, entry.name))
|
|
8049
|
+
.sort((a, b) => a.localeCompare(b));
|
|
8050
|
+
if (logFiles.length === 0) {
|
|
8051
|
+
throw new Error(t.profile.noLogsInDirectory(path, READABLE.join(', ')));
|
|
7526
8052
|
}
|
|
7527
8053
|
}
|
|
7528
8054
|
/**
|
|
7529
|
-
*
|
|
8055
|
+
* Gzipped files are decompressed in memory; everything else is read as text.
|
|
7530
8056
|
*
|
|
7531
|
-
*
|
|
7532
|
-
*
|
|
7533
|
-
*
|
|
7534
|
-
*
|
|
8057
|
+
* Decided by **extension**, not by sniffing the first two bytes: a file
|
|
8058
|
+
* named `.jsonl` whose contents happen to start with 0x1f8b is far more
|
|
8059
|
+
* likely to be a corrupt log than a mislabelled archive, and silently
|
|
8060
|
+
* treating it as one would turn a diagnosable error into an empty report.
|
|
7535
8061
|
*
|
|
7536
|
-
*
|
|
7537
|
-
*
|
|
7538
|
-
*
|
|
7539
|
-
* bound is exact, and the reader can act on it.
|
|
8062
|
+
* A `.gz` that will not decompress is an error naming the file. The
|
|
8063
|
+
* alternative — skipping it — is a total quietly missing a day, which is
|
|
8064
|
+
* the failure this repository refuses in every other place it can occur.
|
|
7540
8065
|
*/
|
|
7541
|
-
|
|
7542
|
-
|
|
7543
|
-
|
|
7544
|
-
|
|
7545
|
-
const label = growth.label === UNLABELLED ? t.profile.unlabelled() : growth.label;
|
|
7546
|
-
console.log();
|
|
7547
|
-
console.log(` ${c.bold(wrap(t.profile.historyGrowth(label, growth.modelName, n(Math.round(growth.minTurnTokens)), n(Math.round(growth.maxTurnTokens)), n(growth.longestSession)), 74, ' '))}`);
|
|
7548
|
-
console.log(` ${c.dim(wrap(t.profile.historyCeiling(formatUsd(growth.growthUsd), pct(growth.shareOfBill), formatUsd(growth.flatUsd), formatUsd(growth.inputUsd)), 74, ' '))}`);
|
|
7549
|
-
}
|
|
7550
|
-
}
|
|
7551
|
-
else if (!report.hasSessions) {
|
|
7552
|
-
/**
|
|
7553
|
-
* Not the same as "no growth". A log without a session field cannot be asked
|
|
7554
|
-
* the question at all, and silence there would read as a clean bill of health
|
|
7555
|
-
* on the line most likely to be the biggest.
|
|
7556
|
-
*/
|
|
7557
|
-
console.log();
|
|
7558
|
-
console.log(sectionHeading(t.profile.historyHeading()));
|
|
7559
|
-
console.log(` ${c.dim(wrap(t.profile.historyNoSessions(), 74, ' '))}`);
|
|
7560
|
-
}
|
|
8066
|
+
const logTexts = await Promise.all(logFiles.map((file) => readUsageLog(file, t)));
|
|
8067
|
+
// A file that does not end in a newline would otherwise glue its last record
|
|
8068
|
+
// to the next file's first one, and both would be reported as unreadable.
|
|
8069
|
+
const raw = logTexts.map((text) => (text.endsWith('\n') ? text : `${text}\n`)).join('');
|
|
7561
8070
|
/**
|
|
7562
|
-
*
|
|
7563
|
-
*
|
|
7564
|
-
*
|
|
7565
|
-
*
|
|
7566
|
-
* of calls holding half of it is a tail, and a tail has a cause worth a
|
|
7567
|
-
* morning; forty-five per cent is what evenly spread looks like, and the only
|
|
7568
|
-
* lever there is asking every answer to be shorter. The threshold between the
|
|
7569
|
-
* two is a quarter of the calls — far enough from both shapes that rounding
|
|
7570
|
-
* cannot flip the message, and stated here because it is a presentation choice,
|
|
7571
|
-
* not a measurement.
|
|
8071
|
+
* The drill-down. A label that matches nothing is an error naming the labels
|
|
8072
|
+
* that exist — the route command's rule, for the route command's reason: a
|
|
8073
|
+
* report over zero calls silently filtered would read as "this workload is
|
|
8074
|
+
* free".
|
|
7572
8075
|
*/
|
|
7573
|
-
|
|
7574
|
-
|
|
7575
|
-
|
|
7576
|
-
|
|
7577
|
-
|
|
7578
|
-
|
|
7579
|
-
|
|
7580
|
-
|
|
7581
|
-
|
|
7582
|
-
|
|
7583
|
-
|
|
7584
|
-
|
|
7585
|
-
|
|
7586
|
-
|
|
7587
|
-
|
|
7588
|
-
|
|
7589
|
-
|
|
7590
|
-
|
|
7591
|
-
|
|
7592
|
-
|
|
7593
|
-
|
|
7594
|
-
|
|
7595
|
-
|
|
8076
|
+
const onlyLabel = stringFlag(args, 'label');
|
|
8077
|
+
/**
|
|
8078
|
+
* The drill-down in time. `--since`/`--until` take a UTC day or a full
|
|
8079
|
+
* timestamp; a bare day means the whole of it — since its first instant,
|
|
8080
|
+
* until its last — because "--until 2026-08-14" excluding the named day is
|
|
8081
|
+
* a trap sprung on everyone who reads dates the way humans do. Internally
|
|
8082
|
+
* the window is half-open `[since, until)`, so two adjacent windows share
|
|
8083
|
+
* no record.
|
|
8084
|
+
*/
|
|
8085
|
+
const now = Date.now();
|
|
8086
|
+
const sinceWhen = parseWhen(args, 'since', false, t, now);
|
|
8087
|
+
const untilWhen = parseWhen(args, 'until', true, t, now);
|
|
8088
|
+
const relativeWindow = sinceWhen.relative || untilWhen.relative;
|
|
8089
|
+
const sinceMs = sinceWhen.ms;
|
|
8090
|
+
const untilMs = untilWhen.ms;
|
|
8091
|
+
if (sinceMs !== undefined && untilMs !== undefined && sinceMs >= untilMs) {
|
|
8092
|
+
throw new Error(t.profile.sinceAfterUntil());
|
|
8093
|
+
}
|
|
8094
|
+
const windowed = sinceMs !== undefined || untilMs !== undefined;
|
|
8095
|
+
/**
|
|
8096
|
+
* How old the price table behind every dollar below is. Stated only when it
|
|
8097
|
+
* is old enough to matter: `models` and `doctor` always print the date, but
|
|
8098
|
+
* a profile is read for its figures, and the one fact that silently
|
|
8099
|
+
* invalidates all of them is a table the provider has re-priced since.
|
|
8100
|
+
* The threshold is in the sentence, and the number behind it is
|
|
8101
|
+
* `STALE_PRICING_DAYS` — shared with the MCP report and the browser's bill,
|
|
8102
|
+
* which used to keep their own copies of it.
|
|
8103
|
+
*/
|
|
8104
|
+
const pricingAgeDays = reviewAgeDays(pricing.lastReviewed, new Date());
|
|
8105
|
+
const pricingStale = pricingAgeDays !== null && pricingAgeDays > STALE_PRICING_DAYS
|
|
8106
|
+
? { date: pricing.lastReviewed, days: pricingAgeDays }
|
|
8107
|
+
: null;
|
|
8108
|
+
const report = profileUsage(raw, { catalogue: pricing, label: onlyLabel, sinceMs, untilMs });
|
|
8109
|
+
if (report.total.calls === 0 && report.unpriced.calls === 0) {
|
|
8110
|
+
if (onlyLabel !== undefined || windowed) {
|
|
8111
|
+
// Diagnose against the log without the failed filter, so the error can
|
|
8112
|
+
// name what does exist instead of describing an absence.
|
|
8113
|
+
const unfiltered = profileUsage(raw, { catalogue: pricing });
|
|
8114
|
+
if (unfiltered.total.calls > 0 || unfiltered.unpriced.calls > 0) {
|
|
8115
|
+
if (onlyLabel !== undefined && !unfiltered.byLabel.some((r) => r.label === onlyLabel)) {
|
|
8116
|
+
const available = unfiltered.byLabel
|
|
8117
|
+
.map((r) => (r.label === UNLABELLED ? t.profile.unlabelled() : r.label))
|
|
8118
|
+
.join(', ');
|
|
8119
|
+
throw new Error(t.route.labelNotFound(onlyLabel, available || '—'));
|
|
8120
|
+
}
|
|
8121
|
+
if (windowed) {
|
|
8122
|
+
/**
|
|
8123
|
+
* A window that matches nothing must not become a $0 report — under
|
|
8124
|
+
* `--max-usd` it would pass a budget gate over a period the log
|
|
8125
|
+
* simply does not cover, which is the flattering non-answer. The
|
|
8126
|
+
* error names what the log *does* cover, or says it has no clock at
|
|
8127
|
+
* all, so the fix is visible in the message.
|
|
8128
|
+
*/
|
|
8129
|
+
if (unfiltered.span === null)
|
|
8130
|
+
throw new Error(t.profile.windowNeedsClock());
|
|
8131
|
+
throw new Error(`${t.profile.windowMatchesNothing(dayOf(unfiltered.span.fromMs), dayOf(unfiltered.span.toMs))}${relativeWindow ? ` ${t.profile.windowRelativeEmpty()}` : ''}`);
|
|
8132
|
+
}
|
|
7596
8133
|
}
|
|
7597
8134
|
}
|
|
7598
8135
|
}
|
|
8136
|
+
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
8137
|
+
const pct = (share) => `${(share * 100).toFixed(1)}%`;
|
|
7599
8138
|
/**
|
|
7600
|
-
*
|
|
8139
|
+
* A gate armed over a report with nothing in it, refused rather than passed.
|
|
7601
8140
|
*
|
|
7602
|
-
* The
|
|
7603
|
-
*
|
|
7604
|
-
*
|
|
7605
|
-
*
|
|
7606
|
-
*
|
|
8141
|
+
* The product had already made this call once and only once. A `--since` that
|
|
8142
|
+
* matches no record throws, and the comment above it says why: "under
|
|
8143
|
+
* --max-usd it would pass a budget gate over a period the log does not
|
|
8144
|
+
* cover". Two other ways of measuring nothing reached the same gate and were
|
|
8145
|
+
* never given the same answer — a log whose every line was unreadable, and a
|
|
8146
|
+
* log with no records at all. Both exited 0 against a budget, on the human
|
|
8147
|
+
* path and under `--json`.
|
|
7607
8148
|
*
|
|
7608
|
-
*
|
|
7609
|
-
*
|
|
7610
|
-
*
|
|
7611
|
-
*
|
|
8149
|
+
* That is the doctrine's own rule, verbatim: *a period, or a service, nobody
|
|
8150
|
+
* measured is not one under budget. Name the gap; never report the absence as
|
|
8151
|
+
* a pass.* A pipeline that stopped writing looks exactly like a quiet month,
|
|
8152
|
+
* and the whole reason the rule exists is that the quiet month is the one
|
|
8153
|
+
* that reads as green.
|
|
8154
|
+
*
|
|
8155
|
+
* `--allow-empty` is the way to say a period with no calls is the expected
|
|
8156
|
+
* answer, because that is a real thing a nightly job wants and it should be
|
|
8157
|
+
* said out loud rather than inferred from silence.
|
|
7612
8158
|
*/
|
|
7613
|
-
|
|
7614
|
-
|
|
7615
|
-
|
|
7616
|
-
|
|
7617
|
-
|
|
7618
|
-
|
|
7619
|
-
|
|
7620
|
-
|
|
7621
|
-
|
|
7622
|
-
|
|
7623
|
-
|
|
7624
|
-
|
|
7625
|
-
|
|
7626
|
-
|
|
7627
|
-
|
|
7628
|
-
|
|
7629
|
-
|
|
7630
|
-
|
|
7631
|
-
|
|
7632
|
-
|
|
7633
|
-
|
|
7634
|
-
/**
|
|
7635
|
-
* What that size actually costs. A cache read is a tenth of input on
|
|
7636
|
-
* Anthropic, so a large slice reading almost everything from cache is a
|
|
7637
|
-
* very different bill from one paying full rate — and the token counts
|
|
7638
|
-
* alone cannot tell them apart.
|
|
7639
|
-
*/
|
|
7640
|
-
if (shape.cachedShare >= 0.5) {
|
|
7641
|
-
console.log(` ${c.dim(wrap(t.profile.inputMostlyCached(pct(shape.cachedShare)), 74, ' '))}`);
|
|
7642
|
-
}
|
|
7643
|
-
else if (shape.cachedShare < 0.1) {
|
|
7644
|
-
console.log(` ${c.dim(wrap(t.profile.inputFullRate(), 74, ' '))}`);
|
|
7645
|
-
}
|
|
7646
|
-
}
|
|
8159
|
+
const gateArmed = typeof args.flags.get('max-usd') === 'string' ||
|
|
8160
|
+
typeof args.flags.get('max-growth-usd') === 'string' ||
|
|
8161
|
+
typeof args.flags.get('max-cache-loss-usd') === 'string' ||
|
|
8162
|
+
typeof args.flags.get('max-day-usd') === 'string' ||
|
|
8163
|
+
typeof args.flags.get('max-session-usd') === 'string' ||
|
|
8164
|
+
config.spend !== undefined;
|
|
8165
|
+
if (gateArmed && report.total.calls === 0 && !boolFlag(args, 'allow-empty')) {
|
|
8166
|
+
const gap = report.unpriced.calls > 0
|
|
8167
|
+
? t.profile.nothingUnpriced(report.unpriced.calls)
|
|
8168
|
+
: report.skippedLines.length > 0
|
|
8169
|
+
? t.profile.nothingUnreadable(report.skippedLines.length)
|
|
8170
|
+
: t.profile.nothingEmpty();
|
|
8171
|
+
throw new Error(t.profile.gateNothingMeasured(gap));
|
|
8172
|
+
}
|
|
8173
|
+
if (boolFlag(args, 'by-source')) {
|
|
8174
|
+
await profileBySource(args, config, logFiles, logTexts, pricing, { onlyLabel, sinceMs, untilMs }, t);
|
|
8175
|
+
return;
|
|
8176
|
+
}
|
|
8177
|
+
if (boolFlag(args, 'dry-run')) {
|
|
8178
|
+
profileDryRun(args, report, t);
|
|
8179
|
+
return;
|
|
7647
8180
|
}
|
|
7648
8181
|
/**
|
|
7649
|
-
*
|
|
8182
|
+
* The previous log, loaded before the output paths split so the growth gate
|
|
8183
|
+
* exists under `--json` too — a CI step reads the JSON and trusts the exit
|
|
8184
|
+
* code, and a gate that only arms in the human rendering is a gate CI never
|
|
8185
|
+
* had.
|
|
8186
|
+
*/
|
|
8187
|
+
/* Filled by `runProfileGates` on whichever output path runs, and read by
|
|
8188
|
+
the side-file writer, which is why it is declared here rather than
|
|
8189
|
+
inlined at either call. */
|
|
8190
|
+
let gates = { failed: false, verdicts: [] };
|
|
8191
|
+
const againstPath = stringFlag(args, 'against');
|
|
8192
|
+
// The same filter on both sides: comparing one workload's bill against the
|
|
8193
|
+
// whole previous log would report every sibling workload as vanished savings.
|
|
8194
|
+
const previous = againstPath !== undefined
|
|
8195
|
+
// The same reader as the log itself, so `--against last-month.jsonl.gz`
|
|
8196
|
+
// works: a comparison that could only read one of the two formats would
|
|
8197
|
+
// be a flag that fails on exactly the rotated file it exists to read.
|
|
8198
|
+
? profileUsage(await readUsageLog(againstPath, t), {
|
|
8199
|
+
catalogue: pricing,
|
|
8200
|
+
label: onlyLabel,
|
|
8201
|
+
// The same window on both sides, for the same reason as the label:
|
|
8202
|
+
// a windowed bill against an unwindowed one compares a slice to a
|
|
8203
|
+
// whole and calls the difference growth.
|
|
8204
|
+
sinceMs,
|
|
8205
|
+
untilMs,
|
|
8206
|
+
})
|
|
8207
|
+
: null;
|
|
8208
|
+
const againstDelta = previous !== null && previous.total.calls > 0
|
|
8209
|
+
? report.total.totalUsd - previous.total.totalUsd
|
|
8210
|
+
: null;
|
|
8211
|
+
/**
|
|
8212
|
+
* The same tokens at another model's rates, computed before the output paths
|
|
8213
|
+
* split so `--json` carries it too.
|
|
7650
8214
|
*
|
|
7651
|
-
*
|
|
7652
|
-
*
|
|
7653
|
-
*
|
|
7654
|
-
*
|
|
7655
|
-
* and quiet above half, so the reader sees it coming either way. No
|
|
7656
|
-
* prediction of *when*: a straight line through two points is a guess
|
|
7657
|
-
* wearing arithmetic's clothes.
|
|
8215
|
+
* An unknown id **throws** rather than printing nothing. A flag that silently
|
|
8216
|
+
* does nothing is worse than a missing feature: the reader typed a question,
|
|
8217
|
+
* got a report with no answer in it, and has no way to tell a typo from a
|
|
8218
|
+
* model this comparison had nothing to say about.
|
|
7658
8219
|
*/
|
|
7659
|
-
|
|
7660
|
-
|
|
7661
|
-
|
|
7662
|
-
|
|
7663
|
-
|
|
7664
|
-
|
|
7665
|
-
|
|
7666
|
-
|
|
7667
|
-
|
|
7668
|
-
|
|
7669
|
-
console.log(` ${c.dim(wrap(t.profile.pressureAdvice(), 74, ' '))}`);
|
|
7670
|
-
}
|
|
7671
|
-
else {
|
|
7672
|
-
console.log(` ${c.dim(wrap(line, 74, ' '))}`);
|
|
7673
|
-
}
|
|
7674
|
-
}
|
|
7675
|
-
}
|
|
8220
|
+
/**
|
|
8221
|
+
* The largest call against each model's window, computed once here so the
|
|
8222
|
+
* terminal, the JSON and the markdown state the same ratio — the
|
|
8223
|
+
* denominator lives in the (possibly overlaid) catalogue.
|
|
8224
|
+
*/
|
|
8225
|
+
const pressures = contextPressure(report, pricing);
|
|
8226
|
+
const whatIfModel = stringFlag(args, 'what-if');
|
|
8227
|
+
const whatIf = whatIfModel !== undefined ? repriceProfile(report, whatIfModel, pricing) : null;
|
|
8228
|
+
if (whatIfModel !== undefined && whatIf === null) {
|
|
8229
|
+
throw new Error(t.profile.whatIfUnknown(whatIfModel, pricing.models.map((m) => m.id).join(', ')));
|
|
7676
8230
|
}
|
|
7677
8231
|
/**
|
|
7678
|
-
* The
|
|
7679
|
-
*
|
|
7680
|
-
*
|
|
8232
|
+
* The drivers of the change, per label and per model, computed once here so
|
|
8233
|
+
* the terminal, the JSON and any future rendering describe the same change.
|
|
8234
|
+
* The model half answers the question the label half cannot: "the growth is
|
|
8235
|
+
* traffic moving from Haiku to Opus" is a fact about the mix, invisible in
|
|
8236
|
+
* per-workload rows whose names did not change.
|
|
7681
8237
|
*/
|
|
7682
|
-
|
|
7683
|
-
|
|
7684
|
-
|
|
7685
|
-
|
|
7686
|
-
|
|
7687
|
-
|
|
7688
|
-
|
|
7689
|
-
|
|
7690
|
-
|
|
7691
|
-
|
|
8238
|
+
const labelDrivers = previous !== null && previous.total.calls > 0
|
|
8239
|
+
? driversBetween(previous.byLabel.map((r) => ({ key: r.label, usd: r.breakdown.totalUsd })), report.byLabel.map((r) => ({ key: r.label, usd: r.breakdown.totalUsd })))
|
|
8240
|
+
: [];
|
|
8241
|
+
const modelDrivers = previous !== null && previous.total.calls > 0
|
|
8242
|
+
? driversBetween(previous.byModel.map((r) => ({ key: r.model, usd: r.breakdown.totalUsd })), report.byModel.map((r) => ({ key: r.model, usd: r.breakdown.totalUsd })))
|
|
8243
|
+
: [];
|
|
8244
|
+
/**
|
|
8245
|
+
* Whether the two logs share any time at all. This comparison is meant for
|
|
8246
|
+
* disjoint periods or snapshots of different systems; when both spans are
|
|
8247
|
+
* known and intersect, the same calls may sit on both sides of the
|
|
8248
|
+
* subtraction and the "growth" is partly the same money counted twice.
|
|
8249
|
+
* Only decidable when both logs carry a clock — three states, as always:
|
|
8250
|
+
* warned, clear, or unknown, and unknown stays silent rather than clear.
|
|
8251
|
+
*/
|
|
8252
|
+
const againstOverlap = previous !== null &&
|
|
8253
|
+
previous.total.calls > 0 &&
|
|
8254
|
+
previous.span !== null &&
|
|
8255
|
+
report.span !== null &&
|
|
8256
|
+
Math.min(report.span.toMs, previous.span.toMs) >=
|
|
8257
|
+
Math.max(report.span.fromMs, previous.span.fromMs)
|
|
8258
|
+
? {
|
|
8259
|
+
fromMs: Math.max(report.span.fromMs, previous.span.fromMs),
|
|
8260
|
+
toMs: Math.min(report.span.toMs, previous.span.toMs),
|
|
7692
8261
|
}
|
|
8262
|
+
: null;
|
|
8263
|
+
// A gate flag that silently does nothing is not an answer — same rule as
|
|
8264
|
+
// --apply-suggestions without --suggest.
|
|
8265
|
+
if (typeof args.flags.get('max-growth-usd') === 'string' && againstPath === undefined) {
|
|
8266
|
+
throw new Error(t.profile.maxGrowthNeedsAgainst());
|
|
8267
|
+
}
|
|
8268
|
+
if (boolFlag(args, 'json')) {
|
|
8269
|
+
/**
|
|
8270
|
+
* The report, plus everything the human output leads on.
|
|
8271
|
+
*
|
|
8272
|
+
* Additive rather than a reshape: `report` keeps the shape `@trazum/core`
|
|
8273
|
+
* returns. The cache verdict is included because leaving a consumer to
|
|
8274
|
+
* re-derive it means two implementations of a sign convention where positive
|
|
8275
|
+
* means *worse*, and one of them will eventually get it backwards.
|
|
8276
|
+
*
|
|
8277
|
+
* `levers` is included because it was not, and that made the flagship
|
|
8278
|
+
* section terminal-only: "What would actually move this bill" — the reason
|
|
8279
|
+
* the command exists — was invisible to any pipeline, dashboard or CI step
|
|
8280
|
+
* reading the JSON. A finding the machine-readable output omits is a finding
|
|
8281
|
+
* the reader's tooling will never surface.
|
|
8282
|
+
*/
|
|
8283
|
+
console.log(JSON.stringify({
|
|
8284
|
+
/**
|
|
8285
|
+
* `schemaVersion` arrives inside `report` now, and that is the point.
|
|
8286
|
+
*
|
|
8287
|
+
* It used to be stamped here, which made the profile the one contract
|
|
8288
|
+
* of the ten whose version existed only if you went through this
|
|
8289
|
+
* command. `docs/format.md` says every document carries it and
|
|
8290
|
+
* `conform` rejects one that does not, so a connector author using
|
|
8291
|
+
* `@trazum/core` produced a profile this tool refuses.
|
|
8292
|
+
*
|
|
8293
|
+
* The contract is documented in docs/json-output.md and enforced by
|
|
8294
|
+
* json-contract.test.js. It changes only when a field's meaning
|
|
8295
|
+
* changes or one is removed — new findings arrive as new keys, so a
|
|
8296
|
+
* consumer that ignores unknown ones keeps working.
|
|
8297
|
+
*/
|
|
8298
|
+
...report,
|
|
8299
|
+
cache: cacheEconomics(report.total),
|
|
8300
|
+
cacheByLabel: report.byLabel.map((r) => ({
|
|
8301
|
+
label: r.label,
|
|
8302
|
+
cache: cacheEconomics(r.breakdown),
|
|
8303
|
+
})),
|
|
8304
|
+
// The provenance of every dollar above: which price table, how old.
|
|
8305
|
+
pricing: { lastReviewed: pricing.lastReviewed, ageDays: pricingAgeDays },
|
|
8306
|
+
levers: billLevers(report, { catalogue: pricing }),
|
|
8307
|
+
// Present only when --against was passed: null delta means the
|
|
8308
|
+
// previous log had nothing priced, which is a different answer from
|
|
8309
|
+
// zero growth.
|
|
8310
|
+
...(previous !== null
|
|
8311
|
+
? {
|
|
8312
|
+
against: {
|
|
8313
|
+
previousTotalUsd: previous.total.totalUsd,
|
|
8314
|
+
deltaUsd: againstDelta,
|
|
8315
|
+
// The same drivers the terminal names, as data. A finding
|
|
8316
|
+
// the machine-readable output omits is a finding the
|
|
8317
|
+
// reader's tooling will never surface.
|
|
8318
|
+
byLabel: labelDrivers,
|
|
8319
|
+
byModel: modelDrivers,
|
|
8320
|
+
},
|
|
8321
|
+
}
|
|
8322
|
+
: {}),
|
|
8323
|
+
// How close each slice's largest call is to its model's window —
|
|
8324
|
+
// derived like `cache` and `levers`, so a dashboard need not
|
|
8325
|
+
// re-derive a ratio whose denominator lives in the catalogue.
|
|
8326
|
+
contextPressure: pressures,
|
|
8327
|
+
// Present only when --what-if was passed. `sameTokensAssumed` rides
|
|
8328
|
+
// along inside it so a consumer cannot print the dollar figure
|
|
8329
|
+
// without the caveat being in the same object.
|
|
8330
|
+
...(whatIf !== null ? { whatIf } : {}),
|
|
8331
|
+
}, null, 2));
|
|
8332
|
+
gates = await runProfileGates({
|
|
8333
|
+
args, config, configDir, report, previous, againstDelta, windowed, now, pricing, t,
|
|
8334
|
+
});
|
|
8335
|
+
await writeProfileSideFiles({
|
|
8336
|
+
args, path, report, previous, pressures, labelDrivers, modelDrivers, againstOverlap,
|
|
8337
|
+
whatIf, pricingStale, windowed, gates, pricing, t,
|
|
8338
|
+
});
|
|
8339
|
+
return;
|
|
7693
8340
|
}
|
|
7694
8341
|
/**
|
|
7695
|
-
*
|
|
8342
|
+
* Nothing priced means there is no report, not a report of zero.
|
|
7696
8343
|
*
|
|
7697
|
-
*
|
|
7698
|
-
*
|
|
7699
|
-
*
|
|
7700
|
-
*
|
|
7701
|
-
*
|
|
7702
|
-
*
|
|
8344
|
+
* The guard was `total.calls === 0 && unpriced.calls === 0`, so a log whose every
|
|
8345
|
+
* model was unknown fell through and printed a full report built from a zeroed
|
|
8346
|
+
* total: `0 calls · $0`, four `$0 / 0.0%` rows, a meaningless "Input is 0.0% of
|
|
8347
|
+
* this bill", and — on a log containing a hundred thousand cache-read tokens —
|
|
8348
|
+
* the flatly false "Caching was never used on these calls".
|
|
8349
|
+
*
|
|
8350
|
+
* Two affirmatively wrong claims and a $0 headline for a real bill. The trailing
|
|
8351
|
+
* unpriced note was the only correct line on screen, and it was the quietest.
|
|
7703
8352
|
*/
|
|
7704
|
-
if (report.
|
|
8353
|
+
if (report.total.calls === 0) {
|
|
7705
8354
|
console.log();
|
|
7706
|
-
console.log(
|
|
7707
|
-
|
|
7708
|
-
|
|
7709
|
-
console.log();
|
|
7710
|
-
console.log(` ${c.yellow('!')} ${c.bold(wrap(t.profile.repeatsFound(label, row.modelName, n(row.repeats), n(row.checkedCalls), n(Math.round(row.withinMs / 1000)), formatUsd(row.usd)), 74, ' '))}`);
|
|
7711
|
-
console.log(` ${c.dim(wrap(t.profile.repeatsAdvice(), 74, ' '))}`);
|
|
7712
|
-
}
|
|
8355
|
+
console.log(c.dim(report.unpriced.calls === 0 ? t.profile.empty() : t.profile.nothingPriced()));
|
|
8356
|
+
reportProfileGaps(report, t, n, pricingStale);
|
|
8357
|
+
return;
|
|
7713
8358
|
}
|
|
8359
|
+
const shares = sharesOf(report.total);
|
|
8360
|
+
const parts = [
|
|
8361
|
+
[t.profile.partInput(), report.total.inputUsd, shares.input, report.total.inputTokens],
|
|
8362
|
+
[t.profile.partCacheRead(), report.total.cacheReadUsd, shares.cacheRead, report.total.cacheReadTokens],
|
|
8363
|
+
[t.profile.partCacheWrite(), report.total.cacheWriteUsd, shares.cacheWrite, report.total.cacheWriteTokens],
|
|
8364
|
+
[t.profile.partOutput(), report.total.outputUsd, shares.output, report.total.outputTokens],
|
|
8365
|
+
];
|
|
8366
|
+
console.log();
|
|
8367
|
+
console.log(sectionHeading(t.profile.heading()));
|
|
8368
|
+
console.log(` ${t.profile.spent(t.profile.calls(report.total.calls), formatUsd(report.total.totalUsd))}`);
|
|
7714
8369
|
/**
|
|
7715
|
-
*
|
|
7716
|
-
*
|
|
7717
|
-
*
|
|
7718
|
-
*
|
|
7719
|
-
*
|
|
7720
|
-
*
|
|
7721
|
-
* the missing-field message, because silence there would read as a clean bill
|
|
7722
|
-
* of health on a question the log never asked.
|
|
8370
|
+
* The period, when the log carries a clock — stated, never extrapolated. A
|
|
8371
|
+
* span makes the reader's own monthly arithmetic valid; a per-month figure
|
|
8372
|
+
* printed from a partial month would be this tool doing the guessing it
|
|
8373
|
+
* exists to end. Partial coverage is said in the same breath, because a span
|
|
8374
|
+
* over a third of the calls silently presented as the log's period is a
|
|
8375
|
+
* figure attributed to something it does not describe.
|
|
7723
8376
|
*/
|
|
7724
|
-
if (report.
|
|
7725
|
-
|
|
7726
|
-
|
|
7727
|
-
|
|
7728
|
-
|
|
7729
|
-
|
|
7730
|
-
* that is simply wrong; 1% is a long tail, and the two call for opposite
|
|
7731
|
-
* responses.
|
|
7732
|
-
*
|
|
7733
|
-
* The rate is over calls that **recorded a stop reason**, never over all
|
|
7734
|
-
* calls: a workload that logs the field on half its traffic must not be
|
|
7735
|
-
* reported as though the unmeasured half completed. Both numbers print,
|
|
7736
|
-
* so the denominator is visible rather than implied.
|
|
7737
|
-
*/
|
|
7738
|
-
const truncatedLabels = report.byLabel
|
|
7739
|
-
.filter((row) => row.breakdown.truncatedCalls > 0)
|
|
7740
|
-
.sort((a, b) => b.breakdown.truncatedOutputUsd - a.breakdown.truncatedOutputUsd);
|
|
7741
|
-
if (truncatedLabels.length > 0 && report.byLabel.length > 1) {
|
|
7742
|
-
for (const row of truncatedLabels.slice(0, 3)) {
|
|
7743
|
-
const name = row.label === UNLABELLED ? t.profile.unlabelled() : row.label;
|
|
7744
|
-
console.log(` ${c.dim(wrap(t.profile.truncatedBy(name, n(row.breakdown.truncatedCalls), n(row.breakdown.stopReasonCalls), pct(row.breakdown.truncatedCalls / row.breakdown.stopReasonCalls), formatUsd(row.breakdown.truncatedOutputUsd)), 74, ' '))}`);
|
|
7745
|
-
}
|
|
7746
|
-
}
|
|
7747
|
-
/**
|
|
7748
|
-
* The ceiling the completed answers actually needed, when the output
|
|
7749
|
-
* shapes measured it: "95% of the answers that finished fit within N
|
|
7750
|
-
* tokens" is the number a max_tokens cap wants, and it sits next to the
|
|
7751
|
-
* evidence that the current cap is too low. Measured on these calls,
|
|
7752
|
-
* promised for nothing.
|
|
7753
|
-
*/
|
|
7754
|
-
const ceiling = report.outputShapes.find((shape) => shape.p95WithinTokens !== null);
|
|
7755
|
-
if (ceiling !== undefined) {
|
|
7756
|
-
console.log(` ${c.dim(wrap(t.profile.truncatedCeiling(n(ceiling.p95WithinTokens)), 74, ' '))}`);
|
|
7757
|
-
}
|
|
7758
|
-
/**
|
|
7759
|
-
* The "billed again" half, measured. The sentence above has always said
|
|
7760
|
-
* truncated answers are frequently retried; this is the count and the
|
|
7761
|
-
* money, when the log can carry it — a pattern, never a certainty, and
|
|
7762
|
-
* attributed to the truncated call's slice, where the ceiling that
|
|
7763
|
-
* caused it lives.
|
|
7764
|
-
*/
|
|
7765
|
-
for (const row of report.truncationRetries.slice(0, 3)) {
|
|
7766
|
-
const name = row.label === UNLABELLED ? t.profile.unlabelled() : row.label;
|
|
7767
|
-
console.log(` ${c.yellow('!')} ${c.bold(wrap(t.profile.truncationRetryLine(name, row.modelName, n(row.retried), n(row.truncatedCalls), n(Math.round(row.withinMs / 1000)), formatUsd(row.wastedUsd), formatUsd(row.retryUsd)), 74, ' '))}`);
|
|
7768
|
-
}
|
|
7769
|
-
if (report.truncationRetries.length > 0) {
|
|
7770
|
-
console.log(` ${c.dim(wrap(t.profile.truncationRetryNote(), 74, ' '))}`);
|
|
7771
|
-
}
|
|
8377
|
+
if (report.span !== null) {
|
|
8378
|
+
const totalParsed = report.total.calls + report.unpriced.calls;
|
|
8379
|
+
const partial = report.span.calls < totalParsed
|
|
8380
|
+
? ` ${t.profile.spanPartial(n(report.span.calls), n(totalParsed))}`
|
|
8381
|
+
: '';
|
|
8382
|
+
console.log(` ${c.dim(wrap(`${t.profile.spanLine(dayOf(report.span.fromMs), dayOf(report.span.toMs), spanDays(report.span.fromMs, report.span.toMs))}${partial}`, 74, ' '))}`);
|
|
7772
8383
|
}
|
|
7773
|
-
|
|
7774
|
-
|
|
7775
|
-
|
|
8384
|
+
// How many files this report covers, when it covers more than one: a total
|
|
8385
|
+
// over "the logs" that silently skipped one is wrong by an unknown amount.
|
|
8386
|
+
if (logFiles.length > 1) {
|
|
8387
|
+
console.log(` ${c.dim(wrap(t.profile.readFiles(logFiles.length, path), 74, ' '))}`);
|
|
7776
8388
|
}
|
|
7777
8389
|
/**
|
|
7778
|
-
*
|
|
8390
|
+
* A doubled bill, said before anything is believed.
|
|
7779
8391
|
*
|
|
7780
|
-
*
|
|
7781
|
-
*
|
|
7782
|
-
*
|
|
7783
|
-
*
|
|
7784
|
-
*
|
|
7785
|
-
*
|
|
8392
|
+
* Reading a directory of rotated logs makes double-counting easy — a log
|
|
8393
|
+
* exported twice, an overlapping export, a copy left in the folder — and
|
|
8394
|
+
* the total then reads high with nothing else able to see it. Only counted
|
|
8395
|
+
* over records with a clock, where an identical line is a claim worth
|
|
8396
|
+
* making. It states the count and the money and stops: whether it is a
|
|
8397
|
+
* double export or a genuinely busy millisecond is the reader's to know.
|
|
7786
8398
|
*/
|
|
7787
|
-
if (
|
|
7788
|
-
console.log();
|
|
7789
|
-
|
|
7790
|
-
|
|
7791
|
-
|
|
7792
|
-
|
|
7793
|
-
|
|
7794
|
-
|
|
7795
|
-
|
|
7796
|
-
|
|
7797
|
-
|
|
7798
|
-
|
|
7799
|
-
|
|
7800
|
-
|
|
7801
|
-
// and before the drivers built from it.
|
|
7802
|
-
if (againstOverlap !== null) {
|
|
7803
|
-
console.log(` ${c.yellow('!')} ${c.dim(wrap(t.profile.againstOverlap(dayOf(againstOverlap.fromMs), dayOf(againstOverlap.toMs)), 74, ' '))}`);
|
|
7804
|
-
}
|
|
7805
|
-
// Drivers: per-key contribution to the change, largest magnitude first,
|
|
7806
|
-
// computed once beside the gates so no rendering derives its own.
|
|
7807
|
-
const driverLine = (d, shown) => d.was === null
|
|
7808
|
-
? t.profile.againstDriverNew(formatSignedUsd(d.delta), shown)
|
|
7809
|
-
: d.now === null
|
|
7810
|
-
? t.profile.againstDriverGone(formatSignedUsd(d.delta), shown)
|
|
7811
|
-
: t.profile.againstDriver(formatSignedUsd(d.delta), shown, formatUsd(d.was), formatUsd(d.now));
|
|
7812
|
-
console.log();
|
|
7813
|
-
for (const d of labelDrivers.slice(0, 5)) {
|
|
7814
|
-
const line = driverLine(d, d.key === UNLABELLED ? t.profile.unlabelled() : d.key);
|
|
7815
|
-
console.log(` ${d.delta > 0 ? c.yellow(line) : c.dim(line)}`);
|
|
7816
|
-
}
|
|
7817
|
-
if (labelDrivers.length > 5) {
|
|
7818
|
-
console.log(` ${c.dim(t.profile.andMoreLabels(labelDrivers.length - 5))}`);
|
|
7819
|
-
}
|
|
7820
|
-
/**
|
|
7821
|
-
* The same change, by model — where the mix moved. The label rows cannot
|
|
7822
|
-
* show it: a workload that kept its name and switched from Haiku to Opus
|
|
7823
|
-
* reads as "chat grew", and the reason is the model. Only printed when
|
|
7824
|
-
* more than one model is involved; with one model on both sides, this
|
|
7825
|
-
* section restates the totals line and says nothing new.
|
|
7826
|
-
*/
|
|
7827
|
-
const modelsInvolved = new Set([
|
|
7828
|
-
...previous.byModel.map((r) => r.model),
|
|
7829
|
-
...report.byModel.map((r) => r.model),
|
|
7830
|
-
]);
|
|
7831
|
-
if (modelDrivers.length > 0 && modelsInvolved.size > 1) {
|
|
7832
|
-
console.log();
|
|
7833
|
-
console.log(` ${c.dim(t.profile.againstByModel())}`);
|
|
7834
|
-
for (const d of modelDrivers.slice(0, 3)) {
|
|
7835
|
-
const line = driverLine(d, d.key);
|
|
7836
|
-
console.log(` ${d.delta > 0 ? c.yellow(line) : c.dim(line)}`);
|
|
7837
|
-
}
|
|
7838
|
-
}
|
|
7839
|
-
/**
|
|
7840
|
-
* What the comparison stopped being able to see.
|
|
7841
|
-
*
|
|
7842
|
-
* Every figure above is dollars, and dollars cannot tell a finding that
|
|
7843
|
-
* was fixed from a finding whose field the log stopped recording — both
|
|
7844
|
-
* are silence. This is the only section that can, so it is loud: a
|
|
7845
|
-
* collapse in coverage invalidates whichever findings depended on it,
|
|
7846
|
-
* and reading the drop as good news is the specific mistake it exists
|
|
7847
|
-
* to prevent.
|
|
7848
|
-
*/
|
|
7849
|
-
const drifts = coverageDrift(previous.fieldCoverage, report.fieldCoverage);
|
|
7850
|
-
if (drifts.length > 0) {
|
|
7851
|
-
console.log();
|
|
7852
|
-
for (const drift of drifts) {
|
|
7853
|
-
const line = t.profile.coverageDrift(t.profile.coverageField(drift.field), pct(drift.was), pct(drift.now));
|
|
7854
|
-
console.log(drift.delta < 0
|
|
7855
|
-
? ` ${c.yellow('!')} ${c.bold(wrap(line, 74, ' '))}`
|
|
7856
|
-
: ` ${c.dim(wrap(line, 74, ' '))}`);
|
|
7857
|
-
/**
|
|
7858
|
-
* Which findings went with it, named. "Some findings are silent" is
|
|
7859
|
-
* not something a reader can act on; knowing that conversation
|
|
7860
|
-
* growth and the cache-TTL fit are now silence rather than absence
|
|
7861
|
-
* tells them exactly which sections of this report to distrust.
|
|
7862
|
-
*/
|
|
7863
|
-
if (drift.delta < 0) {
|
|
7864
|
-
const silenced = t.profile.coverageSilenced(drift.field);
|
|
7865
|
-
if (silenced !== '')
|
|
7866
|
-
console.log(` ${c.dim(wrap(silenced, 72, ' '))}`);
|
|
7867
|
-
}
|
|
7868
|
-
}
|
|
7869
|
-
if (drifts.some((d) => d.delta < 0)) {
|
|
7870
|
-
console.log(` ${c.dim(wrap(t.profile.coverageDriftWhy(), 74, ' '))}`);
|
|
7871
|
-
}
|
|
7872
|
-
}
|
|
8399
|
+
if (report.duplicateLines.count > 0) {
|
|
8400
|
+
console.log(` ${c.yellow('!')} ${c.dim(wrap(t.profile.duplicateLines(report.duplicateLines.count, formatUsd(report.duplicateLines.usd)), 74, ' '))}`);
|
|
8401
|
+
}
|
|
8402
|
+
/**
|
|
8403
|
+
* The window, said before any figure is trusted as "the log": everything
|
|
8404
|
+
* below describes a slice, and a slice presented as the whole is a figure
|
|
8405
|
+
* attributed to something it does not describe. The undated count is loud —
|
|
8406
|
+
* those calls' spend is in the log and not in this report, so the window's
|
|
8407
|
+
* figures are a floor on the period, and only this line says so.
|
|
8408
|
+
*/
|
|
8409
|
+
if (report.timeWindow !== null) {
|
|
8410
|
+
console.log(` ${c.dim(wrap(t.profile.windowLine(stringFlag(args, 'since') ?? '—', stringFlag(args, 'until') ?? '—'), 74, ' '))}`);
|
|
8411
|
+
if (relativeWindow) {
|
|
8412
|
+
console.log(` ${c.dim(wrap(t.profile.windowRelative(), 74, ' '))}`);
|
|
7873
8413
|
}
|
|
7874
|
-
|
|
7875
|
-
|
|
7876
|
-
[t.profile.byLabelHeading(), report.byLabel.map((r) => [r.label === UNLABELLED ? t.profile.unlabelled() : r.label, r.breakdown])],
|
|
7877
|
-
[t.profile.byModelHeading(), report.byModel.map((r) => [r.model, r.breakdown])],
|
|
7878
|
-
]) {
|
|
7879
|
-
if (rows.length <= 1)
|
|
7880
|
-
continue; // One row is the total again, said twice.
|
|
7881
|
-
console.log();
|
|
7882
|
-
console.log(sectionHeading(heading));
|
|
7883
|
-
for (const [name, breakdown] of rows) {
|
|
7884
|
-
const share = report.total.totalUsd > 0 ? breakdown.totalUsd / report.total.totalUsd : 0;
|
|
7885
|
-
console.log(` ${bar(share)} ${t.profile.row(name, formatUsd(breakdown.totalUsd), pct(share), t.profile.calls(breakdown.calls))}`);
|
|
8414
|
+
if (report.timeWindow.undatedExcluded > 0) {
|
|
8415
|
+
console.log(` ${c.yellow(wrap(t.profile.windowUndated(report.timeWindow.undatedExcluded), 74, ' '))}`);
|
|
7886
8416
|
}
|
|
7887
8417
|
}
|
|
8418
|
+
console.log();
|
|
8419
|
+
// Every part, including the zero ones. A row missing because it was zero reads
|
|
8420
|
+
// as a row somebody forgot, and "you are not caching at all" is a finding.
|
|
8421
|
+
for (const [name, usd, share, tokens] of parts) {
|
|
8422
|
+
console.log(` ${bar(share)} ${t.profile.part(name, formatUsd(usd), pct(share), n(tokens))}`);
|
|
8423
|
+
}
|
|
7888
8424
|
/**
|
|
7889
|
-
*
|
|
8425
|
+
* The line the command exists for: which part of the bill to argue with.
|
|
7890
8426
|
*
|
|
7891
|
-
*
|
|
7892
|
-
* and
|
|
7893
|
-
*
|
|
7894
|
-
|
|
7895
|
-
|
|
8427
|
+
* When output is both the biggest part and over half, the two sentences say the
|
|
8428
|
+
* same thing and the second says more — so only the second prints. Reporting a
|
|
8429
|
+
* fact twice in adjacent lines reads as a bug, and it was one.
|
|
8430
|
+
*/
|
|
8431
|
+
const [biggestName, , biggestShare] = parts.reduce((a, b) => (b[1] > a[1] ? b : a));
|
|
8432
|
+
const outputDominates = shares.output > 0.5;
|
|
8433
|
+
console.log();
|
|
8434
|
+
if (outputDominates) {
|
|
8435
|
+
console.log(` ${c.bold(wrap(t.profile.outputDominates(pct(shares.output)), 74, ' '))}`);
|
|
8436
|
+
}
|
|
8437
|
+
else {
|
|
8438
|
+
console.log(` ${c.bold(t.profile.biggestPart(biggestName, pct(biggestShare)))}`);
|
|
8439
|
+
}
|
|
8440
|
+
/**
|
|
8441
|
+
* The most expensive day, with a suspect attached.
|
|
7896
8442
|
*
|
|
7897
|
-
*
|
|
7898
|
-
*
|
|
7899
|
-
*
|
|
8443
|
+
* The shape of a bill over time is the finding the total hides: a steady $3 a
|
|
8444
|
+
* day and a quiet week broken by one $40 spike sum to the same number and call
|
|
8445
|
+
* for opposite responses. Rendered against the **median** day — a mean would
|
|
8446
|
+
* let the spike inflate its own yardstick — and loud only when it clears twice
|
|
8447
|
+
* the median, a threshold stated in the sentence rather than hidden in code.
|
|
7900
8448
|
*/
|
|
8449
|
+
if (report.spendByDay.length >= 2) {
|
|
8450
|
+
const medianUsd = median(report.spendByDay.map((d) => d.usd));
|
|
8451
|
+
const peak = report.spendByDay.reduce((a, b) => (b.usd > a.usd ? b : a));
|
|
8452
|
+
if (medianUsd > 0) {
|
|
8453
|
+
const ratio = (peak.usd / medianUsd).toFixed(1);
|
|
8454
|
+
const line = t.profile.dayPeak(peak.day, formatUsd(peak.usd), ratio);
|
|
8455
|
+
const labelClause = peak.topLabel !== null && report.byLabel.length > 1
|
|
8456
|
+
? ` ${t.profile.dayPeakLabel(peak.topLabel === UNLABELLED ? t.profile.unlabelled() : peak.topLabel, formatUsd(peak.topLabelUsd))}`
|
|
8457
|
+
: '';
|
|
8458
|
+
const loud = peak.usd > 2 * medianUsd;
|
|
8459
|
+
const text = wrap(`${line}${labelClause}`, 74, ' ');
|
|
8460
|
+
console.log(` ${loud ? c.yellow(text) : c.dim(text)}`);
|
|
8461
|
+
}
|
|
8462
|
+
}
|
|
7901
8463
|
/**
|
|
7902
|
-
*
|
|
8464
|
+
* The shape of the day, and what it says about batching.
|
|
7903
8465
|
*
|
|
7904
|
-
*
|
|
7905
|
-
*
|
|
7906
|
-
*
|
|
7907
|
-
* the
|
|
8466
|
+
* Spend packed into the hours a country is awake is interactive traffic
|
|
8467
|
+
* somebody is waiting on; spend spread evenly across twenty-four is
|
|
8468
|
+
* background work — and background work is what the Batch API halves. The
|
|
8469
|
+
* measure is exact and needs no threshold to state: the **fewest hours that
|
|
8470
|
+
* hold 80% of the spend**. Two or three means concentrated; sixteen means
|
|
8471
|
+
* flat.
|
|
7908
8472
|
*
|
|
7909
|
-
*
|
|
7910
|
-
*
|
|
7911
|
-
*
|
|
8473
|
+
* It says what the shape is and stops. Whether a workload can wait is a
|
|
8474
|
+
* product decision Trazum cannot make from counts, so the sentence names
|
|
8475
|
+
* the lever and never claims the saving — the batch figure the levers
|
|
8476
|
+
* section already prints is the one with money attached.
|
|
7912
8477
|
*/
|
|
7913
|
-
{
|
|
7914
|
-
const
|
|
7915
|
-
|
|
7916
|
-
|
|
7917
|
-
|
|
7918
|
-
|
|
7919
|
-
|
|
7920
|
-
|
|
7921
|
-
|
|
7922
|
-
? t.profile.verdictSuccess()
|
|
7923
|
-
: slice.verdict === 'undeclared'
|
|
7924
|
-
? t.profile.verdictUndeclared()
|
|
7925
|
-
: t.profile.verdictOther(),
|
|
7926
|
-
calls: n(slice.calls),
|
|
7927
|
-
spend: formatUsd(slice.usd),
|
|
7928
|
-
}));
|
|
7929
|
-
const w = {
|
|
7930
|
-
value: Math.max(...rows.map((r) => r.value.length), col.outcome.length),
|
|
7931
|
-
verdict: Math.max(...rows.map((r) => r.verdict.length), 0),
|
|
7932
|
-
calls: Math.max(...rows.map((r) => r.calls.length), col.calls.length),
|
|
7933
|
-
spend: Math.max(...rows.map((r) => r.spend.length), col.spend.length),
|
|
7934
|
-
};
|
|
7935
|
-
console.log(c.dim(` ${col.outcome.padEnd(w.value)} ${''.padEnd(w.verdict)} ` +
|
|
7936
|
-
`${col.calls.padStart(w.calls)} ${col.spend.padStart(w.spend)}`));
|
|
7937
|
-
for (const row of rows) {
|
|
7938
|
-
const tint = row.verdict === t.profile.verdictUndeclared()
|
|
7939
|
-
? c.yellow
|
|
7940
|
-
: row.verdict === t.profile.verdictSuccess()
|
|
7941
|
-
? c.green
|
|
7942
|
-
: c.dim;
|
|
7943
|
-
console.log(` ${row.value.padEnd(w.value)} ${tint(row.verdict.padEnd(w.verdict))} ` +
|
|
7944
|
-
`${row.calls.padStart(w.calls)} ${row.spend.padStart(w.spend)}`);
|
|
7945
|
-
}
|
|
7946
|
-
console.log();
|
|
7947
|
-
if (outcomes.successShareOfRecordedUsd !== null) {
|
|
7948
|
-
const declaredUsd = outcomes.slices.reduce((sum, slice) => sum + slice.usd, 0);
|
|
7949
|
-
console.log(` ${wrap(t.profile.outcomeRate(pct(outcomes.successShareOfRecordedUsd), formatUsd(declaredUsd)), 74, ' ')}`);
|
|
7950
|
-
}
|
|
7951
|
-
else if (outcomes.noRate !== null) {
|
|
7952
|
-
console.log(` ${c.dim(wrap(t.profile.outcomeNoRate(outcomes.noRate), 74, ' '))}`);
|
|
7953
|
-
}
|
|
7954
|
-
// What the rate does not cover, every time it is printed. A rate over a
|
|
7955
|
-
// twelfth of the bill is a rate about a twelfth of the bill.
|
|
7956
|
-
if (outcomes.coverage.unrecordedUsd > 0 && report.total.totalUsd > 0) {
|
|
7957
|
-
console.log(` ${c.yellow('!')} ${wrap(t.profile.outcomeUnrecorded(pct(outcomes.coverage.unrecordedUsd / report.total.totalUsd), formatUsd(outcomes.coverage.unrecordedUsd)), 74, ' ')}`);
|
|
7958
|
-
}
|
|
7959
|
-
/**
|
|
7960
|
-
* What an outcome costs, per workload — the finding a total cannot make.
|
|
7961
|
-
*
|
|
7962
|
-
* Printed only when at least one workload has enough recorded outcomes
|
|
7963
|
-
* to say something, and every row that cannot state a figure says which
|
|
7964
|
-
* of the five reasons applies rather than showing a blank.
|
|
7965
|
-
*/
|
|
7966
|
-
const ranking = rankPerOutcome(report.outcomeTallyByLabel.map((entry) => ({
|
|
7967
|
-
key: entry.label,
|
|
7968
|
-
calls: entry.calls,
|
|
7969
|
-
totalUsd: entry.totalUsd,
|
|
7970
|
-
tally: entry.tally,
|
|
7971
|
-
})), config.outcomes ?? null);
|
|
7972
|
-
if (ranking.byCall.length > 0) {
|
|
7973
|
-
console.log();
|
|
7974
|
-
console.log(sectionHeading(t.profile.perOutcomeHeading()));
|
|
7975
|
-
const pcol = t.profile.perOutcomeColumns;
|
|
7976
|
-
const prows = ranking.byCall.map((slice) => ({
|
|
7977
|
-
key: slice.key,
|
|
7978
|
-
perCall: formatUsd(slice.usdPerCall),
|
|
7979
|
-
perOutcome: slice.per.usdPerSuccess !== null
|
|
7980
|
-
? formatUsd(slice.per.usdPerSuccess)
|
|
7981
|
-
: t.profile.perOutcomeWithheld(slice.per.withheld ?? 'no-vocabulary', n(slice.per.successes), pct(slice.per.coverage)),
|
|
7982
|
-
recorded: pct(slice.per.coverage),
|
|
7983
|
-
}));
|
|
7984
|
-
const pw = {
|
|
7985
|
-
key: Math.max(...prows.map((r) => r.key.length), pcol.workload.length),
|
|
7986
|
-
perCall: Math.max(...prows.map((r) => r.perCall.length), pcol.perCall.length),
|
|
7987
|
-
perOutcome: Math.max(...prows.map((r) => r.perOutcome.length), pcol.perOutcome.length),
|
|
7988
|
-
recorded: Math.max(...prows.map((r) => r.recorded.length), pcol.recorded.length),
|
|
7989
|
-
};
|
|
7990
|
-
console.log(c.dim(` ${pcol.workload.padEnd(pw.key)} ${pcol.perCall.padStart(pw.perCall)} ` +
|
|
7991
|
-
`${pcol.perOutcome.padStart(pw.perOutcome)} ${pcol.recorded.padStart(pw.recorded)}`));
|
|
7992
|
-
for (const row of prows) {
|
|
7993
|
-
console.log(` ${row.key.padEnd(pw.key)} ${row.perCall.padStart(pw.perCall)} ` +
|
|
7994
|
-
`${row.perOutcome.padStart(pw.perOutcome)} ${c.dim(row.recorded.padStart(pw.recorded))}`);
|
|
7995
|
-
}
|
|
7996
|
-
console.log();
|
|
7997
|
-
console.log(` ${c.dim(wrap(t.profile.perOutcomeNumerator(), 74, ' '))}`);
|
|
7998
|
-
// The disagreement between the two orders, which is itself the finding.
|
|
7999
|
-
if (ranking.disagreements.length > 0) {
|
|
8000
|
-
console.log();
|
|
8001
|
-
console.log(` ${c.dim(wrap(t.profile.perOutcomeBothOrders(), 74, ' '))}`);
|
|
8002
|
-
for (const d of ranking.disagreements) {
|
|
8003
|
-
console.log(` ${c.yellow('→')} ${wrap(t.profile.perOutcomeDisagreement(d.slice.key, n(d.callRank + 1), n(d.outcomeRank + 1)), 74, ' ')}`);
|
|
8004
|
-
}
|
|
8005
|
-
}
|
|
8006
|
-
}
|
|
8007
|
-
if (outcomes.undeclared.length > 0) {
|
|
8008
|
-
console.log(` ${c.yellow('!')} ${wrap(t.profile.outcomeUndeclared(outcomes.undeclared.map((s) => s.value).join(', ')), 74, ' ')}`);
|
|
8009
|
-
}
|
|
8478
|
+
if (report.spendByHour.length >= 4 && report.total.totalUsd > 0) {
|
|
8479
|
+
const sorted = [...report.spendByHour].sort((a, b) => b.usd - a.usd);
|
|
8480
|
+
let covered = 0;
|
|
8481
|
+
let hoursForMost = 0;
|
|
8482
|
+
for (const hour of sorted) {
|
|
8483
|
+
covered += hour.usd;
|
|
8484
|
+
hoursForMost += 1;
|
|
8485
|
+
if (covered >= 0.8 * report.total.totalUsd)
|
|
8486
|
+
break;
|
|
8010
8487
|
}
|
|
8488
|
+
const busiest = sorted
|
|
8489
|
+
.slice(0, hoursForMost)
|
|
8490
|
+
.map((hour) => hour.hour)
|
|
8491
|
+
.sort((a, b) => a - b)
|
|
8492
|
+
.map((hour) => `${String(hour).padStart(2, '0')}:00`)
|
|
8493
|
+
.join(', ');
|
|
8494
|
+
console.log();
|
|
8495
|
+
console.log(` ${c.dim(wrap(hoursForMost <= 8 ? t.profile.hoursConcentrated(n(hoursForMost), busiest) : t.profile.hoursFlat(n(hoursForMost)), 74, ' '))}`);
|
|
8011
8496
|
}
|
|
8012
|
-
|
|
8013
|
-
|
|
8014
|
-
|
|
8015
|
-
|
|
8016
|
-
|
|
8017
|
-
|
|
8018
|
-
|
|
8019
|
-
|
|
8020
|
-
|
|
8021
|
-
|
|
8022
|
-
|
|
8023
|
-
|
|
8024
|
-
|
|
8025
|
-
|
|
8026
|
-
|
|
8027
|
-
|
|
8028
|
-
|
|
8029
|
-
|
|
8030
|
-
|
|
8031
|
-
|
|
8032
|
-
|
|
8033
|
-
|
|
8034
|
-
|
|
8035
|
-
|
|
8036
|
-
if (coverage.cacheWrites > 0 && coverage.cacheTtl < coverage.cacheWrites) {
|
|
8037
|
-
missing.push(t.profile.needsCacheTtl(`${n(coverage.cacheTtl)}/${n(coverage.cacheWrites)}`));
|
|
8038
|
-
}
|
|
8039
|
-
if (missing.length > 0) {
|
|
8040
|
-
console.log();
|
|
8041
|
-
console.log(sectionHeading(t.profile.coverageHeading()));
|
|
8042
|
-
for (const line of missing)
|
|
8043
|
-
console.log(` ${c.dim(wrap(line, 74, ' '))}`);
|
|
8497
|
+
reportCacheVerdict(report, t);
|
|
8498
|
+
await reportLabelsBudgetsAndCache(report, config, pricing, t);
|
|
8499
|
+
reportLevers(report, pricing, t);
|
|
8500
|
+
reportWhatIf(whatIf, report, t);
|
|
8501
|
+
reportConversationHistory(report, t);
|
|
8502
|
+
reportOutputShape(report, t);
|
|
8503
|
+
reportInputShape(report, t);
|
|
8504
|
+
reportContextPressure(pressures, t);
|
|
8505
|
+
reportMixDrift(report, t);
|
|
8506
|
+
reportRepeatedCalls(report, t);
|
|
8507
|
+
reportAgainstPrevious({
|
|
8508
|
+
report, previous, againstOverlap, labelDrivers, modelDrivers, gates, now, t,
|
|
8509
|
+
});
|
|
8510
|
+
for (const [heading, rows] of [
|
|
8511
|
+
[t.profile.byLabelHeading(), report.byLabel.map((r) => [r.label === UNLABELLED ? t.profile.unlabelled() : r.label, r.breakdown])],
|
|
8512
|
+
[t.profile.byModelHeading(), report.byModel.map((r) => [r.model, r.breakdown])],
|
|
8513
|
+
]) {
|
|
8514
|
+
if (rows.length <= 1)
|
|
8515
|
+
continue; // One row is the total again, said twice.
|
|
8516
|
+
console.log();
|
|
8517
|
+
console.log(sectionHeading(heading));
|
|
8518
|
+
for (const [name, breakdown] of rows) {
|
|
8519
|
+
const share = report.total.totalUsd > 0 ? breakdown.totalUsd / report.total.totalUsd : 0;
|
|
8520
|
+
console.log(` ${bar(share)} ${t.profile.row(name, formatUsd(breakdown.totalUsd), pct(share), t.profile.calls(breakdown.calls))}`);
|
|
8044
8521
|
}
|
|
8045
8522
|
}
|
|
8523
|
+
reportOutcomesAndCoverage(report, config, t);
|
|
8046
8524
|
reportProfileGaps(report, t, n, pricingStale);
|
|
8047
|
-
|
|
8048
|
-
|
|
8049
|
-
|
|
8525
|
+
gates = await runProfileGates({
|
|
8526
|
+
args, config, configDir, report, previous, againstDelta, windowed, now, pricing, t,
|
|
8527
|
+
});
|
|
8528
|
+
await writeProfileSideFiles({
|
|
8529
|
+
args, path, report, previous, pressures, labelDrivers, modelDrivers, againstOverlap,
|
|
8530
|
+
whatIf, pricingStale, windowed, gates, pricing, t,
|
|
8531
|
+
});
|
|
8050
8532
|
}
|
|
8051
8533
|
/**
|
|
8052
8534
|
* What the profile could not account for, said out loud.
|
|
@@ -9634,12 +10116,18 @@ async function main() {
|
|
|
9634
10116
|
case 'from-claude-code':
|
|
9635
10117
|
await commandFromClaudeCode(args, t);
|
|
9636
10118
|
break;
|
|
10119
|
+
case 'receipt':
|
|
10120
|
+
await commandReceipt(args, pricing, t);
|
|
10121
|
+
break;
|
|
9637
10122
|
case 'from-otel':
|
|
9638
10123
|
await commandFromOtel(args, t);
|
|
9639
10124
|
break;
|
|
9640
10125
|
case 'from-litellm':
|
|
9641
10126
|
await commandFromLiteLlm(args, t);
|
|
9642
10127
|
break;
|
|
10128
|
+
case 'from-langsmith':
|
|
10129
|
+
await commandFromLangsmith(args, t);
|
|
10130
|
+
break;
|
|
9643
10131
|
case 'from-helicone':
|
|
9644
10132
|
await commandFromHelicone(args, t);
|
|
9645
10133
|
break;
|
|
@@ -9683,9 +10171,47 @@ async function main() {
|
|
|
9683
10171
|
throw new Error(t.errors.unknownCommand(args.command));
|
|
9684
10172
|
}
|
|
9685
10173
|
}
|
|
10174
|
+
/**
|
|
10175
|
+
* A filesystem error, said in this product's voice.
|
|
10176
|
+
*
|
|
10177
|
+
* `optimize`, `check`, `profile`, `position`, `diff`, `semantic` and `conform`
|
|
10178
|
+
* all read a path the reader typed, and all seven let Node's own error through:
|
|
10179
|
+
* `ENOENT: no such file or directory, open '/nope/x.txt'` on the commonest
|
|
10180
|
+
* mistake there is. The five converters `stat` first and answer
|
|
10181
|
+
* `/nope/x.json: not found`, so the CLI disagreed with itself about how to
|
|
10182
|
+
* refuse, and the majority spelling was the one that names a syscall.
|
|
10183
|
+
*
|
|
10184
|
+
* Translated here rather than at each of the seven, because the seven is the
|
|
10185
|
+
* part that changes: a command added next year reads a path too, and a fix
|
|
10186
|
+
* applied per call site is one somebody has to remember. A converter's own
|
|
10187
|
+
* message is more specific and is thrown earlier, so it still wins.
|
|
10188
|
+
*/
|
|
10189
|
+
function filesystemRefusal(error, t) {
|
|
10190
|
+
if (!(error instanceof Error) || !('code' in error))
|
|
10191
|
+
return null;
|
|
10192
|
+
const path = 'path' in error && typeof error.path === 'string' ? error.path : null;
|
|
10193
|
+
/*
|
|
10194
|
+
`readFile` on a directory fails at `read` rather than `open`, and Node
|
|
10195
|
+
attaches no path to that one. The refusal still has to arrive with
|
|
10196
|
+
something in it, so it names what went wrong without naming which file.
|
|
10197
|
+
*/
|
|
10198
|
+
if (path === null)
|
|
10199
|
+
return error.code === 'EISDIR' ? t.errors.fileIsDirectoryUnnamed() : null;
|
|
10200
|
+
switch (error.code) {
|
|
10201
|
+
case 'ENOENT':
|
|
10202
|
+
return t.errors.fileNotFound(path);
|
|
10203
|
+
case 'EISDIR':
|
|
10204
|
+
return t.errors.fileIsDirectory(path);
|
|
10205
|
+
case 'EACCES':
|
|
10206
|
+
case 'EPERM':
|
|
10207
|
+
return t.errors.fileNotReadable(path);
|
|
10208
|
+
default:
|
|
10209
|
+
return null;
|
|
10210
|
+
}
|
|
10211
|
+
}
|
|
9686
10212
|
main().catch((error) => {
|
|
9687
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
9688
10213
|
const t = getCliMessages(localeFromArgv(process.argv.slice(2)));
|
|
10214
|
+
const message = filesystemRefusal(error, t) ?? (error instanceof Error ? error.message : String(error));
|
|
9689
10215
|
console.error(`\n${c.red(t.errors.errorLabel())}: ${message}\n`);
|
|
9690
10216
|
process.exitCode = 1;
|
|
9691
10217
|
});
|