@trazum/cli 1.81.0 → 1.83.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/gateway-server.d.ts.map +1 -1
- package/dist/gateway-server.js +14 -0
- package/dist/gateway-server.js.map +1 -1
- package/dist/i18n/en.d.ts.map +1 -1
- package/dist/i18n/en.js +127 -6
- package/dist/i18n/en.js.map +1 -1
- package/dist/i18n/es.d.ts.map +1 -1
- package/dist/i18n/es.js +133 -6
- package/dist/i18n/es.js.map +1 -1
- package/dist/i18n/types.d.ts +137 -7
- package/dist/i18n/types.d.ts.map +1 -1
- package/dist/index.d.ts +54 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2502 -1639
- package/dist/index.js.map +1 -1
- package/dist/markdown.d.ts +10 -0
- package/dist/markdown.d.ts.map +1 -1
- package/dist/markdown.js +4 -4
- package/dist/markdown.js.map +1 -1
- package/package.json +2 -2
- package/src/gateway-server.ts +14 -0
- package/src/i18n/en.ts +160 -8
- package/src/i18n/es.ts +166 -8
- package/src/i18n/types.ts +141 -7
- package/src/index.ts +3108 -2132
- package/src/markdown.ts +14 -4
package/dist/index.js
CHANGED
|
@@ -7,7 +7,7 @@ import { createInterface } from 'node:readline/promises';
|
|
|
7
7
|
import { dirname, join, resolve as resolvePath } from 'node:path';
|
|
8
8
|
import { fileURLToPath } from 'node:url';
|
|
9
9
|
import { gunzipSync } from 'node:zlib';
|
|
10
|
-
import { CONTRACT_NAMES, contractSchema, applyRewrites, BASELINE_FILENAME, BASELINE_VERSION, breaches, cacheableMinimum, analyzeCachePrefix, billLevers, bucketedCacheEconomics, bucketedProfile, buildHistory, buildPlan, connectorFor, CONNECTORS, normalizeAnthropicUsage, normalizeOpenAIUsage, bucketsFromRecords, evaluateWatch, firedKey, pruneRecords, recordsFromBuckets, storeInventory, storedReportFrom, verifyPlan, cacheEconomics, cacheHitRate, contextPressure, comparePrompts, compareToBaseline, computeSavings, bandGoverns, countTokensAnthropic, DEFAULT_USAGE, budgetPositions, conform, BREAK_EVEN_BAND, allocate, annualRecord, replayCommitment, coversTheTerm, runExperiment, qualityGate, semanticPassCost, verifySemanticProposals, SEMANTIC_SYSTEM_PROMPT, ladderPosition, validateLadder, outcomeReport, rankPerOutcome, FAILURE_POLICIES, detectFromSource, matchLocale, parsePlanDocument, waiverDay, waiverHistory, proposeInit, MIN_RATE_DAYS, parseConfig, coverageDrift, driversBetween, explainGateFailure, assignSources, fleetRollup, rollUp, heartbeats, ruleYield, labelCoverage, measuredUsage, gateMargin, GATE_MARGIN_TIGHT, estimateTokens, evaluate, extractPrompts, findExamples, formatBaseline, formatSignedUsd, formatUsd, getMessages, getModel, hasMarker, LOCALES, MAX_BASELINE_BYTES, matchGlob, MAX_INPUT_CHARS, moneyIsComparable, mostSpecificMatch, assemble, interview, nearestName, slot, SLOT_IDS, optimize, parseBaseline, PHRASE_LANGUAGES, detectTextLanguage, dictionaryStanding, languagesWithStanding, indexUsage, parseUsageLine, plannedCalls, claudeCodeRecords, otelRecords, ownRate, positionAt, positionReport, profilePrompt, profileToCsv, profileUsage, promptId, providerFromEnv, pruneExamples, refineWithLlm, rejectionText, reorderForCache, repriceProfile, switchAnalysis, reviewAgeDays, reviewExamples, RULES, sharedPrefixes, sharesOf, SOURCE_EXTENSIONS, suggestRewrites, toOtlpMetrics, toPromptfoo, TTL_1H_MS, UNLABELLED, withExactTokenCounts, } from '@trazum/core';
|
|
10
|
+
import { CONTRACT_NAMES, contractSchema, applyRewrites, BASELINE_FILENAME, BASELINE_VERSION, breaches, cacheableMinimum, analyzeCachePrefix, billLevers, bucketedCacheEconomics, bucketedProfile, buildHistory, buildPlan, connectorFor, CONNECTORS, normalizeAnthropicUsage, normalizeOpenAIUsage, bucketsFromRecords, evaluateWatch, firedKey, pruneRecords, recordsFromBuckets, storeInventory, storedReportFrom, verifyPlan, cacheEconomics, cacheHitRate, contextPressure, comparePrompts, compareToBaseline, computeSavings, bandGoverns, countTokensAnthropic, DEFAULT_USAGE, budgetPositions, conform, BREAK_EVEN_BAND, allocate, annualRecord, replayCommitment, coversTheTerm, runExperiment, qualityGate, semanticPassCost, verifySemanticProposals, SEMANTIC_SYSTEM_PROMPT, ladderPosition, validateLadder, outcomeReport, rankPerOutcome, FAILURE_POLICIES, detectFromSource, matchLocale, parsePlanDocument, waiverDay, waiverHistory, proposeInit, MIN_RATE_DAYS, parseConfig, coverageDrift, driversBetween, explainGateFailure, assignSources, fleetRollup, rollUp, heartbeats, ruleYield, labelCoverage, measuredUsage, gateMargin, GATE_MARGIN_TIGHT, estimateTokens, evaluate, extractPrompts, findExamples, formatBaseline, formatSignedUsd, formatUsd, receiptFrom, getMessages, getModel, hasMarker, LOCALES, MAX_BASELINE_BYTES, matchGlob, MAX_INPUT_CHARS, moneyIsComparable, mostSpecificMatch, assemble, interview, nearestName, slot, SLOT_IDS, optimize, parseBaseline, PHRASE_LANGUAGES, detectTextLanguage, dictionaryStanding, languagesWithStanding, indexUsage, parseUsageLine, plannedCalls, claudeCodeRecords, ESTIMATE_ERROR_BAND_PCT, bandFor, foreignTokenizer, measuredForeignError, heliconeRecords, langsmithRecords, litellmRecords, otelRecords, ownRate, positionAt, positionReport, PRICING_LAST_REVIEWED, PROVIDER_REVIEWED, STALE_PRICING_DAYS, profilePrompt, profileToCsv, profileUsage, promptId, providerFromEnv, pruneExamples, refineWithLlm, rejectionText, reorderForCache, repriceProfile, switchAnalysis, reviewAgeDays, reviewExamples, RULES, sharedPrefixes, sharesOf, isOffered, SOURCE_EXTENSIONS, suggestRewrites, toOtlpMetrics, toPromptfoo, TTL_1H_MS, UNLABELLED, withExactTokenCounts, } from '@trazum/core';
|
|
11
11
|
import { cacheDir, cacheStats, cachingProvider, clearCache } from './suggest-cache.js';
|
|
12
12
|
import { dayOf, formatGap, median, spanDays } from './time.js';
|
|
13
13
|
// Everything that reads the filesystem, on its own entry point so the web
|
|
@@ -362,7 +362,7 @@ const COMMAND_FLAGS = {
|
|
|
362
362
|
],
|
|
363
363
|
check: ['max-tokens', 'level', 'exact-tokens', 'markdown-out', 'baseline', 'files-from'],
|
|
364
364
|
baseline: ['model', 'calls', 'output-tokens', 'cache-hit-rate', 'batch', 'exact-tokens', 'out', 'o'],
|
|
365
|
-
profile: ['json', 'pricing', 'pricing-live', 'against', 'what-if', 'markdown-out', 'html-out', 'csv-out', 'csv-shape', 'max-usd', 'max-growth-usd', 'max-cache-loss-usd', 'max-day-usd', 'max-session-usd', 'label', 'since', 'until', 'dry-run', 'markdown-summary', 'by-source'],
|
|
365
|
+
profile: ['json', 'pricing', 'pricing-live', 'against', 'what-if', 'markdown-out', 'html-out', 'csv-out', 'csv-shape', 'max-usd', 'max-growth-usd', 'max-cache-loss-usd', 'max-day-usd', 'max-session-usd', 'label', 'since', 'until', 'dry-run', 'markdown-summary', 'by-source', 'allow-empty'],
|
|
366
366
|
plan: ['json', 'out', 'markdown-out', 'min-usd', 'pricing', 'pricing-live'],
|
|
367
367
|
verify: ['against', 'gate', 'json', 'markdown-out', 'pricing', 'pricing-live'],
|
|
368
368
|
history: ['store', 'json', 'markdown-out'],
|
|
@@ -381,8 +381,12 @@ const COMMAND_FLAGS = {
|
|
|
381
381
|
schema: [],
|
|
382
382
|
rollup: ['json', 'html-out'],
|
|
383
383
|
position: ['json', 'html-out', 'pricing', 'pricing-live'],
|
|
384
|
+
receipt: ['out', 'o', 'stamp', 'pricing', 'pricing-live'],
|
|
384
385
|
'from-claude-code': ['label', 'label-from-project', 'out', 'o', 'state'],
|
|
385
386
|
'from-otel': ['label-from-service', 'out', 'o'],
|
|
387
|
+
'from-litellm': ['out', 'o'],
|
|
388
|
+
'from-helicone': ['out', 'o'],
|
|
389
|
+
'from-langsmith': ['out', 'o'],
|
|
386
390
|
switch: ['to', 'migration-usd', 'cases'],
|
|
387
391
|
ownrate: ['gpu-usd-hour', 'tokens-per-second', 'utilization'],
|
|
388
392
|
pulse: ['json', 'max-stale-hours'],
|
|
@@ -483,11 +487,11 @@ function renderDiff(before, after, t) {
|
|
|
483
487
|
/**
|
|
484
488
|
* The provider's name when the estimator was not calibrated for it.
|
|
485
489
|
*
|
|
486
|
-
* `estimateTokens` is a heuristic tuned against Claude's tokenizer, and
|
|
487
|
-
*
|
|
488
|
-
*
|
|
489
|
-
*
|
|
490
|
-
*
|
|
490
|
+
* `estimateTokens` is a heuristic tuned against Claude's tokenizer, and every
|
|
491
|
+
* band descends from that. Printing one beside a GPT or Kimi figure states a
|
|
492
|
+
* precision nobody has measured for that family — and since the catalogue grew
|
|
493
|
+
* past Anthropic, that is most of it. Returns null when the model is
|
|
494
|
+
* Anthropic's, where the band is at least the claim it was written for.
|
|
491
495
|
*/
|
|
492
496
|
function offFamilyName(modelId) {
|
|
493
497
|
const provider = getModel(modelId).provider;
|
|
@@ -495,6 +499,17 @@ function offFamilyName(modelId) {
|
|
|
495
499
|
return null;
|
|
496
500
|
return getModel(modelId).displayName;
|
|
497
501
|
}
|
|
502
|
+
/**
|
|
503
|
+
* How far off the estimator has been measured on this model's family.
|
|
504
|
+
*
|
|
505
|
+
* Read from the catalogue's provider rather than from the display name, because
|
|
506
|
+
* the name is what a reader sees and the id is what was measured. Null on the
|
|
507
|
+
* calibrated family and on every family nobody has run — an unmeasured one gets
|
|
508
|
+
* a sentence saying so, never a borrowed figure.
|
|
509
|
+
*/
|
|
510
|
+
function offFamilyError(modelId) {
|
|
511
|
+
return measuredForeignError(foreignTokenizer(getModel(modelId).provider ?? null));
|
|
512
|
+
}
|
|
498
513
|
/**
|
|
499
514
|
* Language codes as names, in the reader's language.
|
|
500
515
|
*
|
|
@@ -559,7 +574,7 @@ namedScenario = false,
|
|
|
559
574
|
measured = null) {
|
|
560
575
|
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
561
576
|
const sourceNote = result.tokenSource === 'heuristic'
|
|
562
|
-
? c.dim(t.report.estimated(offFamilyName(result.usage.model)))
|
|
577
|
+
? c.dim(t.report.estimated(offFamilyName(result.usage.model), bandFor(result.optimized), offFamilyError(result.usage.model)))
|
|
563
578
|
: c.dim(t.report.exactCount());
|
|
564
579
|
/**
|
|
565
580
|
* The largest lever, first.
|
|
@@ -756,7 +771,7 @@ measured = null) {
|
|
|
756
771
|
*
|
|
757
772
|
* `optimize` is the first command anybody runs, and it reports the smallest
|
|
758
773
|
* line item on the bill: measured, about 1% of a monthly figure. Everything
|
|
759
|
-
* that moves
|
|
774
|
+
* that moves 60% to 80% — which model the call goes to, the Batch API,
|
|
760
775
|
* caching, what re-sending the conversation costs — lives in `profile`, which
|
|
761
776
|
* needs a usage log a new reader does not have and has no reason to go looking
|
|
762
777
|
* for.
|
|
@@ -980,7 +995,23 @@ function defaultModelFor(provider, pricing) {
|
|
|
980
995
|
// not an edge one.
|
|
981
996
|
const RANK = { small: 0, mid: 1, large: 2, frontier: 3 };
|
|
982
997
|
const want = RANK[getModel(DEFAULT_USAGE.model).capability] ?? 2;
|
|
983
|
-
|
|
998
|
+
/**
|
|
999
|
+
* Offered first, and a retired model only when the provider has nothing else.
|
|
1000
|
+
*
|
|
1001
|
+
* `isOffered` is the rule everywhere a model is recommended, and this is not
|
|
1002
|
+
* quite that: nothing is being recommended here, a provider's call is being
|
|
1003
|
+
* priced. The distinction matters because three providers in this catalogue
|
|
1004
|
+
* have **only** retired ids today, and dropping them would return null and
|
|
1005
|
+
* fall through to the global default — printing "goes to deepseek / priced as
|
|
1006
|
+
* Claude Opus 5", which is wrong by an order of magnitude in the direction
|
|
1007
|
+
* this function exists to avoid.
|
|
1008
|
+
*
|
|
1009
|
+
* So a retired model's price is still that provider's price and is used as a
|
|
1010
|
+
* last resort. What the reader is not left to infer is that it is current:
|
|
1011
|
+
* `buildAdvisories` warns by name on a retired id, quoting the provider.
|
|
1012
|
+
*/
|
|
1013
|
+
const offered = pricing.models.filter((m) => m.provider === provider && isOffered(m));
|
|
1014
|
+
const candidates = offered.length > 0 ? offered : pricing.models.filter((m) => m.provider === provider);
|
|
984
1015
|
const best = candidates.reduce((chosen, m) => {
|
|
985
1016
|
if (chosen === null)
|
|
986
1017
|
return m;
|
|
@@ -2275,6 +2306,77 @@ async function commandFromClaudeCode(args, t) {
|
|
|
2275
2306
|
console.error(t.fromClaudeCode.labelled());
|
|
2276
2307
|
console.error(t.fromClaudeCode.skipped(otherLines, unparseable, withoutUsage));
|
|
2277
2308
|
}
|
|
2309
|
+
/**
|
|
2310
|
+
* `trazum receipt <usage.jsonl|dir>` — the bill's counts, with the provenance
|
|
2311
|
+
* of every figure attached.
|
|
2312
|
+
*
|
|
2313
|
+
* A profile answers a question on the terminal that ran it. The same figures
|
|
2314
|
+
* filed against an invoice, or read next month by somebody who was not there,
|
|
2315
|
+
* stop answering it: a dollar total with no provenance cannot tell a repricing
|
|
2316
|
+
* from a team whose spend moved. This writes the shape that keeps answering.
|
|
2317
|
+
*
|
|
2318
|
+
* **It sends nothing anywhere.** There is no endpoint, no key, no retry and no
|
|
2319
|
+
* queue: the document goes to a path you name or to standard output, and what
|
|
2320
|
+
* happens to it next is not this command's business. A command in this package
|
|
2321
|
+
* that phoned home would break the roadmap's first rule in the same release
|
|
2322
|
+
* that claims to be protecting it.
|
|
2323
|
+
*
|
|
2324
|
+
* **The document is undated unless you ask for a stamp**, and that is a
|
|
2325
|
+
* deliberate default rather than an omission. This product's first promise is
|
|
2326
|
+
* the same answer every time; a command that wrote the current clock into its
|
|
2327
|
+
* output would produce different bytes on every run, so it could not be
|
|
2328
|
+
* committed, diffed in a pull request, or compared against yesterday's. The
|
|
2329
|
+
* period the figures actually cover is already in the document, read from the
|
|
2330
|
+
* log's own clock. `--stamp` adds the emission time for whoever wants it, and
|
|
2331
|
+
* an undated receipt is valid: `receiptFrom` says so, and says why.
|
|
2332
|
+
*
|
|
2333
|
+
* The offline guard found this. The command stamped by default, two runs
|
|
2334
|
+
* disagreed, and the test that exists to catch a hidden network call caught a
|
|
2335
|
+
* hidden clock instead.
|
|
2336
|
+
*
|
|
2337
|
+
* The redaction property is not enforced here. It is a property of
|
|
2338
|
+
* `receiptFrom`, which takes a `UsageProfileReport` -- a shape with no field
|
|
2339
|
+
* that can hold prompt text -- and it is held by `receipt-redaction.test.js`
|
|
2340
|
+
* planting the 4 things that must never appear. This function only chooses the
|
|
2341
|
+
* log and the destination.
|
|
2342
|
+
*/
|
|
2343
|
+
async function commandReceipt(args, pricing, t) {
|
|
2344
|
+
const file = args.positional[0];
|
|
2345
|
+
if (file === undefined)
|
|
2346
|
+
throw new Error(t.receipt.noLog());
|
|
2347
|
+
const text = await readUsageLog(file, t);
|
|
2348
|
+
const report = profileUsage(text, { catalogue: pricing });
|
|
2349
|
+
const document = receiptFrom(report, pricing, boolFlag(args, 'stamp') ? { emittedAt: new Date() } : {});
|
|
2350
|
+
const json = JSON.stringify(document, null, 2);
|
|
2351
|
+
const out = stringFlag(args, 'out') ?? stringFlag(args, 'o');
|
|
2352
|
+
if (out !== undefined) {
|
|
2353
|
+
await writeFile(out, json + '\n', 'utf8');
|
|
2354
|
+
console.error(t.receipt.written(out, document.lines.length));
|
|
2355
|
+
}
|
|
2356
|
+
else {
|
|
2357
|
+
console.log(json);
|
|
2358
|
+
}
|
|
2359
|
+
/*
|
|
2360
|
+
* Everything below goes to stderr, so `trazum receipt log.jsonl > receipt.json`
|
|
2361
|
+
* writes a document and not a document with a summary stapled to the front.
|
|
2362
|
+
* The gaps are read off the document rather than recomputed, so what the
|
|
2363
|
+
* reader is told and what the file carries cannot drift apart.
|
|
2364
|
+
*/
|
|
2365
|
+
if (document.lines.length === 0) {
|
|
2366
|
+
console.error(t.receipt.nothingToBill());
|
|
2367
|
+
}
|
|
2368
|
+
else {
|
|
2369
|
+
console.error(t.receipt.summary(document.lines.length, formatUsd(document.total.usd)));
|
|
2370
|
+
}
|
|
2371
|
+
for (const gap of document.gaps) {
|
|
2372
|
+
if (gap.kind === 'unpriced')
|
|
2373
|
+
console.error(t.receipt.unpriced(gap.models.length, gap.calls));
|
|
2374
|
+
if (gap.kind === 'unread-lines')
|
|
2375
|
+
console.error(t.receipt.unread(gap.count));
|
|
2376
|
+
if (gap.kind === 'no-clock')
|
|
2377
|
+
console.error(t.receipt.noClock());
|
|
2378
|
+
}
|
|
2379
|
+
}
|
|
2278
2380
|
/**
|
|
2279
2381
|
* `trazum from-otel <file|dir>` — OpenTelemetry GenAI spans as a usage log.
|
|
2280
2382
|
*
|
|
@@ -2477,6 +2579,272 @@ async function commandFromOtel(args, t) {
|
|
|
2477
2579
|
if (unparseable > 0)
|
|
2478
2580
|
console.error(t.fromOtel.unparseable(unparseable));
|
|
2479
2581
|
}
|
|
2582
|
+
/**
|
|
2583
|
+
* `trazum from-litellm <file|dir>` — a LiteLLM spend log as a usage log.
|
|
2584
|
+
*
|
|
2585
|
+
* The same pure-converter pattern again, pointed at the gateway a great many
|
|
2586
|
+
* teams already put in front of every provider: `LiteLLM_SpendLogs` is the
|
|
2587
|
+
* export most likely to already exist on somebody's disk. `litellmRecords` in
|
|
2588
|
+
* core does the reading; this is the walk and the honesty.
|
|
2589
|
+
*
|
|
2590
|
+
* Three of the four counts it prints exist because the alternative is a
|
|
2591
|
+
* flattering silence. Rows naming no model are not priced by the route they
|
|
2592
|
+
* took, rows with no tokens are logged calls nobody can price, and a
|
|
2593
|
+
* `cache_hit` flag is not a token split — so the cache verdicts read "cannot
|
|
2594
|
+
* tell" rather than a fabricated one, exactly as `from-otel` does.
|
|
2595
|
+
*
|
|
2596
|
+
* The fourth is the one that matters most: LiteLLM prices the same calls with
|
|
2597
|
+
* its own table, and that figure is printed beside Trazum's and never merged
|
|
2598
|
+
* into it. Two price tables summed into one total is how a report becomes
|
|
2599
|
+
* quietly wrong.
|
|
2600
|
+
*
|
|
2601
|
+
* The row carries `messages`, `response`, `api_key`, `requester_ip_address`
|
|
2602
|
+
* and `end_user`. None of it is read; a fixture plants a marker in each and
|
|
2603
|
+
* greps the whole output.
|
|
2604
|
+
*/
|
|
2605
|
+
async function commandFromLiteLlm(args, t) {
|
|
2606
|
+
const target = args.positional[0];
|
|
2607
|
+
if (target === undefined)
|
|
2608
|
+
throw new Error(t.fromLiteLlm.noPath());
|
|
2609
|
+
let info;
|
|
2610
|
+
try {
|
|
2611
|
+
info = await stat(target);
|
|
2612
|
+
}
|
|
2613
|
+
catch {
|
|
2614
|
+
throw new Error(t.fromLiteLlm.notFound(target));
|
|
2615
|
+
}
|
|
2616
|
+
const files = [];
|
|
2617
|
+
if (info.isDirectory()) {
|
|
2618
|
+
const entries = await readdir(target, { recursive: true, withFileTypes: true });
|
|
2619
|
+
for (const entry of entries) {
|
|
2620
|
+
if (entry.isFile() && /\.(json|jsonl|ndjson)$/i.test(entry.name)) {
|
|
2621
|
+
files.push(join(entry.parentPath, entry.name));
|
|
2622
|
+
}
|
|
2623
|
+
}
|
|
2624
|
+
files.sort();
|
|
2625
|
+
if (files.length === 0)
|
|
2626
|
+
throw new Error(t.fromLiteLlm.noExports(target));
|
|
2627
|
+
}
|
|
2628
|
+
else {
|
|
2629
|
+
files.push(target);
|
|
2630
|
+
}
|
|
2631
|
+
const lines = [];
|
|
2632
|
+
let rows = 0;
|
|
2633
|
+
let unnamedModel = 0;
|
|
2634
|
+
let noTokens = 0;
|
|
2635
|
+
let cacheFlagged = 0;
|
|
2636
|
+
let unparseable = 0;
|
|
2637
|
+
let reportedSpend = 0;
|
|
2638
|
+
let sawSpend = false;
|
|
2639
|
+
for (const file of files) {
|
|
2640
|
+
const conversion = litellmRecords(await readFile(file, 'utf8'));
|
|
2641
|
+
for (const record of conversion.records)
|
|
2642
|
+
lines.push(JSON.stringify(record));
|
|
2643
|
+
rows += conversion.rows;
|
|
2644
|
+
unnamedModel += conversion.unnamedModel;
|
|
2645
|
+
noTokens += conversion.noTokens;
|
|
2646
|
+
cacheFlagged += conversion.cacheFlagged;
|
|
2647
|
+
unparseable += conversion.unparseable;
|
|
2648
|
+
if (conversion.reportedSpendUsd !== null) {
|
|
2649
|
+
reportedSpend += conversion.reportedSpendUsd;
|
|
2650
|
+
sawSpend = true;
|
|
2651
|
+
}
|
|
2652
|
+
}
|
|
2653
|
+
const out = stringFlag(args, 'out') ?? stringFlag(args, 'o');
|
|
2654
|
+
if (out !== undefined) {
|
|
2655
|
+
await writeFile(out, lines.join('\n') + (lines.length > 0 ? '\n' : ''), 'utf8');
|
|
2656
|
+
console.error(t.fromLiteLlm.written(out));
|
|
2657
|
+
}
|
|
2658
|
+
else {
|
|
2659
|
+
for (const line of lines)
|
|
2660
|
+
console.log(line);
|
|
2661
|
+
}
|
|
2662
|
+
console.error(t.fromLiteLlm.summary(files.length, rows));
|
|
2663
|
+
if (unnamedModel > 0)
|
|
2664
|
+
console.error(t.fromLiteLlm.unnamedModel(unnamedModel));
|
|
2665
|
+
if (noTokens > 0)
|
|
2666
|
+
console.error(t.fromLiteLlm.noTokens(noTokens));
|
|
2667
|
+
if (cacheFlagged > 0)
|
|
2668
|
+
console.error(t.fromLiteLlm.cacheFlagged(cacheFlagged));
|
|
2669
|
+
if (sawSpend)
|
|
2670
|
+
console.error(t.fromLiteLlm.reportedSpend(formatUsd(reportedSpend)));
|
|
2671
|
+
if (unparseable > 0)
|
|
2672
|
+
console.error(t.fromLiteLlm.unparseable(unparseable));
|
|
2673
|
+
}
|
|
2674
|
+
/**
|
|
2675
|
+
* `trazum from-langsmith <file|dir>` — a LangSmith run export as a usage log.
|
|
2676
|
+
*
|
|
2677
|
+
* The fifth converter, and the one where the unit is wrong before anything
|
|
2678
|
+
* else can be right: LangSmith records a **run**, and a trace is a tree of
|
|
2679
|
+
* them. The chain that wrapped a model call carries the same tokens as the
|
|
2680
|
+
* call, so summing the export bills them once per level. Only `run_type: llm`
|
|
2681
|
+
* is a call, and every run that is not one is counted out loud — skipping two
|
|
2682
|
+
* thirds of a file silently would look exactly like reading it.
|
|
2683
|
+
*
|
|
2684
|
+
* The model is refused rather than inferred. There is no model column; the
|
|
2685
|
+
* name lives in the metadata, and the obvious substitute is the run's own
|
|
2686
|
+
* `name`, which LangChain sets to the client class. Pricing a call by
|
|
2687
|
+
* `ChatAnthropic` would attribute a figure to something it does not describe.
|
|
2688
|
+
*
|
|
2689
|
+
* LangSmith's own cost is reported on its own line and never merged into
|
|
2690
|
+
* anything Trazum computes, the way `from-litellm` keeps the gateway's
|
|
2691
|
+
* arithmetic apart. Two price tables summed into one total is how a report
|
|
2692
|
+
* becomes quietly wrong.
|
|
2693
|
+
*/
|
|
2694
|
+
async function commandFromLangsmith(args, t) {
|
|
2695
|
+
const target = args.positional[0];
|
|
2696
|
+
if (target === undefined)
|
|
2697
|
+
throw new Error(t.fromLangsmith.noPath());
|
|
2698
|
+
let info;
|
|
2699
|
+
try {
|
|
2700
|
+
info = await stat(target);
|
|
2701
|
+
}
|
|
2702
|
+
catch {
|
|
2703
|
+
throw new Error(t.fromLangsmith.notFound(target));
|
|
2704
|
+
}
|
|
2705
|
+
const files = [];
|
|
2706
|
+
if (info.isDirectory()) {
|
|
2707
|
+
const entries = await readdir(target, { recursive: true, withFileTypes: true });
|
|
2708
|
+
for (const entry of entries) {
|
|
2709
|
+
if (entry.isFile() && /\.(json|jsonl|ndjson)$/i.test(entry.name)) {
|
|
2710
|
+
files.push(join(entry.parentPath, entry.name));
|
|
2711
|
+
}
|
|
2712
|
+
}
|
|
2713
|
+
files.sort();
|
|
2714
|
+
if (files.length === 0)
|
|
2715
|
+
throw new Error(t.fromLangsmith.noExports(target));
|
|
2716
|
+
}
|
|
2717
|
+
else {
|
|
2718
|
+
files.push(target);
|
|
2719
|
+
}
|
|
2720
|
+
const lines = [];
|
|
2721
|
+
let rows = 0;
|
|
2722
|
+
let notModelCalls = 0;
|
|
2723
|
+
let unnamedModel = 0;
|
|
2724
|
+
let noTokens = 0;
|
|
2725
|
+
let unparseable = 0;
|
|
2726
|
+
let reportedCostUsd = null;
|
|
2727
|
+
for (const file of files) {
|
|
2728
|
+
const conversion = langsmithRecords(await readFile(file, 'utf8'));
|
|
2729
|
+
for (const record of conversion.records)
|
|
2730
|
+
lines.push(JSON.stringify(record));
|
|
2731
|
+
rows += conversion.rows;
|
|
2732
|
+
notModelCalls += conversion.notModelCalls;
|
|
2733
|
+
unnamedModel += conversion.unnamedModel;
|
|
2734
|
+
noTokens += conversion.noTokens;
|
|
2735
|
+
unparseable += conversion.unparseable;
|
|
2736
|
+
if (conversion.reportedCostUsd !== null) {
|
|
2737
|
+
reportedCostUsd = (reportedCostUsd ?? 0) + conversion.reportedCostUsd;
|
|
2738
|
+
}
|
|
2739
|
+
}
|
|
2740
|
+
const out = stringFlag(args, 'out') ?? stringFlag(args, 'o');
|
|
2741
|
+
if (out !== undefined) {
|
|
2742
|
+
await writeFile(out, lines.join('\n') + (lines.length > 0 ? '\n' : ''), 'utf8');
|
|
2743
|
+
console.error(t.fromLangsmith.written(out));
|
|
2744
|
+
}
|
|
2745
|
+
else {
|
|
2746
|
+
for (const line of lines)
|
|
2747
|
+
console.log(line);
|
|
2748
|
+
}
|
|
2749
|
+
console.error(t.fromLangsmith.summary(files.length, rows));
|
|
2750
|
+
if (notModelCalls > 0)
|
|
2751
|
+
console.error(t.fromLangsmith.notModelCalls(notModelCalls));
|
|
2752
|
+
if (unnamedModel > 0)
|
|
2753
|
+
console.error(t.fromLangsmith.unnamedModel(unnamedModel));
|
|
2754
|
+
if (noTokens > 0)
|
|
2755
|
+
console.error(t.fromLangsmith.noTokens(noTokens));
|
|
2756
|
+
if (reportedCostUsd !== null) {
|
|
2757
|
+
console.error(t.fromLangsmith.reportedCost(formatUsd(reportedCostUsd)));
|
|
2758
|
+
}
|
|
2759
|
+
if (unparseable > 0)
|
|
2760
|
+
console.error(t.fromLangsmith.unparseable(unparseable));
|
|
2761
|
+
}
|
|
2762
|
+
/**
|
|
2763
|
+
* `trazum from-helicone <file|dir>` — a Helicone request export as a usage log.
|
|
2764
|
+
*
|
|
2765
|
+
* The third converter, and the one that needs three columns where the others
|
|
2766
|
+
* need one: Helicone carries `request_model`, `model_override` and
|
|
2767
|
+
* `response_model`, and they can disagree. The response wins, because a bill
|
|
2768
|
+
* is about what was billed rather than what was intended, and the
|
|
2769
|
+
* disagreements are counted so a substitution is something the reader sees.
|
|
2770
|
+
*
|
|
2771
|
+
* Two absences are stated rather than filled in. There is no cache token
|
|
2772
|
+
* split on the row, only a flag; and a Helicone request id names one call and
|
|
2773
|
+
* never a conversation, so the records carry no session and the
|
|
2774
|
+
* conversation-shaped findings stay unavailable. Both are printed, because a
|
|
2775
|
+
* gap that is not said reads as a gap that is not there.
|
|
2776
|
+
*/
|
|
2777
|
+
async function commandFromHelicone(args, t) {
|
|
2778
|
+
const target = args.positional[0];
|
|
2779
|
+
if (target === undefined)
|
|
2780
|
+
throw new Error(t.fromHelicone.noPath());
|
|
2781
|
+
let info;
|
|
2782
|
+
try {
|
|
2783
|
+
info = await stat(target);
|
|
2784
|
+
}
|
|
2785
|
+
catch {
|
|
2786
|
+
throw new Error(t.fromHelicone.notFound(target));
|
|
2787
|
+
}
|
|
2788
|
+
const files = [];
|
|
2789
|
+
if (info.isDirectory()) {
|
|
2790
|
+
const entries = await readdir(target, { recursive: true, withFileTypes: true });
|
|
2791
|
+
for (const entry of entries) {
|
|
2792
|
+
if (entry.isFile() && /\.(json|jsonl|ndjson)$/i.test(entry.name)) {
|
|
2793
|
+
files.push(join(entry.parentPath, entry.name));
|
|
2794
|
+
}
|
|
2795
|
+
}
|
|
2796
|
+
files.sort();
|
|
2797
|
+
if (files.length === 0)
|
|
2798
|
+
throw new Error(t.fromHelicone.noExports(target));
|
|
2799
|
+
}
|
|
2800
|
+
else {
|
|
2801
|
+
files.push(target);
|
|
2802
|
+
}
|
|
2803
|
+
const lines = [];
|
|
2804
|
+
let rows = 0;
|
|
2805
|
+
let unnamedModel = 0;
|
|
2806
|
+
let disagreements = 0;
|
|
2807
|
+
let noTokens = 0;
|
|
2808
|
+
let cacheFlagged = 0;
|
|
2809
|
+
let unparseable = 0;
|
|
2810
|
+
for (const file of files) {
|
|
2811
|
+
const conversion = heliconeRecords(await readFile(file, 'utf8'));
|
|
2812
|
+
for (const record of conversion.records)
|
|
2813
|
+
lines.push(JSON.stringify(record));
|
|
2814
|
+
rows += conversion.rows;
|
|
2815
|
+
unnamedModel += conversion.unnamedModel;
|
|
2816
|
+
disagreements += conversion.modelDisagreements;
|
|
2817
|
+
noTokens += conversion.noTokens;
|
|
2818
|
+
cacheFlagged += conversion.cacheFlagged;
|
|
2819
|
+
unparseable += conversion.unparseable;
|
|
2820
|
+
}
|
|
2821
|
+
const out = stringFlag(args, 'out') ?? stringFlag(args, 'o');
|
|
2822
|
+
if (out !== undefined) {
|
|
2823
|
+
await writeFile(out, lines.join('\n') + (lines.length > 0 ? '\n' : ''), 'utf8');
|
|
2824
|
+
console.error(t.fromHelicone.written(out));
|
|
2825
|
+
}
|
|
2826
|
+
else {
|
|
2827
|
+
for (const line of lines)
|
|
2828
|
+
console.log(line);
|
|
2829
|
+
}
|
|
2830
|
+
console.error(t.fromHelicone.summary(files.length, rows));
|
|
2831
|
+
if (unnamedModel > 0)
|
|
2832
|
+
console.error(t.fromHelicone.unnamedModel(unnamedModel));
|
|
2833
|
+
if (disagreements > 0)
|
|
2834
|
+
console.error(t.fromHelicone.disagreements(disagreements));
|
|
2835
|
+
if (noTokens > 0)
|
|
2836
|
+
console.error(t.fromHelicone.noTokens(noTokens));
|
|
2837
|
+
if (cacheFlagged > 0)
|
|
2838
|
+
console.error(t.fromHelicone.cacheFlagged(cacheFlagged));
|
|
2839
|
+
// Said on every run that produced records, not only when something is
|
|
2840
|
+
// missing: the absence is a property of the format, and a reader who has
|
|
2841
|
+
// just converted a month needs to know which questions this log cannot
|
|
2842
|
+
// answer before they go looking for the answers.
|
|
2843
|
+
if (rows > 0)
|
|
2844
|
+
console.error(t.fromHelicone.noSessions());
|
|
2845
|
+
if (unparseable > 0)
|
|
2846
|
+
console.error(t.fromHelicone.unparseable(unparseable));
|
|
2847
|
+
}
|
|
2480
2848
|
/**
|
|
2481
2849
|
* `trazum position <usage.jsonl>` — where the month stands, measured.
|
|
2482
2850
|
*
|
|
@@ -3925,7 +4293,35 @@ function commandModels(t, pricing) {
|
|
|
3925
4293
|
const col = t.models.columns;
|
|
3926
4294
|
console.log();
|
|
3927
4295
|
console.log(sectionHeading(`${t.models.title()}${t.models.unit()}`));
|
|
3928
|
-
|
|
4296
|
+
const now = new Date();
|
|
4297
|
+
console.log(c.dim(t.models.reviewedOn(pricing.lastReviewed, reviewAgeDays(pricing.lastReviewed, now))));
|
|
4298
|
+
/**
|
|
4299
|
+
* The headline date is the oldest provider's, which is the honest answer to
|
|
4300
|
+
* "how old is this table" and the wrong answer to "how old is the price I am
|
|
4301
|
+
* being quoted". A reader pricing Claude calls was told two months when
|
|
4302
|
+
* their half had been checked that morning, so the providers say for
|
|
4303
|
+
* themselves — and only when they disagree, since one date repeated seven
|
|
4304
|
+
* times is noise.
|
|
4305
|
+
*
|
|
4306
|
+
* Suppressed under an overlay: `--pricing` and `--pricing-live` replace
|
|
4307
|
+
* prices with numbers whose provenance is the overlay's own date, and these
|
|
4308
|
+
* are the bundled table's.
|
|
4309
|
+
*/
|
|
4310
|
+
const dates = [...new Set(Object.values(PROVIDER_REVIEWED))];
|
|
4311
|
+
if (dates.length > 1 && pricing.lastReviewed === PRICING_LAST_REVIEWED) {
|
|
4312
|
+
const priced = new Set(pricing.models.map((m) => m.provider));
|
|
4313
|
+
/* Grouped by date: six providers repeating one date is noise, not provenance. */
|
|
4314
|
+
const byDate = new Map();
|
|
4315
|
+
for (const [provider, date] of Object.entries(PROVIDER_REVIEWED)) {
|
|
4316
|
+
if (!priced.has(provider))
|
|
4317
|
+
continue;
|
|
4318
|
+
byDate.set(date, [...(byDate.get(date) ?? []), provider]);
|
|
4319
|
+
}
|
|
4320
|
+
const rows = [...byDate.entries()]
|
|
4321
|
+
.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
|
|
4322
|
+
.map(([date, providers]) => t.models.reviewedGroup(date, reviewAgeDays(date, now), providers.join(', ')));
|
|
4323
|
+
console.log(c.dim(t.models.reviewedByProvider(rows.join(' · '))));
|
|
4324
|
+
}
|
|
3929
4325
|
console.log();
|
|
3930
4326
|
const rows = pricing.models.map((m) => ({
|
|
3931
4327
|
id: m.id,
|
|
@@ -4547,6 +4943,7 @@ async function commandCheck(args, config, pricing, t, locale) {
|
|
|
4547
4943
|
verdicts: [verdict],
|
|
4548
4944
|
level,
|
|
4549
4945
|
tokenSource: counter.source,
|
|
4946
|
+
band: bandFor(prompt),
|
|
4550
4947
|
truncated: false,
|
|
4551
4948
|
t,
|
|
4552
4949
|
}));
|
|
@@ -4654,11 +5051,16 @@ async function checkEmbedded(path, extraction, budget, args, counter, level, t,
|
|
|
4654
5051
|
}
|
|
4655
5052
|
const failures = verdicts.filter(isOverBudget);
|
|
4656
5053
|
const ok = failures.length === 0 && extraction.declined.length === 0;
|
|
5054
|
+
// Widest across the prompts this report covers, never an average: a run over
|
|
5055
|
+
// prose and a CSV covers text the estimator is 6% and 33% out on, and a
|
|
5056
|
+
// figure between them would describe neither.
|
|
5057
|
+
const band = widestBand(extraction.prompts.map((prompt) => prompt.text));
|
|
4657
5058
|
await writeMarkdown(args, () => renderCheckMarkdown({
|
|
4658
5059
|
target: path,
|
|
4659
5060
|
verdicts,
|
|
4660
5061
|
level,
|
|
4661
5062
|
tokenSource: counter.source,
|
|
5063
|
+
band,
|
|
4662
5064
|
truncated: false,
|
|
4663
5065
|
t,
|
|
4664
5066
|
}));
|
|
@@ -5654,564 +6056,264 @@ async function commandVerify(args, pricing, t) {
|
|
|
5654
6056
|
}
|
|
5655
6057
|
}
|
|
5656
6058
|
/**
|
|
5657
|
-
*
|
|
5658
|
-
*
|
|
5659
|
-
*
|
|
5660
|
-
*
|
|
5661
|
-
*
|
|
5662
|
-
*
|
|
5663
|
-
*
|
|
6059
|
+
* `--by-source`: one report per service, plus the rollup, the fleet.
|
|
6060
|
+
*
|
|
6061
|
+
* A merged bill is right for one service and wrong for twelve: it hides which
|
|
6062
|
+
* service the money comes from, per-service budgets cannot exist, and the
|
|
6063
|
+
* findings a comparison between services could make are invisible. Files are
|
|
6064
|
+
* assigned to sources by the most specific matching glob from the config’s
|
|
6065
|
+
* `sources` block; a file matching no source is named loudly, because a log
|
|
6066
|
+
* that silently joined no report is spend missing from every bill.
|
|
6067
|
+
*
|
|
6068
|
+
* Lifted out of `commandProfile` unchanged. It was one of several complete
|
|
6069
|
+
* outputs sharing a 2,359-line body with the report they are alternatives to,
|
|
6070
|
+
* and it needs a fraction of what that body had in scope: naming the fraction
|
|
6071
|
+
* is most of the point of moving it.
|
|
5664
6072
|
*/
|
|
5665
|
-
async function
|
|
5666
|
-
const path = args.positional[0];
|
|
5667
|
-
if (path === undefined)
|
|
5668
|
-
throw new Error(t.plan.noTarget());
|
|
5669
|
-
const GZ = LOG_EXTENSIONS.map((ext) => `${ext}.gz`);
|
|
5670
|
-
const READABLE = [...LOG_EXTENSIONS, ...GZ];
|
|
5671
|
-
const target = await stat(path).catch(() => null);
|
|
5672
|
-
let files = [path];
|
|
5673
|
-
if (target?.isDirectory()) {
|
|
5674
|
-
const entries = await readdir(path, { withFileTypes: true });
|
|
5675
|
-
files = entries
|
|
5676
|
-
.filter((entry) => entry.isFile() && READABLE.some((ext) => entry.name.endsWith(ext)))
|
|
5677
|
-
.map((entry) => join(path, entry.name))
|
|
5678
|
-
.sort((a, b) => a.localeCompare(b));
|
|
5679
|
-
if (files.length === 0)
|
|
5680
|
-
throw new Error(t.profile.noLogsInDirectory(path, READABLE.join(', ')));
|
|
5681
|
-
}
|
|
5682
|
-
const texts = await Promise.all(files.map((file) => readUsageLog(file, t)));
|
|
5683
|
-
const raw = texts.map((text) => (text.endsWith('\n') ? text : `${text}\n`)).join('');
|
|
5684
|
-
const report = profileUsage(raw, { catalogue: pricing });
|
|
5685
|
-
if (report.total.calls === 0)
|
|
5686
|
-
throw new Error(t.plan.nothingPriced());
|
|
5687
|
-
const levers = billLevers(report, { catalogue: pricing });
|
|
5688
|
-
const plan = buildPlan(report, levers, pricing.lastReviewed);
|
|
5689
|
-
const minUsd = typeof args.flags.get('min-usd') === 'string' ? numberFlag(args, 'min-usd', 0, t) : 0;
|
|
5690
|
-
const actions = plan.actions.filter((a) => (a.savingUsd ?? a.stakeUsd ?? 0) >= minUsd);
|
|
5691
|
-
const filtered = plan.actions.length - actions.length;
|
|
5692
|
-
const droppedUsd = plan.actions
|
|
5693
|
-
.filter((a) => (a.savingUsd ?? a.stakeUsd ?? 0) < minUsd)
|
|
5694
|
-
.reduce((sum, a) => sum + (a.savingUsd ?? a.stakeUsd ?? 0), 0);
|
|
6073
|
+
async function profileBySource(args, config, logFiles, logTexts, pricing, window, t) {
|
|
5695
6074
|
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
5696
|
-
|
|
5697
|
-
|
|
5698
|
-
|
|
5699
|
-
|
|
5700
|
-
|
|
5701
|
-
|
|
5702
|
-
|
|
5703
|
-
|
|
5704
|
-
|
|
5705
|
-
|
|
5706
|
-
|
|
5707
|
-
|
|
5708
|
-
|
|
5709
|
-
|
|
5710
|
-
const
|
|
5711
|
-
|
|
5712
|
-
|
|
6075
|
+
const pct = (share) => `${(share * 100).toFixed(1)}%`;
|
|
6076
|
+
const { onlyLabel, sinceMs, untilMs } = window;
|
|
6077
|
+
/* The same join the caller makes, from the same texts: one log, all files. */
|
|
6078
|
+
const raw = logTexts.map((text) => (text.endsWith('\n') ? text : `${text}\n`)).join('');
|
|
6079
|
+
const sourceDefs = config.sources;
|
|
6080
|
+
if (sourceDefs === undefined || Object.keys(sourceDefs).length === 0) {
|
|
6081
|
+
throw new Error(t.profile.bySourceNeedsConfig());
|
|
6082
|
+
}
|
|
6083
|
+
const { bySource, unmatched } = assignSources(logFiles, sourceDefs);
|
|
6084
|
+
if (bySource.size === 0) {
|
|
6085
|
+
throw new Error(t.profile.bySourceNothingMatched(Object.keys(sourceDefs).join(', ')));
|
|
6086
|
+
}
|
|
6087
|
+
const textByFile = new Map(logFiles.map((file, i) => [file, logTexts[i]]));
|
|
6088
|
+
const fleetSources = [];
|
|
6089
|
+
const cacheDeltas = new Map();
|
|
6090
|
+
for (const [name, files] of [...bySource.entries()].sort((a, b) => a[0].localeCompare(b[0]))) {
|
|
6091
|
+
const text = files
|
|
6092
|
+
.map((file) => textByFile.get(file))
|
|
6093
|
+
.map((chunk) => (chunk.endsWith('\n') ? chunk : `${chunk}\n`))
|
|
6094
|
+
.join('');
|
|
6095
|
+
const sourceReport = profileUsage(text, { catalogue: pricing, label: onlyLabel, sinceMs, untilMs });
|
|
6096
|
+
fleetSources.push({ name, report: sourceReport });
|
|
6097
|
+
cacheDeltas.set(name, cacheEconomics(sourceReport.total).deltaUsd);
|
|
6098
|
+
}
|
|
6099
|
+
const aggregate = profileUsage(raw, { catalogue: pricing, label: onlyLabel, sinceMs, untilMs });
|
|
6100
|
+
const rollup = fleetRollup(fleetSources, {
|
|
6101
|
+
cacheDeltas,
|
|
6102
|
+
aggregateCacheDelta: cacheEconomics(aggregate.total).deltaUsd,
|
|
6103
|
+
});
|
|
6104
|
+
if (boolFlag(args, 'json')) {
|
|
6105
|
+
console.log(JSON.stringify({
|
|
6106
|
+
schemaVersion: 1,
|
|
6107
|
+
bySource: fleetSources.map((source) => ({ name: source.name, report: source.report })),
|
|
6108
|
+
rollup: {
|
|
6109
|
+
totalUsd: rollup.totalUsd,
|
|
6110
|
+
calls: rollup.calls,
|
|
6111
|
+
sources: rollup.sources,
|
|
6112
|
+
worst: rollup.worst,
|
|
6113
|
+
mismatchedSpans: rollup.mismatchedSpans,
|
|
6114
|
+
splitBrains: rollup.splitBrains,
|
|
6115
|
+
cacheUnderwater: rollup.cacheUnderwater,
|
|
6116
|
+
unmatchedFiles: unmatched,
|
|
6117
|
+
},
|
|
6118
|
+
}, (key, value) => (value instanceof Map ? undefined : value), 2));
|
|
5713
6119
|
}
|
|
5714
|
-
|
|
5715
|
-
|
|
5716
|
-
|
|
5717
|
-
|
|
5718
|
-
|
|
5719
|
-
if (plan.span === null) {
|
|
5720
|
-
lines.push('');
|
|
5721
|
-
lines.push(`_${t.plan.noClock()}_`);
|
|
6120
|
+
else {
|
|
6121
|
+
console.log(sectionHeading(t.profile.fleetHeading(n(rollup.sources.length), formatUsd(rollup.totalUsd), t.profile.calls(rollup.calls))));
|
|
6122
|
+
for (const row of rollup.sources) {
|
|
6123
|
+
const span = row.spanDays === null ? t.profile.fleetNoClock() : t.profile.fleetSpan(row.spanDays.toFixed(1));
|
|
6124
|
+
console.log(` ${t.profile.fleetRow(row.name, formatUsd(row.usd), pct(row.share), t.profile.calls(row.calls), span)}`);
|
|
5722
6125
|
}
|
|
5723
|
-
|
|
5724
|
-
|
|
5725
|
-
|
|
5726
|
-
? t.plan.projected(formatUsd(action.savingUsd))
|
|
5727
|
-
: t.plan.staked(formatUsd(action.stakeUsd ?? 0));
|
|
5728
|
-
lines.push('');
|
|
5729
|
-
lines.push(`### ${t.plan.action(action.kind, name, action.model)} — ${money}`);
|
|
5730
|
-
if (action.detail.routeTo !== undefined)
|
|
5731
|
-
lines.push(`- ${t.plan.routeTo(action.detail.routeTo.displayName)}`);
|
|
5732
|
-
for (const assumption of action.assumes)
|
|
5733
|
-
lines.push(`- ${t.plan.assume(assumption)}`);
|
|
5734
|
-
if (action.check !== null)
|
|
5735
|
-
lines.push(`- ${t.plan.check(action.check)}`);
|
|
6126
|
+
if (rollup.worst !== null && rollup.sources.length > 1) {
|
|
6127
|
+
console.log();
|
|
6128
|
+
console.log(` ${c.yellow('!')} ${c.bold(wrap(t.profile.fleetWorst(rollup.worst.name, formatUsd(rollup.worst.usd), pct(rollup.worst.share)), 74, ' '))}`);
|
|
5736
6129
|
}
|
|
5737
|
-
if (
|
|
5738
|
-
|
|
5739
|
-
lines.push(`_${t.plan.filtered(n(filtered), formatUsd(minUsd), formatUsd(droppedUsd))}_`);
|
|
6130
|
+
if (rollup.mismatchedSpans) {
|
|
6131
|
+
console.log(` ${c.dim(wrap(t.profile.fleetMismatchedSpans(), 74, ' '))}`);
|
|
5740
6132
|
}
|
|
5741
|
-
|
|
5742
|
-
|
|
5743
|
-
|
|
5744
|
-
});
|
|
5745
|
-
if (boolFlag(args, 'json')) {
|
|
5746
|
-
console.log(JSON.stringify(stamped, null, 2));
|
|
5747
|
-
return;
|
|
5748
|
-
}
|
|
5749
|
-
console.log(sectionHeading(t.plan.heading(n(actions.length), formatUsd(plan.totalUsd))));
|
|
5750
|
-
console.log(` ${wrap(t.plan.totals(formatUsd(stamped.projectedSavingUsd), formatUsd(stamped.measuredStakeUsd)), 74, ' ')}`);
|
|
5751
|
-
if (plan.span === null) {
|
|
5752
|
-
console.log(` ${c.dim(wrap(t.plan.noClock(), 74, ' '))}`);
|
|
5753
|
-
}
|
|
5754
|
-
for (const action of actions) {
|
|
5755
|
-
const name = action.label === UNLABELLED ? t.profile.unlabelled() : action.label;
|
|
5756
|
-
const money = action.savingUsd !== null
|
|
5757
|
-
? t.plan.projected(formatUsd(action.savingUsd))
|
|
5758
|
-
: t.plan.staked(formatUsd(action.stakeUsd ?? 0));
|
|
5759
|
-
console.log();
|
|
5760
|
-
console.log(` ${c.green('→')} ${c.bold(t.plan.action(action.kind, name, action.model))} ${money}`);
|
|
5761
|
-
if (action.detail.routeTo !== undefined) {
|
|
5762
|
-
console.log(` ${c.dim(t.plan.routeTo(action.detail.routeTo.displayName))}`);
|
|
6133
|
+
for (const split of rollup.splitBrains.slice(0, 3)) {
|
|
6134
|
+
console.log();
|
|
6135
|
+
console.log(` ${c.yellow('!')} ${wrap(t.profile.fleetSplitBrain(split.label, split.sources.map((v) => `${v.name} → ${v.model} (${formatUsd(v.usd)})`).join(', ')), 74, ' ')}`);
|
|
5763
6136
|
}
|
|
5764
|
-
for (const
|
|
5765
|
-
console.log(`
|
|
6137
|
+
for (const under of rollup.cacheUnderwater.slice(0, 3)) {
|
|
6138
|
+
console.log(` ${c.yellow('!')} ${wrap(t.profile.fleetCacheUnderwater(under.name, formatUsd(under.deltaUsd)), 74, ' ')}`);
|
|
5766
6139
|
}
|
|
5767
|
-
|
|
5768
|
-
console.log(`
|
|
6140
|
+
for (const file of unmatched) {
|
|
6141
|
+
console.log(` ${c.yellow('!')} ${wrap(t.profile.fleetUnmatched(file), 74, ' ')}`);
|
|
5769
6142
|
}
|
|
5770
|
-
}
|
|
5771
|
-
if (filtered > 0) {
|
|
5772
|
-
console.log();
|
|
5773
|
-
console.log(` ${c.dim(wrap(t.plan.filtered(n(filtered), formatUsd(minUsd), formatUsd(droppedUsd)), 74, ' '))}`);
|
|
5774
|
-
}
|
|
5775
|
-
console.log();
|
|
5776
|
-
console.log(` ${c.dim(wrap(t.plan.footer(), 74, ' '))}`);
|
|
5777
|
-
if (outPath !== undefined)
|
|
5778
|
-
console.log(c.dim(wrap(t.plan.wrote(outPath), 74, '')));
|
|
5779
|
-
}
|
|
5780
|
-
async function commandProfile(args, config, configDir, pricing, t) {
|
|
5781
|
-
const path = args.positional[0];
|
|
5782
|
-
if (path === undefined) {
|
|
5783
|
-
console.log();
|
|
5784
|
-
console.log(c.dim(wrap(t.profile.noTarget(), 74, ' ')));
|
|
5785
6143
|
console.log();
|
|
5786
|
-
|
|
6144
|
+
console.log(` ${c.dim(wrap(t.profile.fleetFooter(), 74, ' '))}`);
|
|
5787
6145
|
}
|
|
5788
6146
|
/**
|
|
5789
|
-
*
|
|
5790
|
-
*
|
|
5791
|
-
*
|
|
5792
|
-
*
|
|
5793
|
-
* will read them is a setup cost that gets a tool skipped, and doing it for
|
|
5794
|
-
* them is a directory listing.
|
|
5795
|
-
*
|
|
5796
|
-
* Files are read in name order — which for dated names is time order — and
|
|
5797
|
-
* how many were read is stated, because a report over "the logs" that
|
|
5798
|
-
* silently skipped one is a total that is wrong by an unknown amount. A
|
|
5799
|
-
* directory holding nothing readable is an error naming what it looked for,
|
|
5800
|
-
* not an empty report.
|
|
5801
|
-
*/
|
|
5802
|
-
/**
|
|
5803
|
-
* The same names, gzipped — which is what a rotated log actually looks like
|
|
5804
|
-
* a day after it rotates.
|
|
5805
|
-
*
|
|
5806
|
-
* `logrotate`, Docker's json-file driver and every cloud log export compress
|
|
5807
|
-
* yesterday's file, so a directory of a month's logs is one plain file and
|
|
5808
|
-
* twenty-nine `.gz` ones. Reading only the plain one and saying nothing
|
|
5809
|
-
* would report a month's bill from a day of it, in the flattering
|
|
5810
|
-
* direction, which is exactly the failure directory mode was added to
|
|
5811
|
-
* prevent.
|
|
6147
|
+
* The per-source gates. Each budget judges its own service and the run
|
|
6148
|
+
* fails naming the service — a total that hides which source crossed its
|
|
6149
|
+
* line is the rendering this mode exists to end. Waivable per source
|
|
6150
|
+
* through `bySource:<name>`, under the same expiry discipline.
|
|
5812
6151
|
*/
|
|
5813
|
-
const
|
|
5814
|
-
const
|
|
5815
|
-
|
|
5816
|
-
|
|
5817
|
-
|
|
5818
|
-
|
|
5819
|
-
|
|
5820
|
-
|
|
5821
|
-
|
|
5822
|
-
|
|
5823
|
-
|
|
5824
|
-
|
|
5825
|
-
|
|
5826
|
-
|
|
5827
|
-
.filter((entry) => entry.isFile() && READABLE.some((ext) => entry.name.endsWith(ext)))
|
|
5828
|
-
.map((entry) => join(entry.parentPath ?? path, entry.name))
|
|
5829
|
-
.sort((a, b) => a.localeCompare(b));
|
|
5830
|
-
if (logFiles.length === 0) {
|
|
5831
|
-
throw new Error(t.profile.noLogsInDirectory(path, READABLE.join(', ')));
|
|
6152
|
+
const bySourceBudgets = config.spend?.bySource ?? {};
|
|
6153
|
+
for (const [name, limit] of Object.entries(bySourceBudgets)) {
|
|
6154
|
+
const found = fleetSources.find((source) => source.name === name);
|
|
6155
|
+
if (found === undefined) {
|
|
6156
|
+
console.error(c.dim(t.profile.fleetBudgetMissing(name)));
|
|
6157
|
+
continue;
|
|
6158
|
+
}
|
|
6159
|
+
const usd = found.report.total.totalUsd;
|
|
6160
|
+
if (usd > limit) {
|
|
6161
|
+
console.error(c.red(t.profile.fleetBudgetFailed(name, formatUsd(usd), formatUsd(limit))));
|
|
6162
|
+
process.exitCode = 1;
|
|
6163
|
+
}
|
|
6164
|
+
else {
|
|
6165
|
+
console.error(c.dim(t.profile.fleetBudgetOk(name, formatUsd(usd), formatUsd(limit))));
|
|
5832
6166
|
}
|
|
5833
6167
|
}
|
|
6168
|
+
}
|
|
6169
|
+
/**
|
|
6170
|
+
* `--dry-run`: what this log could and could not answer, and no bill.
|
|
6171
|
+
*
|
|
6172
|
+
* The question somebody has *before* wiring Trazum into CI is not "what did we
|
|
6173
|
+
* spend" but "will this log support the gates I want", and answering it with a
|
|
6174
|
+
* full report makes them read a bill to find out a field is missing. This path
|
|
6175
|
+
* states readiness per capability and produces no dollar figure at all, so
|
|
6176
|
+
* nothing here can be mistaken for spend. It also refuses to coexist with the
|
|
6177
|
+
* gates: a gate over a report that was never produced would exit green having
|
|
6178
|
+
* judged nothing.
|
|
6179
|
+
*/
|
|
6180
|
+
function profileDryRun(args, report, t) {
|
|
6181
|
+
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
6182
|
+
const pct = (share) => `${(share * 100).toFixed(1)}%`;
|
|
6183
|
+
const gateFlags = ['max-usd', 'max-growth-usd', 'max-cache-loss-usd', 'max-day-usd', 'max-session-usd'];
|
|
6184
|
+
if (gateFlags.some((flag) => typeof args.flags.get(flag) === 'string')) {
|
|
6185
|
+
throw new Error(t.profile.dryRunNoGates());
|
|
6186
|
+
}
|
|
6187
|
+
const cov = report.fieldCoverage;
|
|
6188
|
+
const parsed = cov.parsed;
|
|
6189
|
+
console.log(sectionHeading(t.profile.dryRunHeading()));
|
|
6190
|
+
console.log(` ${wrap(t.profile.dryRunParsed(n(parsed), n(report.skippedLines.length)), 74, ' ')}`);
|
|
6191
|
+
if (report.unpricedModels.length > 0) {
|
|
6192
|
+
console.log(` ${c.yellow('!')} ${wrap(t.profile.dryRunUnpriced(report.unpricedModels.join(', ')), 74, ' ')}`);
|
|
6193
|
+
}
|
|
6194
|
+
console.log();
|
|
6195
|
+
const can = (ok, line) => {
|
|
6196
|
+
console.log(` ${ok ? c.green('✓') : c.yellow('✗')} ${wrap(line, 74, ' ')}`);
|
|
6197
|
+
};
|
|
6198
|
+
const share = (count) => (parsed > 0 ? pct(count / parsed) : '0%');
|
|
6199
|
+
can(parsed > 0, t.profile.dryRunTotals());
|
|
6200
|
+
can(cov.label > 0, t.profile.dryRunLabels(share(cov.label)));
|
|
6201
|
+
can(cov.ts > 0, t.profile.dryRunClock(share(cov.ts)));
|
|
6202
|
+
can(cov.session > 0, t.profile.dryRunSessions(share(cov.session)));
|
|
6203
|
+
can(cov.outcome > 0, t.profile.dryRunOutcomes(share(cov.outcome)));
|
|
6204
|
+
can(cov.stopReason > 0, t.profile.dryRunStopReason(share(cov.stopReason)));
|
|
6205
|
+
// "No cache traffic" is not a missing field: the split can only exist on
|
|
6206
|
+
// records that wrote, and a log that never wrote has nothing to record.
|
|
6207
|
+
if (cov.cacheWrites > 0) {
|
|
6208
|
+
can(cov.cacheTtl > 0, t.profile.dryRunCacheTtl(n(cov.cacheTtl), n(cov.cacheWrites)));
|
|
6209
|
+
}
|
|
6210
|
+
else {
|
|
6211
|
+
console.log(` ${c.dim('·')} ${wrap(t.profile.dryRunNoCacheTraffic(), 74, ' ')}`);
|
|
6212
|
+
}
|
|
6213
|
+
console.log();
|
|
6214
|
+
console.log(` ${c.dim(wrap(t.profile.dryRunFooter(), 74, ' '))}`);
|
|
6215
|
+
if (parsed === 0)
|
|
6216
|
+
process.exitCode = 1;
|
|
6217
|
+
}
|
|
6218
|
+
/**
|
|
6219
|
+
* The money gates: run them, and say what they said.
|
|
6220
|
+
*
|
|
6221
|
+
* `check` gates tokens before the money is spent; these gate the spend itself,
|
|
6222
|
+
* from the provider’s own billed counts. No period is assumed — the budget
|
|
6223
|
+
* applies to exactly the log handed in, so a nightly job that profiles
|
|
6224
|
+
* yesterday’s log has a daily budget without Trazum ever guessing what a day is.
|
|
6225
|
+
*
|
|
6226
|
+
* ## Why this is one function and not five
|
|
6227
|
+
*
|
|
6228
|
+
* `waiverFor`, `waived`, the uses it collects, the verdict capture and the
|
|
6229
|
+
* flush to the waiver log were five closures inside `commandProfile`, sharing
|
|
6230
|
+
* its 2,359-line scope with the report they have nothing to do with. They are
|
|
6231
|
+
* one concern — policy — and the only two things the rest of the command needs
|
|
6232
|
+
* from them are whether a gate failed and what the gates said.
|
|
6233
|
+
*
|
|
6234
|
+
* ## Why it returns rather than assigns
|
|
6235
|
+
*
|
|
6236
|
+
* `gateFailed` and `gateVerdicts` were `let` and `const` in the enclosing
|
|
6237
|
+
* scope, written here and read three hundred lines away by the side-file
|
|
6238
|
+
* writer. That is an output channel nothing declares. They are the return
|
|
6239
|
+
* value now, and `failed` is still read off `process.exitCode` exactly as it
|
|
6240
|
+
* was — the gates set the exit code, and this reports what they set.
|
|
6241
|
+
*/
|
|
6242
|
+
async function runProfileGates(context) {
|
|
6243
|
+
const { args, config, configDir, report, previous, againstDelta, windowed, now, pricing, t } = context;
|
|
6244
|
+
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
6245
|
+
const pct = (share) => `${(share * 100).toFixed(1)}%`;
|
|
6246
|
+
/*
|
|
6247
|
+
What a failing gate points at. The enclosing command computes this too, a
|
|
6248
|
+
few hundred lines further down, and the gate block used to reach forward
|
|
6249
|
+
into that declaration — legal only because the closure that read it was
|
|
6250
|
+
called later. `billLevers` is pure, so the honest fix is to ask for it here.
|
|
6251
|
+
*/
|
|
6252
|
+
const levers = billLevers(report, { catalogue: pricing });
|
|
5834
6253
|
/**
|
|
5835
|
-
*
|
|
5836
|
-
*
|
|
5837
|
-
* Decided by **extension**, not by sniffing the first two bytes: a file
|
|
5838
|
-
* named `.jsonl` whose contents happen to start with 0x1f8b is far more
|
|
5839
|
-
* likely to be a corrupt log than a mislabelled archive, and silently
|
|
5840
|
-
* treating it as one would turn a diagnosable error into an empty report.
|
|
6254
|
+
* The money gates, armed by flags and applied on every output path.
|
|
5841
6255
|
*
|
|
5842
|
-
*
|
|
5843
|
-
*
|
|
5844
|
-
* the
|
|
6256
|
+
* `check` gates tokens before the money is spent; these gate the spend
|
|
6257
|
+
* itself, from the provider's own billed counts. No period is assumed —
|
|
6258
|
+
* the budget applies to exactly the log handed in, so a nightly job that
|
|
6259
|
+
* profiles yesterday's log has a daily budget without Trazum ever
|
|
6260
|
+
* guessing what a day is.
|
|
5845
6261
|
*/
|
|
5846
|
-
const logTexts = await Promise.all(logFiles.map((file) => readUsageLog(file, t)));
|
|
5847
|
-
// A file that does not end in a newline would otherwise glue its last record
|
|
5848
|
-
// to the next file's first one, and both would be reported as unreadable.
|
|
5849
|
-
const raw = logTexts.map((text) => (text.endsWith('\n') ? text : `${text}\n`)).join('');
|
|
5850
6262
|
/**
|
|
5851
|
-
* The
|
|
5852
|
-
*
|
|
5853
|
-
*
|
|
5854
|
-
*
|
|
6263
|
+
* The gate verdicts, kept so the markdown summary can carry them.
|
|
6264
|
+
*
|
|
6265
|
+
* Collected by wrapping `console.error` for the duration of `applyGates`
|
|
6266
|
+
* rather than by threading a return value through every gate. That is the
|
|
6267
|
+
* unusual choice here and it is deliberate: a gate added later reaches the
|
|
6268
|
+
* summary without anyone remembering to register it, and the alternative —
|
|
6269
|
+
* one push per verdict at a dozen call sites — is a list that goes stale
|
|
6270
|
+
* silently. Colour is stripped, because a summary is markdown and an
|
|
6271
|
+
* escape sequence in it is noise a reader has to look past.
|
|
5855
6272
|
*/
|
|
5856
|
-
const
|
|
6273
|
+
const gateVerdicts = [];
|
|
6274
|
+
let gateFailed = false;
|
|
5857
6275
|
/**
|
|
5858
|
-
*
|
|
5859
|
-
*
|
|
5860
|
-
*
|
|
5861
|
-
*
|
|
5862
|
-
*
|
|
5863
|
-
*
|
|
6276
|
+
* Findings as policy. A waiver silences one gate's exit code for a bounded
|
|
6277
|
+
* time, on the record: the failure still prints (waived is shown as
|
|
6278
|
+
* waived, never hidden — the bill still counts it), the reason and the
|
|
6279
|
+
* days left print beside it, and the day the waiver expires the gate
|
|
6280
|
+
* fails again louder, naming the date and the reason somebody wrote.
|
|
6281
|
+
* That expiry is the entire mechanism by which a waiver stays a decision
|
|
6282
|
+
* instead of becoming a habit.
|
|
5864
6283
|
*/
|
|
5865
|
-
const
|
|
5866
|
-
|
|
5867
|
-
|
|
5868
|
-
|
|
5869
|
-
|
|
5870
|
-
|
|
5871
|
-
|
|
5872
|
-
|
|
5873
|
-
}
|
|
5874
|
-
const windowed = sinceMs !== undefined || untilMs !== undefined;
|
|
6284
|
+
const waiverFor = (gate) => {
|
|
6285
|
+
const entry = (config.waive ?? []).find((w) => w.gate === gate);
|
|
6286
|
+
if (entry === undefined)
|
|
6287
|
+
return null;
|
|
6288
|
+
// The waiver covers its named day whole: expiry begins the next UTC day.
|
|
6289
|
+
const expiresMs = Date.parse(`${entry.until}T00:00:00Z`) + 86_400_000;
|
|
6290
|
+
return { entry, expired: Date.now() >= expiresMs };
|
|
6291
|
+
};
|
|
5875
6292
|
/**
|
|
5876
|
-
*
|
|
5877
|
-
*
|
|
5878
|
-
*
|
|
5879
|
-
*
|
|
5880
|
-
* The threshold is in the sentence, not hidden here.
|
|
6293
|
+
* Applies a waiver to one failing gate. Returns true when the failure is
|
|
6294
|
+
* silenced — the caller skips its exitCode — and prints the record either
|
|
6295
|
+
* way, because a waived failure that vanished from the output would be a
|
|
6296
|
+
* finding deleted with extra steps.
|
|
5881
6297
|
*/
|
|
5882
|
-
const STALE_PRICING_DAYS = 45;
|
|
5883
|
-
const pricingAgeDays = reviewAgeDays(pricing.lastReviewed, new Date());
|
|
5884
|
-
const pricingStale = pricingAgeDays !== null && pricingAgeDays > STALE_PRICING_DAYS
|
|
5885
|
-
? { date: pricing.lastReviewed, days: pricingAgeDays }
|
|
5886
|
-
: null;
|
|
5887
|
-
const report = profileUsage(raw, { catalogue: pricing, label: onlyLabel, sinceMs, untilMs });
|
|
5888
|
-
if (report.total.calls === 0 && report.unpriced.calls === 0) {
|
|
5889
|
-
if (onlyLabel !== undefined || windowed) {
|
|
5890
|
-
// Diagnose against the log without the failed filter, so the error can
|
|
5891
|
-
// name what does exist instead of describing an absence.
|
|
5892
|
-
const unfiltered = profileUsage(raw, { catalogue: pricing });
|
|
5893
|
-
if (unfiltered.total.calls > 0 || unfiltered.unpriced.calls > 0) {
|
|
5894
|
-
if (onlyLabel !== undefined && !unfiltered.byLabel.some((r) => r.label === onlyLabel)) {
|
|
5895
|
-
const available = unfiltered.byLabel
|
|
5896
|
-
.map((r) => (r.label === UNLABELLED ? t.profile.unlabelled() : r.label))
|
|
5897
|
-
.join(', ');
|
|
5898
|
-
throw new Error(t.route.labelNotFound(onlyLabel, available || '—'));
|
|
5899
|
-
}
|
|
5900
|
-
if (windowed) {
|
|
5901
|
-
/**
|
|
5902
|
-
* A window that matches nothing must not become a $0 report — under
|
|
5903
|
-
* `--max-usd` it would pass a budget gate over a period the log
|
|
5904
|
-
* simply does not cover, which is the flattering non-answer. The
|
|
5905
|
-
* error names what the log *does* cover, or says it has no clock at
|
|
5906
|
-
* all, so the fix is visible in the message.
|
|
5907
|
-
*/
|
|
5908
|
-
if (unfiltered.span === null)
|
|
5909
|
-
throw new Error(t.profile.windowNeedsClock());
|
|
5910
|
-
throw new Error(`${t.profile.windowMatchesNothing(dayOf(unfiltered.span.fromMs), dayOf(unfiltered.span.toMs))}${relativeWindow ? ` ${t.profile.windowRelativeEmpty()}` : ''}`);
|
|
5911
|
-
}
|
|
5912
|
-
}
|
|
5913
|
-
}
|
|
5914
|
-
}
|
|
5915
|
-
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
5916
|
-
const pct = (share) => `${(share * 100).toFixed(1)}%`;
|
|
5917
6298
|
/**
|
|
5918
|
-
*
|
|
6299
|
+
* Uses recorded this run, flushed after the gates have finished.
|
|
5919
6300
|
*
|
|
5920
|
-
*
|
|
5921
|
-
*
|
|
5922
|
-
*
|
|
5923
|
-
*
|
|
5924
|
-
* config's `sources` block; a file matching no source is named loudly,
|
|
5925
|
-
* because a log that silently joined no report is spend missing from every
|
|
5926
|
-
* bill.
|
|
6301
|
+
* Collected rather than written inline because `waived` is synchronous and
|
|
6302
|
+
* called from seven places inside the gate pass. Writing from each of them
|
|
6303
|
+
* would mean seven awaits threaded through the exit-code logic — the one
|
|
6304
|
+
* part of this command where a mistake turns a red build green.
|
|
5927
6305
|
*/
|
|
5928
|
-
|
|
5929
|
-
|
|
5930
|
-
|
|
5931
|
-
|
|
5932
|
-
|
|
5933
|
-
|
|
5934
|
-
|
|
5935
|
-
|
|
5936
|
-
}
|
|
5937
|
-
const textByFile = new Map(logFiles.map((file, i) => [file, logTexts[i]]));
|
|
5938
|
-
const fleetSources = [];
|
|
5939
|
-
const cacheDeltas = new Map();
|
|
5940
|
-
for (const [name, files] of [...bySource.entries()].sort((a, b) => a[0].localeCompare(b[0]))) {
|
|
5941
|
-
const text = files
|
|
5942
|
-
.map((file) => textByFile.get(file))
|
|
5943
|
-
.map((chunk) => (chunk.endsWith('\n') ? chunk : `${chunk}\n`))
|
|
5944
|
-
.join('');
|
|
5945
|
-
const sourceReport = profileUsage(text, { catalogue: pricing, label: onlyLabel, sinceMs, untilMs });
|
|
5946
|
-
fleetSources.push({ name, report: sourceReport });
|
|
5947
|
-
cacheDeltas.set(name, cacheEconomics(sourceReport.total).deltaUsd);
|
|
5948
|
-
}
|
|
5949
|
-
const aggregate = profileUsage(raw, { catalogue: pricing, label: onlyLabel, sinceMs, untilMs });
|
|
5950
|
-
const rollup = fleetRollup(fleetSources, {
|
|
5951
|
-
cacheDeltas,
|
|
5952
|
-
aggregateCacheDelta: cacheEconomics(aggregate.total).deltaUsd,
|
|
5953
|
-
});
|
|
5954
|
-
if (boolFlag(args, 'json')) {
|
|
5955
|
-
console.log(JSON.stringify({
|
|
5956
|
-
schemaVersion: 1,
|
|
5957
|
-
bySource: fleetSources.map((source) => ({ name: source.name, report: source.report })),
|
|
5958
|
-
rollup: {
|
|
5959
|
-
totalUsd: rollup.totalUsd,
|
|
5960
|
-
calls: rollup.calls,
|
|
5961
|
-
sources: rollup.sources,
|
|
5962
|
-
worst: rollup.worst,
|
|
5963
|
-
mismatchedSpans: rollup.mismatchedSpans,
|
|
5964
|
-
splitBrains: rollup.splitBrains,
|
|
5965
|
-
cacheUnderwater: rollup.cacheUnderwater,
|
|
5966
|
-
unmatchedFiles: unmatched,
|
|
5967
|
-
},
|
|
5968
|
-
}, (key, value) => (value instanceof Map ? undefined : value), 2));
|
|
5969
|
-
}
|
|
5970
|
-
else {
|
|
5971
|
-
console.log(sectionHeading(t.profile.fleetHeading(n(rollup.sources.length), formatUsd(rollup.totalUsd), t.profile.calls(rollup.calls))));
|
|
5972
|
-
for (const row of rollup.sources) {
|
|
5973
|
-
const span = row.spanDays === null ? t.profile.fleetNoClock() : t.profile.fleetSpan(row.spanDays.toFixed(1));
|
|
5974
|
-
console.log(` ${t.profile.fleetRow(row.name, formatUsd(row.usd), pct(row.share), t.profile.calls(row.calls), span)}`);
|
|
5975
|
-
}
|
|
5976
|
-
if (rollup.worst !== null && rollup.sources.length > 1) {
|
|
5977
|
-
console.log();
|
|
5978
|
-
console.log(` ${c.yellow('!')} ${c.bold(wrap(t.profile.fleetWorst(rollup.worst.name, formatUsd(rollup.worst.usd), pct(rollup.worst.share)), 74, ' '))}`);
|
|
5979
|
-
}
|
|
5980
|
-
if (rollup.mismatchedSpans) {
|
|
5981
|
-
console.log(` ${c.dim(wrap(t.profile.fleetMismatchedSpans(), 74, ' '))}`);
|
|
5982
|
-
}
|
|
5983
|
-
for (const split of rollup.splitBrains.slice(0, 3)) {
|
|
5984
|
-
console.log();
|
|
5985
|
-
console.log(` ${c.yellow('!')} ${wrap(t.profile.fleetSplitBrain(split.label, split.sources.map((v) => `${v.name} → ${v.model} (${formatUsd(v.usd)})`).join(', ')), 74, ' ')}`);
|
|
5986
|
-
}
|
|
5987
|
-
for (const under of rollup.cacheUnderwater.slice(0, 3)) {
|
|
5988
|
-
console.log(` ${c.yellow('!')} ${wrap(t.profile.fleetCacheUnderwater(under.name, formatUsd(under.deltaUsd)), 74, ' ')}`);
|
|
5989
|
-
}
|
|
5990
|
-
for (const file of unmatched) {
|
|
5991
|
-
console.log(` ${c.yellow('!')} ${wrap(t.profile.fleetUnmatched(file), 74, ' ')}`);
|
|
5992
|
-
}
|
|
5993
|
-
console.log();
|
|
5994
|
-
console.log(` ${c.dim(wrap(t.profile.fleetFooter(), 74, ' '))}`);
|
|
6306
|
+
const waiverUses = [];
|
|
6307
|
+
const waived = (gate, measuredUsd = null, limitUsd = null) => {
|
|
6308
|
+
const found = waiverFor(gate);
|
|
6309
|
+
if (found === null)
|
|
6310
|
+
return false;
|
|
6311
|
+
if (found.expired) {
|
|
6312
|
+
console.error(c.red(t.profile.waiveExpired(gate, found.entry.until, found.entry.reason)));
|
|
6313
|
+
return false;
|
|
5995
6314
|
}
|
|
5996
|
-
|
|
5997
|
-
|
|
5998
|
-
* fails naming the service — a total that hides which source crossed its
|
|
5999
|
-
* line is the rendering this mode exists to end. Waivable per source
|
|
6000
|
-
* through `bySource:<name>`, under the same expiry discipline.
|
|
6001
|
-
*/
|
|
6002
|
-
const bySourceBudgets = config.spend?.bySource ?? {};
|
|
6003
|
-
for (const [name, limit] of Object.entries(bySourceBudgets)) {
|
|
6004
|
-
const found = fleetSources.find((source) => source.name === name);
|
|
6005
|
-
if (found === undefined) {
|
|
6006
|
-
console.error(c.dim(t.profile.fleetBudgetMissing(name)));
|
|
6007
|
-
continue;
|
|
6008
|
-
}
|
|
6009
|
-
const usd = found.report.total.totalUsd;
|
|
6010
|
-
if (usd > limit) {
|
|
6011
|
-
console.error(c.red(t.profile.fleetBudgetFailed(name, formatUsd(usd), formatUsd(limit))));
|
|
6012
|
-
process.exitCode = 1;
|
|
6013
|
-
}
|
|
6014
|
-
else {
|
|
6015
|
-
console.error(c.dim(t.profile.fleetBudgetOk(name, formatUsd(usd), formatUsd(limit))));
|
|
6016
|
-
}
|
|
6017
|
-
}
|
|
6018
|
-
return;
|
|
6019
|
-
}
|
|
6020
|
-
/**
|
|
6021
|
-
* `--dry-run`: what this log could and could not answer, and no bill.
|
|
6022
|
-
*
|
|
6023
|
-
* The question somebody has *before* wiring Trazum into CI is not "what did
|
|
6024
|
-
* we spend" but "will this log support the gates I want" — and answering it
|
|
6025
|
-
* with a full report makes them read a bill to find out a field is missing.
|
|
6026
|
-
* This path states readiness per capability and produces no dollar figure
|
|
6027
|
-
* at all, so nothing here can be mistaken for spend. It also refuses to
|
|
6028
|
-
* coexist with the gates: a gate over a report that was never produced
|
|
6029
|
-
* would exit green having judged nothing.
|
|
6030
|
-
*/
|
|
6031
|
-
if (boolFlag(args, 'dry-run')) {
|
|
6032
|
-
const gateFlags = ['max-usd', 'max-growth-usd', 'max-cache-loss-usd', 'max-day-usd', 'max-session-usd'];
|
|
6033
|
-
if (gateFlags.some((flag) => typeof args.flags.get(flag) === 'string')) {
|
|
6034
|
-
throw new Error(t.profile.dryRunNoGates());
|
|
6035
|
-
}
|
|
6036
|
-
const cov = report.fieldCoverage;
|
|
6037
|
-
const parsed = cov.parsed;
|
|
6038
|
-
console.log(sectionHeading(t.profile.dryRunHeading()));
|
|
6039
|
-
console.log(` ${wrap(t.profile.dryRunParsed(n(parsed), n(report.skippedLines.length)), 74, ' ')}`);
|
|
6040
|
-
if (report.unpricedModels.length > 0) {
|
|
6041
|
-
console.log(` ${c.yellow('!')} ${wrap(t.profile.dryRunUnpriced(report.unpricedModels.join(', ')), 74, ' ')}`);
|
|
6042
|
-
}
|
|
6043
|
-
console.log();
|
|
6044
|
-
const can = (ok, line) => {
|
|
6045
|
-
console.log(` ${ok ? c.green('✓') : c.yellow('✗')} ${wrap(line, 74, ' ')}`);
|
|
6046
|
-
};
|
|
6047
|
-
const share = (count) => (parsed > 0 ? pct(count / parsed) : '0%');
|
|
6048
|
-
can(parsed > 0, t.profile.dryRunTotals());
|
|
6049
|
-
can(cov.label > 0, t.profile.dryRunLabels(share(cov.label)));
|
|
6050
|
-
can(cov.ts > 0, t.profile.dryRunClock(share(cov.ts)));
|
|
6051
|
-
can(cov.session > 0, t.profile.dryRunSessions(share(cov.session)));
|
|
6052
|
-
can(cov.outcome > 0, t.profile.dryRunOutcomes(share(cov.outcome)));
|
|
6053
|
-
can(cov.stopReason > 0, t.profile.dryRunStopReason(share(cov.stopReason)));
|
|
6054
|
-
// "No cache traffic" is not a missing field: the split can only exist on
|
|
6055
|
-
// records that wrote, and a log that never wrote has nothing to record.
|
|
6056
|
-
if (cov.cacheWrites > 0) {
|
|
6057
|
-
can(cov.cacheTtl > 0, t.profile.dryRunCacheTtl(n(cov.cacheTtl), n(cov.cacheWrites)));
|
|
6058
|
-
}
|
|
6059
|
-
else {
|
|
6060
|
-
console.log(` ${c.dim('·')} ${wrap(t.profile.dryRunNoCacheTraffic(), 74, ' ')}`);
|
|
6061
|
-
}
|
|
6062
|
-
console.log();
|
|
6063
|
-
console.log(` ${c.dim(wrap(t.profile.dryRunFooter(), 74, ' '))}`);
|
|
6064
|
-
if (parsed === 0)
|
|
6065
|
-
process.exitCode = 1;
|
|
6066
|
-
return;
|
|
6067
|
-
}
|
|
6068
|
-
/**
|
|
6069
|
-
* The previous log, loaded before the output paths split so the growth gate
|
|
6070
|
-
* exists under `--json` too — a CI step reads the JSON and trusts the exit
|
|
6071
|
-
* code, and a gate that only arms in the human rendering is a gate CI never
|
|
6072
|
-
* had.
|
|
6073
|
-
*/
|
|
6074
|
-
const againstPath = stringFlag(args, 'against');
|
|
6075
|
-
// The same filter on both sides: comparing one workload's bill against the
|
|
6076
|
-
// whole previous log would report every sibling workload as vanished savings.
|
|
6077
|
-
const previous = againstPath !== undefined
|
|
6078
|
-
// The same reader as the log itself, so `--against last-month.jsonl.gz`
|
|
6079
|
-
// works: a comparison that could only read one of the two formats would
|
|
6080
|
-
// be a flag that fails on exactly the rotated file it exists to read.
|
|
6081
|
-
? profileUsage(await readUsageLog(againstPath, t), {
|
|
6082
|
-
catalogue: pricing,
|
|
6083
|
-
label: onlyLabel,
|
|
6084
|
-
// The same window on both sides, for the same reason as the label:
|
|
6085
|
-
// a windowed bill against an unwindowed one compares a slice to a
|
|
6086
|
-
// whole and calls the difference growth.
|
|
6087
|
-
sinceMs,
|
|
6088
|
-
untilMs,
|
|
6089
|
-
})
|
|
6090
|
-
: null;
|
|
6091
|
-
const againstDelta = previous !== null && previous.total.calls > 0
|
|
6092
|
-
? report.total.totalUsd - previous.total.totalUsd
|
|
6093
|
-
: null;
|
|
6094
|
-
/**
|
|
6095
|
-
* The same tokens at another model's rates, computed before the output paths
|
|
6096
|
-
* split so `--json` carries it too.
|
|
6097
|
-
*
|
|
6098
|
-
* An unknown id **throws** rather than printing nothing. A flag that silently
|
|
6099
|
-
* does nothing is worse than a missing feature: the reader typed a question,
|
|
6100
|
-
* got a report with no answer in it, and has no way to tell a typo from a
|
|
6101
|
-
* model this comparison had nothing to say about.
|
|
6102
|
-
*/
|
|
6103
|
-
/**
|
|
6104
|
-
* The largest call against each model's window, computed once here so the
|
|
6105
|
-
* terminal, the JSON and the markdown state the same ratio — the
|
|
6106
|
-
* denominator lives in the (possibly overlaid) catalogue.
|
|
6107
|
-
*/
|
|
6108
|
-
const pressures = contextPressure(report, pricing);
|
|
6109
|
-
const whatIfModel = stringFlag(args, 'what-if');
|
|
6110
|
-
const whatIf = whatIfModel !== undefined ? repriceProfile(report, whatIfModel, pricing) : null;
|
|
6111
|
-
if (whatIfModel !== undefined && whatIf === null) {
|
|
6112
|
-
throw new Error(t.profile.whatIfUnknown(whatIfModel, pricing.models.map((m) => m.id).join(', ')));
|
|
6113
|
-
}
|
|
6114
|
-
/**
|
|
6115
|
-
* The drivers of the change, per label and per model, computed once here so
|
|
6116
|
-
* the terminal, the JSON and any future rendering describe the same change.
|
|
6117
|
-
* The model half answers the question the label half cannot: "the growth is
|
|
6118
|
-
* traffic moving from Haiku to Opus" is a fact about the mix, invisible in
|
|
6119
|
-
* per-workload rows whose names did not change.
|
|
6120
|
-
*/
|
|
6121
|
-
const labelDrivers = previous !== null && previous.total.calls > 0
|
|
6122
|
-
? driversBetween(previous.byLabel.map((r) => ({ key: r.label, usd: r.breakdown.totalUsd })), report.byLabel.map((r) => ({ key: r.label, usd: r.breakdown.totalUsd })))
|
|
6123
|
-
: [];
|
|
6124
|
-
const modelDrivers = previous !== null && previous.total.calls > 0
|
|
6125
|
-
? driversBetween(previous.byModel.map((r) => ({ key: r.model, usd: r.breakdown.totalUsd })), report.byModel.map((r) => ({ key: r.model, usd: r.breakdown.totalUsd })))
|
|
6126
|
-
: [];
|
|
6127
|
-
/**
|
|
6128
|
-
* Whether the two logs share any time at all. This comparison is meant for
|
|
6129
|
-
* disjoint periods or snapshots of different systems; when both spans are
|
|
6130
|
-
* known and intersect, the same calls may sit on both sides of the
|
|
6131
|
-
* subtraction and the "growth" is partly the same money counted twice.
|
|
6132
|
-
* Only decidable when both logs carry a clock — three states, as always:
|
|
6133
|
-
* warned, clear, or unknown, and unknown stays silent rather than clear.
|
|
6134
|
-
*/
|
|
6135
|
-
const againstOverlap = previous !== null &&
|
|
6136
|
-
previous.total.calls > 0 &&
|
|
6137
|
-
previous.span !== null &&
|
|
6138
|
-
report.span !== null &&
|
|
6139
|
-
Math.min(report.span.toMs, previous.span.toMs) >=
|
|
6140
|
-
Math.max(report.span.fromMs, previous.span.fromMs)
|
|
6141
|
-
? {
|
|
6142
|
-
fromMs: Math.max(report.span.fromMs, previous.span.fromMs),
|
|
6143
|
-
toMs: Math.min(report.span.toMs, previous.span.toMs),
|
|
6144
|
-
}
|
|
6145
|
-
: null;
|
|
6146
|
-
// A gate flag that silently does nothing is not an answer — same rule as
|
|
6147
|
-
// --apply-suggestions without --suggest.
|
|
6148
|
-
if (typeof args.flags.get('max-growth-usd') === 'string' && againstPath === undefined) {
|
|
6149
|
-
throw new Error(t.profile.maxGrowthNeedsAgainst());
|
|
6150
|
-
}
|
|
6151
|
-
/**
|
|
6152
|
-
* The money gates, armed by flags and applied on every output path.
|
|
6153
|
-
*
|
|
6154
|
-
* `check` gates tokens before the money is spent; these gate the spend
|
|
6155
|
-
* itself, from the provider's own billed counts. No period is assumed —
|
|
6156
|
-
* the budget applies to exactly the log handed in, so a nightly job that
|
|
6157
|
-
* profiles yesterday's log has a daily budget without Trazum ever
|
|
6158
|
-
* guessing what a day is.
|
|
6159
|
-
*/
|
|
6160
|
-
/**
|
|
6161
|
-
* The gate verdicts, kept so the markdown summary can carry them.
|
|
6162
|
-
*
|
|
6163
|
-
* Collected by wrapping `console.error` for the duration of `applyGates`
|
|
6164
|
-
* rather than by threading a return value through every gate. That is the
|
|
6165
|
-
* unusual choice here and it is deliberate: a gate added later reaches the
|
|
6166
|
-
* summary without anyone remembering to register it, and the alternative —
|
|
6167
|
-
* one push per verdict at a dozen call sites — is a list that goes stale
|
|
6168
|
-
* silently. Colour is stripped, because a summary is markdown and an
|
|
6169
|
-
* escape sequence in it is noise a reader has to look past.
|
|
6170
|
-
*/
|
|
6171
|
-
const gateVerdicts = [];
|
|
6172
|
-
let gateFailed = false;
|
|
6173
|
-
/**
|
|
6174
|
-
* Findings as policy. A waiver silences one gate's exit code for a bounded
|
|
6175
|
-
* time, on the record: the failure still prints (waived is shown as
|
|
6176
|
-
* waived, never hidden — the bill still counts it), the reason and the
|
|
6177
|
-
* days left print beside it, and the day the waiver expires the gate
|
|
6178
|
-
* fails again louder, naming the date and the reason somebody wrote.
|
|
6179
|
-
* That expiry is the entire mechanism by which a waiver stays a decision
|
|
6180
|
-
* instead of becoming a habit.
|
|
6181
|
-
*/
|
|
6182
|
-
const waiverFor = (gate) => {
|
|
6183
|
-
const entry = (config.waive ?? []).find((w) => w.gate === gate);
|
|
6184
|
-
if (entry === undefined)
|
|
6185
|
-
return null;
|
|
6186
|
-
// The waiver covers its named day whole: expiry begins the next UTC day.
|
|
6187
|
-
const expiresMs = Date.parse(`${entry.until}T00:00:00Z`) + 86_400_000;
|
|
6188
|
-
return { entry, expired: Date.now() >= expiresMs };
|
|
6189
|
-
};
|
|
6190
|
-
/**
|
|
6191
|
-
* Applies a waiver to one failing gate. Returns true when the failure is
|
|
6192
|
-
* silenced — the caller skips its exitCode — and prints the record either
|
|
6193
|
-
* way, because a waived failure that vanished from the output would be a
|
|
6194
|
-
* finding deleted with extra steps.
|
|
6195
|
-
*/
|
|
6196
|
-
/**
|
|
6197
|
-
* Uses recorded this run, flushed after the gates have finished.
|
|
6198
|
-
*
|
|
6199
|
-
* Collected rather than written inline because `waived` is synchronous and
|
|
6200
|
-
* called from seven places inside the gate pass. Writing from each of them
|
|
6201
|
-
* would mean seven awaits threaded through the exit-code logic — the one
|
|
6202
|
-
* part of this command where a mistake turns a red build green.
|
|
6203
|
-
*/
|
|
6204
|
-
const waiverUses = [];
|
|
6205
|
-
const waived = (gate, measuredUsd = null, limitUsd = null) => {
|
|
6206
|
-
const found = waiverFor(gate);
|
|
6207
|
-
if (found === null)
|
|
6208
|
-
return false;
|
|
6209
|
-
if (found.expired) {
|
|
6210
|
-
console.error(c.red(t.profile.waiveExpired(gate, found.entry.until, found.entry.reason)));
|
|
6211
|
-
return false;
|
|
6212
|
-
}
|
|
6213
|
-
const daysLeft = Math.max(0, Math.ceil((Date.parse(`${found.entry.until}T00:00:00Z`) + 86_400_000 - Date.now()) / 86_400_000));
|
|
6214
|
-
console.error(c.yellow(t.profile.waiveActive(gate, found.entry.reason, found.entry.until, String(daysLeft))));
|
|
6315
|
+
const daysLeft = Math.max(0, Math.ceil((Date.parse(`${found.entry.until}T00:00:00Z`) + 86_400_000 - Date.now()) / 86_400_000));
|
|
6316
|
+
console.error(c.yellow(t.profile.waiveActive(gate, found.entry.reason, found.entry.until, String(daysLeft))));
|
|
6215
6317
|
/**
|
|
6216
6318
|
* Recorded **when it silences something**, never when it is configured.
|
|
6217
6319
|
*
|
|
@@ -6544,343 +6646,1114 @@ async function commandProfile(args, config, configDir, pricing, t) {
|
|
|
6544
6646
|
}
|
|
6545
6647
|
}
|
|
6546
6648
|
};
|
|
6649
|
+
recordGates();
|
|
6650
|
+
await recordWaiverUses();
|
|
6651
|
+
return { failed: gateFailed, verdicts: gateVerdicts };
|
|
6652
|
+
}
|
|
6653
|
+
/**
|
|
6654
|
+
* The side files the caller asked for.
|
|
6655
|
+
*
|
|
6656
|
+
* Written on **both** output paths: under `--json` the human rendering returns
|
|
6657
|
+
* early, and the first version of `--csv-out` therefore wrote nothing at all
|
|
6658
|
+
* there — a flag that silently did nothing, which is the fault this repository
|
|
6659
|
+
* keeps refusing elsewhere.
|
|
6660
|
+
*
|
|
6661
|
+
* Its inputs are named because it had eleven of them and declared none: it was
|
|
6662
|
+
* a closure over `commandProfile`’s scope, so what a CSV or a markdown summary
|
|
6663
|
+
* is built from could only be learned by reading the whole body it sat in.
|
|
6664
|
+
*/
|
|
6665
|
+
async function writeProfileSideFiles(context) {
|
|
6666
|
+
const { args, path, report, previous, pressures, labelDrivers, modelDrivers, againstOverlap, whatIf, pricingStale, windowed, gates, pricing, t, } = context;
|
|
6547
6667
|
/**
|
|
6548
|
-
*
|
|
6549
|
-
*
|
|
6550
|
-
*
|
|
6551
|
-
*
|
|
6668
|
+
* Where the "wrote to" notice goes. Under `--json`, stdout carries the
|
|
6669
|
+
* report and nothing else — a status line there turns a parseable
|
|
6670
|
+
* document into a parse error, which is how a pipeline discovers the
|
|
6671
|
+
* feature. The gates already route their verdicts to stderr for the same
|
|
6672
|
+
* reason.
|
|
6552
6673
|
*/
|
|
6553
|
-
const
|
|
6554
|
-
|
|
6555
|
-
|
|
6556
|
-
|
|
6557
|
-
|
|
6558
|
-
|
|
6559
|
-
|
|
6560
|
-
|
|
6561
|
-
|
|
6562
|
-
|
|
6563
|
-
|
|
6564
|
-
/**
|
|
6565
|
-
* The same report as GitHub-flavoured markdown, for a job summary or a PR
|
|
6566
|
-
* comment. Written from the same message catalogue the terminal used, because
|
|
6567
|
-
* two renderings of one finding drift the moment they are worded twice.
|
|
6568
|
-
*/
|
|
6569
|
-
const markdownOut = stringFlag(args, 'markdown-out');
|
|
6570
|
-
const htmlOut = stringFlag(args, 'html-out');
|
|
6571
|
-
if (markdownOut !== undefined || htmlOut !== undefined) {
|
|
6572
|
-
/**
|
|
6573
|
-
* Derived here, not borrowed from the terminal path. Under `--json`
|
|
6574
|
-
* that path never runs, and the outer `levers` this block used to
|
|
6575
|
-
* reach for was an uninitialised binding — `profile --json
|
|
6576
|
-
* --markdown-out` crashed with a ReferenceError from 1.59 until the
|
|
6577
|
-
* HTML door's test drove both flags together and found it. The flag
|
|
6578
|
-
* did not silently do nothing; it loudly did nothing, and no test had
|
|
6579
|
-
* ever asked.
|
|
6580
|
-
*/
|
|
6581
|
-
const levers = billLevers(report, { catalogue: pricing });
|
|
6582
|
-
const cache = cacheEconomics(report.total);
|
|
6583
|
-
/**
|
|
6584
|
-
* One input object for both doors, built once: the HTML file and the
|
|
6585
|
-
* Markdown file are two projections of the same figures, and building
|
|
6586
|
-
* the input twice is how two renderings of one bill start disagreeing.
|
|
6587
|
-
*/
|
|
6588
|
-
const renderInput = ({
|
|
6589
|
-
report,
|
|
6590
|
-
levers,
|
|
6591
|
-
cache,
|
|
6592
|
-
t,
|
|
6593
|
-
...(windowed
|
|
6594
|
-
? { window: { since: stringFlag(args, 'since') ?? '—', until: stringFlag(args, 'until') ?? '—' } }
|
|
6595
|
-
: {}),
|
|
6596
|
-
...(pricingStale !== null ? { stalePricing: pricingStale } : {}),
|
|
6597
|
-
// The verdict, where the person reading CI will see it. recordGates()
|
|
6598
|
-
// runs before the side files for exactly this.
|
|
6599
|
-
...(gateVerdicts.length > 0 ? { gates: { failed: gateFailed, lines: gateVerdicts } } : {}),
|
|
6600
|
-
// The short form, for a reader who is not in the terminal.
|
|
6601
|
-
...(boolFlag(args, 'markdown-summary') ? { summary: true } : {}),
|
|
6602
|
-
// The repricing, when --what-if was given: computed once above and
|
|
6603
|
-
// handed over, so the summary in a pull request cannot disagree
|
|
6604
|
-
// with the terminal about what a move would cost.
|
|
6605
|
-
...(whatIf !== null ? { whatIf } : {}),
|
|
6606
|
-
pressure: pressures,
|
|
6607
|
-
// The comparison, when there was one — the same figures and the same
|
|
6608
|
-
// drivers the terminal printed, never re-derived here.
|
|
6609
|
-
...(previous !== null
|
|
6610
|
-
? {
|
|
6611
|
-
against: {
|
|
6612
|
-
previousTotalUsd: previous.total.totalUsd,
|
|
6613
|
-
previousCalls: previous.total.calls,
|
|
6614
|
-
labelDrivers,
|
|
6615
|
-
modelDrivers: new Set([
|
|
6616
|
-
...previous.byModel.map((r) => r.model),
|
|
6617
|
-
...report.byModel.map((r) => r.model),
|
|
6618
|
-
]).size > 1
|
|
6619
|
-
? modelDrivers
|
|
6620
|
-
: [],
|
|
6621
|
-
overlap: againstOverlap !== null
|
|
6622
|
-
? { from: dayOf(againstOverlap.fromMs), to: dayOf(againstOverlap.toMs) }
|
|
6623
|
-
: null,
|
|
6624
|
-
nothingPriced: previous.total.calls === 0,
|
|
6625
|
-
},
|
|
6626
|
-
}
|
|
6627
|
-
: {}),
|
|
6628
|
-
});
|
|
6629
|
-
if (markdownOut !== undefined) {
|
|
6630
|
-
await writeFile(markdownOut, renderProfileMarkdown(renderInput), 'utf8');
|
|
6631
|
-
notice(c.dim(t.report.wroteTo(markdownOut)));
|
|
6632
|
-
}
|
|
6633
|
-
if (htmlOut !== undefined) {
|
|
6634
|
-
await writeFile(htmlOut, renderProfileHtml(renderInput), 'utf8');
|
|
6635
|
-
notice(c.dim(t.html.written(htmlOut)));
|
|
6636
|
-
}
|
|
6637
|
-
}
|
|
6674
|
+
const notice = boolFlag(args, 'json')
|
|
6675
|
+
? (message) => console.error(message)
|
|
6676
|
+
: (message) => console.log(message);
|
|
6677
|
+
/**
|
|
6678
|
+
* The same report as GitHub-flavoured markdown, for a job summary or a PR
|
|
6679
|
+
* comment. Written from the same message catalogue the terminal used, because
|
|
6680
|
+
* two renderings of one finding drift the moment they are worded twice.
|
|
6681
|
+
*/
|
|
6682
|
+
const markdownOut = stringFlag(args, 'markdown-out');
|
|
6683
|
+
const htmlOut = stringFlag(args, 'html-out');
|
|
6684
|
+
if (markdownOut !== undefined || htmlOut !== undefined) {
|
|
6638
6685
|
/**
|
|
6639
|
-
*
|
|
6640
|
-
*
|
|
6641
|
-
*
|
|
6642
|
-
*
|
|
6686
|
+
* Derived here, not borrowed from the terminal path. Under `--json`
|
|
6687
|
+
* that path never runs, and the outer `levers` this block used to
|
|
6688
|
+
* reach for was an uninitialised binding — `profile --json
|
|
6689
|
+
* --markdown-out` crashed with a ReferenceError from 1.59 until the
|
|
6690
|
+
* HTML door's test drove both flags together and found it. The flag
|
|
6691
|
+
* did not silently do nothing; it loudly did nothing, and no test had
|
|
6692
|
+
* ever asked.
|
|
6643
6693
|
*/
|
|
6644
|
-
const
|
|
6645
|
-
|
|
6646
|
-
/**
|
|
6647
|
-
* Which table the file holds. One row shape per file on purpose: a
|
|
6648
|
-
* spreadsheet that has to filter before it can sum is a spreadsheet
|
|
6649
|
-
* somebody sums wrong.
|
|
6650
|
-
*/
|
|
6651
|
-
const shape = stringFlag(args, 'csv-shape') ?? 'slice';
|
|
6652
|
-
if (shape !== 'slice' && shape !== 'day' && shape !== 'hour' && shape !== 'model-day') {
|
|
6653
|
-
throw new Error(t.profile.badCsvShape(shape));
|
|
6654
|
-
}
|
|
6655
|
-
await writeFile(csvOut, profileToCsv(report, { unlabelled: t.profile.unlabelled(), shape }), 'utf8');
|
|
6656
|
-
notice(c.dim(t.report.wroteTo(csvOut)));
|
|
6657
|
-
}
|
|
6658
|
-
};
|
|
6659
|
-
if (boolFlag(args, 'json')) {
|
|
6694
|
+
const levers = billLevers(report, { catalogue: pricing });
|
|
6695
|
+
const cache = cacheEconomics(report.total);
|
|
6660
6696
|
/**
|
|
6661
|
-
*
|
|
6662
|
-
*
|
|
6663
|
-
*
|
|
6664
|
-
* returns. The cache verdict is included because leaving a consumer to
|
|
6665
|
-
* re-derive it means two implementations of a sign convention where positive
|
|
6666
|
-
* means *worse*, and one of them will eventually get it backwards.
|
|
6667
|
-
*
|
|
6668
|
-
* `levers` is included because it was not, and that made the flagship
|
|
6669
|
-
* section terminal-only: "What would actually move this bill" — the reason
|
|
6670
|
-
* the command exists — was invisible to any pipeline, dashboard or CI step
|
|
6671
|
-
* reading the JSON. A finding the machine-readable output omits is a finding
|
|
6672
|
-
* the reader's tooling will never surface.
|
|
6697
|
+
* One input object for both doors, built once: the HTML file and the
|
|
6698
|
+
* Markdown file are two projections of the same figures, and building
|
|
6699
|
+
* the input twice is how two renderings of one bill start disagreeing.
|
|
6673
6700
|
*/
|
|
6674
|
-
|
|
6675
|
-
|
|
6676
|
-
|
|
6677
|
-
|
|
6678
|
-
|
|
6679
|
-
|
|
6680
|
-
|
|
6681
|
-
|
|
6682
|
-
|
|
6683
|
-
|
|
6684
|
-
|
|
6685
|
-
|
|
6686
|
-
|
|
6687
|
-
|
|
6688
|
-
|
|
6689
|
-
|
|
6690
|
-
|
|
6691
|
-
|
|
6692
|
-
|
|
6693
|
-
|
|
6694
|
-
|
|
6695
|
-
// The provenance of every dollar above: which price table, how old.
|
|
6696
|
-
pricing: { lastReviewed: pricing.lastReviewed, ageDays: pricingAgeDays },
|
|
6697
|
-
levers: billLevers(report, { catalogue: pricing }),
|
|
6698
|
-
// Present only when --against was passed: null delta means the
|
|
6699
|
-
// previous log had nothing priced, which is a different answer from
|
|
6700
|
-
// zero growth.
|
|
6701
|
+
const renderInput = ({
|
|
6702
|
+
report,
|
|
6703
|
+
levers,
|
|
6704
|
+
cache,
|
|
6705
|
+
t,
|
|
6706
|
+
...(windowed
|
|
6707
|
+
? { window: { since: stringFlag(args, 'since') ?? '—', until: stringFlag(args, 'until') ?? '—' } }
|
|
6708
|
+
: {}),
|
|
6709
|
+
...(pricingStale !== null ? { stalePricing: pricingStale } : {}),
|
|
6710
|
+
// The verdict, where the person reading CI will see it. recordGates()
|
|
6711
|
+
// runs before the side files for exactly this.
|
|
6712
|
+
...(gates.verdicts.length > 0 ? { gates: { failed: gates.failed, lines: gates.verdicts } } : {}),
|
|
6713
|
+
// The short form, for a reader who is not in the terminal.
|
|
6714
|
+
...(boolFlag(args, 'markdown-summary') ? { summary: true } : {}),
|
|
6715
|
+
// The repricing, when --what-if was given: computed once above and
|
|
6716
|
+
// handed over, so the summary in a pull request cannot disagree
|
|
6717
|
+
// with the terminal about what a move would cost.
|
|
6718
|
+
...(whatIf !== null ? { whatIf } : {}),
|
|
6719
|
+
pressure: pressures,
|
|
6720
|
+
// The comparison, when there was one — the same figures and the same
|
|
6721
|
+
// drivers the terminal printed, never re-derived here.
|
|
6701
6722
|
...(previous !== null
|
|
6702
6723
|
? {
|
|
6703
6724
|
against: {
|
|
6704
6725
|
previousTotalUsd: previous.total.totalUsd,
|
|
6705
|
-
|
|
6706
|
-
|
|
6707
|
-
|
|
6708
|
-
|
|
6709
|
-
|
|
6710
|
-
|
|
6726
|
+
previousCalls: previous.total.calls,
|
|
6727
|
+
labelDrivers,
|
|
6728
|
+
modelDrivers: new Set([
|
|
6729
|
+
...previous.byModel.map((r) => r.model),
|
|
6730
|
+
...report.byModel.map((r) => r.model),
|
|
6731
|
+
]).size > 1
|
|
6732
|
+
? modelDrivers
|
|
6733
|
+
: [],
|
|
6734
|
+
overlap: againstOverlap !== null
|
|
6735
|
+
? { from: dayOf(againstOverlap.fromMs), to: dayOf(againstOverlap.toMs) }
|
|
6736
|
+
: null,
|
|
6737
|
+
nothingPriced: previous.total.calls === 0,
|
|
6711
6738
|
},
|
|
6712
6739
|
}
|
|
6713
6740
|
: {}),
|
|
6714
|
-
|
|
6715
|
-
|
|
6716
|
-
|
|
6717
|
-
|
|
6718
|
-
|
|
6719
|
-
|
|
6720
|
-
|
|
6721
|
-
|
|
6722
|
-
}
|
|
6723
|
-
recordGates();
|
|
6724
|
-
await recordWaiverUses();
|
|
6725
|
-
await writeSideFiles();
|
|
6726
|
-
return;
|
|
6741
|
+
});
|
|
6742
|
+
if (markdownOut !== undefined) {
|
|
6743
|
+
await writeFile(markdownOut, renderProfileMarkdown(renderInput), 'utf8');
|
|
6744
|
+
notice(c.dim(t.report.wroteTo(markdownOut)));
|
|
6745
|
+
}
|
|
6746
|
+
if (htmlOut !== undefined) {
|
|
6747
|
+
await writeFile(htmlOut, renderProfileHtml(renderInput), 'utf8');
|
|
6748
|
+
notice(c.dim(t.html.written(htmlOut)));
|
|
6749
|
+
}
|
|
6727
6750
|
}
|
|
6728
6751
|
/**
|
|
6729
|
-
*
|
|
6730
|
-
*
|
|
6731
|
-
*
|
|
6732
|
-
*
|
|
6733
|
-
|
|
6734
|
-
|
|
6735
|
-
|
|
6752
|
+
* The same report as a spreadsheet, one row per label and model — the grain
|
|
6753
|
+
* a routing or budget decision is made at. Deliberately without a total
|
|
6754
|
+
* row: a total inside a data file is summed with the data and doubles every
|
|
6755
|
+
* figure downstream.
|
|
6756
|
+
*/
|
|
6757
|
+
const csvOut = stringFlag(args, 'csv-out');
|
|
6758
|
+
if (csvOut !== undefined) {
|
|
6759
|
+
/**
|
|
6760
|
+
* Which table the file holds. One row shape per file on purpose: a
|
|
6761
|
+
* spreadsheet that has to filter before it can sum is a spreadsheet
|
|
6762
|
+
* somebody sums wrong.
|
|
6763
|
+
*/
|
|
6764
|
+
const shape = stringFlag(args, 'csv-shape') ?? 'slice';
|
|
6765
|
+
if (shape !== 'slice' && shape !== 'day' && shape !== 'hour' && shape !== 'model-day') {
|
|
6766
|
+
throw new Error(t.profile.badCsvShape(shape));
|
|
6767
|
+
}
|
|
6768
|
+
await writeFile(csvOut, profileToCsv(report, { unlabelled: t.profile.unlabelled(), shape }), 'utf8');
|
|
6769
|
+
notice(c.dim(t.report.wroteTo(csvOut)));
|
|
6770
|
+
}
|
|
6771
|
+
}
|
|
6772
|
+
/**
|
|
6773
|
+
* What this log recorded about outcomes, and what it cannot answer.
|
|
6774
|
+
*
|
|
6775
|
+
* Every finding past the totals needs a field the format does not require, and a
|
|
6776
|
+
* reader who never adds them sees a report quietly missing half of itself, with no
|
|
6777
|
+
* way to tell "nothing to report" from "nothing recorded". Named with counts rather
|
|
6778
|
+
* than booleans: twelve labelled records out of forty thousand is not a labelled
|
|
6779
|
+
* log, and a boolean would call it one.
|
|
6780
|
+
*
|
|
6781
|
+
* Outcomes print above coverage rather than below, and the two travel together here
|
|
6782
|
+
* for the same reason they do on screen: coverage is the sentence that qualifies
|
|
6783
|
+
* whatever the outcomes just claimed.
|
|
6784
|
+
*/
|
|
6785
|
+
function reportOutcomesAndCoverage(report, config, t) {
|
|
6786
|
+
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
6787
|
+
const pct = (share) => `${(share * 100).toFixed(1)}%`;
|
|
6788
|
+
/**
|
|
6789
|
+
* What this log cannot answer, and what would fix it.
|
|
6790
|
+
*
|
|
6791
|
+
* Every finding past the totals needs a field the format does not require,
|
|
6792
|
+
* and a reader who never adds them sees a report quietly missing half of
|
|
6793
|
+
* itself — with no way to tell "nothing to report" from "nothing recorded".
|
|
6794
|
+
* Named with counts rather than booleans: twelve labelled records out of
|
|
6795
|
+
* forty thousand is not a labelled log, and a boolean would call it one.
|
|
6796
|
+
*
|
|
6797
|
+
* Only fields that are actually missing are listed. A complete log gets no
|
|
6798
|
+
* section at all, because a paragraph of things that are fine is the
|
|
6799
|
+
* paragraph readers learn to skip.
|
|
6800
|
+
*/
|
|
6801
|
+
/**
|
|
6802
|
+
* Outcomes — the counterpart, where somebody recorded one.
|
|
6803
|
+
*
|
|
6804
|
+
* Printed above the coverage section rather than below it, because when this
|
|
6805
|
+
* section is present it is the most valuable thing on the page: every other
|
|
6806
|
+
* figure in this report is a cost, and this is the only one that says what
|
|
6807
|
+
* the money bought.
|
|
6808
|
+
*
|
|
6809
|
+
* Silent when nothing recorded an outcome. The coverage section below
|
|
6810
|
+
* already names the missing field and what it would unlock, and printing an
|
|
6811
|
+
* empty Outcomes heading above it would be the same sentence twice.
|
|
6812
|
+
*/
|
|
6813
|
+
{
|
|
6814
|
+
const outcomes = outcomeReport(report.outcomeTally, config.outcomes ?? null);
|
|
6815
|
+
if (outcomes.coverage.recorded > 0) {
|
|
6816
|
+
console.log();
|
|
6817
|
+
console.log(sectionHeading(t.profile.outcomeHeading()));
|
|
6818
|
+
const col = t.profile.outcomeColumns;
|
|
6819
|
+
const rows = [...outcomes.slices, ...outcomes.undeclared].map((slice) => ({
|
|
6820
|
+
value: slice.value,
|
|
6821
|
+
verdict: slice.verdict === 'success'
|
|
6822
|
+
? t.profile.verdictSuccess()
|
|
6823
|
+
: slice.verdict === 'undeclared'
|
|
6824
|
+
? t.profile.verdictUndeclared()
|
|
6825
|
+
: t.profile.verdictOther(),
|
|
6826
|
+
calls: n(slice.calls),
|
|
6827
|
+
spend: formatUsd(slice.usd),
|
|
6828
|
+
}));
|
|
6829
|
+
const w = {
|
|
6830
|
+
value: Math.max(...rows.map((r) => r.value.length), col.outcome.length),
|
|
6831
|
+
verdict: Math.max(...rows.map((r) => r.verdict.length), 0),
|
|
6832
|
+
calls: Math.max(...rows.map((r) => r.calls.length), col.calls.length),
|
|
6833
|
+
spend: Math.max(...rows.map((r) => r.spend.length), col.spend.length),
|
|
6834
|
+
};
|
|
6835
|
+
console.log(c.dim(` ${col.outcome.padEnd(w.value)} ${''.padEnd(w.verdict)} ` +
|
|
6836
|
+
`${col.calls.padStart(w.calls)} ${col.spend.padStart(w.spend)}`));
|
|
6837
|
+
for (const row of rows) {
|
|
6838
|
+
const tint = row.verdict === t.profile.verdictUndeclared()
|
|
6839
|
+
? c.yellow
|
|
6840
|
+
: row.verdict === t.profile.verdictSuccess()
|
|
6841
|
+
? c.green
|
|
6842
|
+
: c.dim;
|
|
6843
|
+
console.log(` ${row.value.padEnd(w.value)} ${tint(row.verdict.padEnd(w.verdict))} ` +
|
|
6844
|
+
`${row.calls.padStart(w.calls)} ${row.spend.padStart(w.spend)}`);
|
|
6845
|
+
}
|
|
6846
|
+
console.log();
|
|
6847
|
+
if (outcomes.successShareOfRecordedUsd !== null) {
|
|
6848
|
+
const declaredUsd = outcomes.slices.reduce((sum, slice) => sum + slice.usd, 0);
|
|
6849
|
+
console.log(` ${wrap(t.profile.outcomeRate(pct(outcomes.successShareOfRecordedUsd), formatUsd(declaredUsd)), 74, ' ')}`);
|
|
6850
|
+
}
|
|
6851
|
+
else if (outcomes.noRate !== null) {
|
|
6852
|
+
console.log(` ${c.dim(wrap(t.profile.outcomeNoRate(outcomes.noRate), 74, ' '))}`);
|
|
6853
|
+
}
|
|
6854
|
+
// What the rate does not cover, every time it is printed. A rate over a
|
|
6855
|
+
// twelfth of the bill is a rate about a twelfth of the bill.
|
|
6856
|
+
if (outcomes.coverage.unrecordedUsd > 0 && report.total.totalUsd > 0) {
|
|
6857
|
+
console.log(` ${c.yellow('!')} ${wrap(t.profile.outcomeUnrecorded(pct(outcomes.coverage.unrecordedUsd / report.total.totalUsd), formatUsd(outcomes.coverage.unrecordedUsd)), 74, ' ')}`);
|
|
6858
|
+
}
|
|
6859
|
+
/**
|
|
6860
|
+
* What an outcome costs, per workload — the finding a total cannot make.
|
|
6861
|
+
*
|
|
6862
|
+
* Printed only when at least one workload has enough recorded outcomes
|
|
6863
|
+
* to say something, and every row that cannot state a figure says which
|
|
6864
|
+
* of the five reasons applies rather than showing a blank.
|
|
6865
|
+
*/
|
|
6866
|
+
const ranking = rankPerOutcome(report.outcomeTallyByLabel.map((entry) => ({
|
|
6867
|
+
key: entry.label,
|
|
6868
|
+
calls: entry.calls,
|
|
6869
|
+
totalUsd: entry.totalUsd,
|
|
6870
|
+
tally: entry.tally,
|
|
6871
|
+
})), config.outcomes ?? null);
|
|
6872
|
+
if (ranking.byCall.length > 0) {
|
|
6873
|
+
console.log();
|
|
6874
|
+
console.log(sectionHeading(t.profile.perOutcomeHeading()));
|
|
6875
|
+
const pcol = t.profile.perOutcomeColumns;
|
|
6876
|
+
const prows = ranking.byCall.map((slice) => ({
|
|
6877
|
+
key: slice.key,
|
|
6878
|
+
perCall: formatUsd(slice.usdPerCall),
|
|
6879
|
+
perOutcome: slice.per.usdPerSuccess !== null
|
|
6880
|
+
? formatUsd(slice.per.usdPerSuccess)
|
|
6881
|
+
: t.profile.perOutcomeWithheld(slice.per.withheld ?? 'no-vocabulary', n(slice.per.successes), pct(slice.per.coverage)),
|
|
6882
|
+
recorded: pct(slice.per.coverage),
|
|
6883
|
+
}));
|
|
6884
|
+
const pw = {
|
|
6885
|
+
key: Math.max(...prows.map((r) => r.key.length), pcol.workload.length),
|
|
6886
|
+
perCall: Math.max(...prows.map((r) => r.perCall.length), pcol.perCall.length),
|
|
6887
|
+
perOutcome: Math.max(...prows.map((r) => r.perOutcome.length), pcol.perOutcome.length),
|
|
6888
|
+
recorded: Math.max(...prows.map((r) => r.recorded.length), pcol.recorded.length),
|
|
6889
|
+
};
|
|
6890
|
+
console.log(c.dim(` ${pcol.workload.padEnd(pw.key)} ${pcol.perCall.padStart(pw.perCall)} ` +
|
|
6891
|
+
`${pcol.perOutcome.padStart(pw.perOutcome)} ${pcol.recorded.padStart(pw.recorded)}`));
|
|
6892
|
+
for (const row of prows) {
|
|
6893
|
+
console.log(` ${row.key.padEnd(pw.key)} ${row.perCall.padStart(pw.perCall)} ` +
|
|
6894
|
+
`${row.perOutcome.padStart(pw.perOutcome)} ${c.dim(row.recorded.padStart(pw.recorded))}`);
|
|
6895
|
+
}
|
|
6896
|
+
console.log();
|
|
6897
|
+
console.log(` ${c.dim(wrap(t.profile.perOutcomeNumerator(), 74, ' '))}`);
|
|
6898
|
+
// The disagreement between the two orders, which is itself the finding.
|
|
6899
|
+
if (ranking.disagreements.length > 0) {
|
|
6900
|
+
console.log();
|
|
6901
|
+
console.log(` ${c.dim(wrap(t.profile.perOutcomeBothOrders(), 74, ' '))}`);
|
|
6902
|
+
for (const d of ranking.disagreements) {
|
|
6903
|
+
console.log(` ${c.yellow('→')} ${wrap(t.profile.perOutcomeDisagreement(d.slice.key, n(d.callRank + 1), n(d.outcomeRank + 1)), 74, ' ')}`);
|
|
6904
|
+
}
|
|
6905
|
+
}
|
|
6906
|
+
}
|
|
6907
|
+
if (outcomes.undeclared.length > 0) {
|
|
6908
|
+
console.log(` ${c.yellow('!')} ${wrap(t.profile.outcomeUndeclared(outcomes.undeclared.map((s) => s.value).join(', ')), 74, ' ')}`);
|
|
6909
|
+
}
|
|
6910
|
+
}
|
|
6911
|
+
}
|
|
6912
|
+
const coverage = report.fieldCoverage;
|
|
6913
|
+
if (coverage.parsed > 0) {
|
|
6914
|
+
const missing = [];
|
|
6915
|
+
const partial = (seen) => `${n(seen)}/${n(coverage.parsed)}`;
|
|
6916
|
+
if (coverage.label < coverage.parsed) {
|
|
6917
|
+
missing.push(t.profile.needsLabel(partial(coverage.label)));
|
|
6918
|
+
}
|
|
6919
|
+
if (coverage.session < coverage.parsed) {
|
|
6920
|
+
missing.push(t.profile.needsSession(partial(coverage.session)));
|
|
6921
|
+
}
|
|
6922
|
+
/**
|
|
6923
|
+
* Listed first among the missing when it is missing entirely, because it
|
|
6924
|
+
* is the one field that changes what every other figure here *means*. The
|
|
6925
|
+
* rest sharpen a cost; this one gives it a counterpart.
|
|
6926
|
+
*/
|
|
6927
|
+
if (coverage.outcome < coverage.parsed) {
|
|
6928
|
+
missing.push(t.profile.needsOutcome(partial(coverage.outcome)));
|
|
6929
|
+
}
|
|
6930
|
+
if (coverage.ts < coverage.parsed) {
|
|
6931
|
+
missing.push(t.profile.needsTs(partial(coverage.ts)));
|
|
6932
|
+
}
|
|
6933
|
+
if (coverage.stopReason < coverage.parsed) {
|
|
6934
|
+
missing.push(t.profile.needsStopReason(partial(coverage.stopReason)));
|
|
6935
|
+
}
|
|
6936
|
+
if (coverage.cacheWrites > 0 && coverage.cacheTtl < coverage.cacheWrites) {
|
|
6937
|
+
missing.push(t.profile.needsCacheTtl(`${n(coverage.cacheTtl)}/${n(coverage.cacheWrites)}`));
|
|
6938
|
+
}
|
|
6939
|
+
if (missing.length > 0) {
|
|
6940
|
+
console.log();
|
|
6941
|
+
console.log(sectionHeading(t.profile.coverageHeading()));
|
|
6942
|
+
for (const line of missing)
|
|
6943
|
+
console.log(` ${c.dim(wrap(line, 74, ' '))}`);
|
|
6944
|
+
}
|
|
6945
|
+
}
|
|
6946
|
+
}
|
|
6947
|
+
/**
|
|
6948
|
+
* This bill against the previous one: how spend actually gets out of hand.
|
|
6949
|
+
*
|
|
6950
|
+
* Nobody adds five thousand a month in one day; bills grow four percent a week
|
|
6951
|
+
* while every snapshot looks reasonable. This is the baseline gate the prompts
|
|
6952
|
+
* already had, applied to the money itself. **Positive means the bill grew** (the
|
|
6953
|
+
* diff convention), and every figure is between exactly these two files.
|
|
6954
|
+
*
|
|
6955
|
+
* Its inputs are stated because it had eight of them and declared none. Two are
|
|
6956
|
+
* drivers computed three hundred lines earlier and used nowhere else, which is a
|
|
6957
|
+
* thing you could only find out by reading both places.
|
|
6958
|
+
*/
|
|
6959
|
+
function reportAgainstPrevious(context) {
|
|
6960
|
+
const { report, previous, againstOverlap, labelDrivers, modelDrivers, gates, now, t } = context;
|
|
6961
|
+
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
6962
|
+
const pct = (share) => `${(share * 100).toFixed(1)}%`;
|
|
6963
|
+
/**
|
|
6964
|
+
* This bill against the previous one — how spend actually gets out of hand.
|
|
6965
|
+
*
|
|
6966
|
+
* Nobody adds five thousand a month in one day; bills grow four percent a week
|
|
6967
|
+
* while every snapshot looks reasonable. This is the baseline gate the prompts
|
|
6968
|
+
* already had, applied to the money itself. **Positive means the bill grew**
|
|
6969
|
+
* (the diff convention), and every figure is between exactly these two files:
|
|
6970
|
+
* no period is assumed, so the call counts print beside the money for the
|
|
6971
|
+
* reader to judge comparability before judging the trend.
|
|
6972
|
+
*/
|
|
6973
|
+
if (previous !== null) {
|
|
6974
|
+
console.log();
|
|
6975
|
+
console.log(sectionHeading(t.profile.againstHeading()));
|
|
6976
|
+
if (previous.total.calls === 0) {
|
|
6977
|
+
console.log(` ${c.dim(wrap(t.profile.againstNothingPriced(), 74, ' '))}`);
|
|
6978
|
+
}
|
|
6979
|
+
else {
|
|
6980
|
+
const delta = report.total.totalUsd - previous.total.totalUsd;
|
|
6981
|
+
const growthPct = previous.total.totalUsd > 0
|
|
6982
|
+
? `${delta >= 0 ? '+' : ''}${((delta / previous.total.totalUsd) * 100).toFixed(1)}%`
|
|
6983
|
+
: '—';
|
|
6984
|
+
console.log(` ${c.bold(wrap(t.profile.againstTotals(formatUsd(previous.total.totalUsd), formatUsd(report.total.totalUsd), formatSignedUsd(delta), growthPct, t.profile.calls(previous.total.calls), t.profile.calls(report.total.calls)), 74, ' '))}`);
|
|
6985
|
+
// Overlapping spans mean part of this "growth" is the same money on
|
|
6986
|
+
// both sides of the subtraction. Said after the figure it qualifies
|
|
6987
|
+
// and before the drivers built from it.
|
|
6988
|
+
if (againstOverlap !== null) {
|
|
6989
|
+
console.log(` ${c.yellow('!')} ${c.dim(wrap(t.profile.againstOverlap(dayOf(againstOverlap.fromMs), dayOf(againstOverlap.toMs)), 74, ' '))}`);
|
|
6990
|
+
}
|
|
6991
|
+
// Drivers: per-key contribution to the change, largest magnitude first,
|
|
6992
|
+
// computed once beside the gates so no rendering derives its own.
|
|
6993
|
+
const driverLine = (d, shown) => d.was === null
|
|
6994
|
+
? t.profile.againstDriverNew(formatSignedUsd(d.delta), shown)
|
|
6995
|
+
: d.now === null
|
|
6996
|
+
? t.profile.againstDriverGone(formatSignedUsd(d.delta), shown)
|
|
6997
|
+
: t.profile.againstDriver(formatSignedUsd(d.delta), shown, formatUsd(d.was), formatUsd(d.now));
|
|
6998
|
+
console.log();
|
|
6999
|
+
for (const d of labelDrivers.slice(0, 5)) {
|
|
7000
|
+
const line = driverLine(d, d.key === UNLABELLED ? t.profile.unlabelled() : d.key);
|
|
7001
|
+
console.log(` ${d.delta > 0 ? c.yellow(line) : c.dim(line)}`);
|
|
7002
|
+
}
|
|
7003
|
+
if (labelDrivers.length > 5) {
|
|
7004
|
+
console.log(` ${c.dim(t.profile.andMoreLabels(labelDrivers.length - 5))}`);
|
|
7005
|
+
}
|
|
7006
|
+
/**
|
|
7007
|
+
* The same change, by model — where the mix moved. The label rows cannot
|
|
7008
|
+
* show it: a workload that kept its name and switched from Haiku to Opus
|
|
7009
|
+
* reads as "chat grew", and the reason is the model. Only printed when
|
|
7010
|
+
* more than one model is involved; with one model on both sides, this
|
|
7011
|
+
* section restates the totals line and says nothing new.
|
|
7012
|
+
*/
|
|
7013
|
+
const modelsInvolved = new Set([
|
|
7014
|
+
...previous.byModel.map((r) => r.model),
|
|
7015
|
+
...report.byModel.map((r) => r.model),
|
|
7016
|
+
]);
|
|
7017
|
+
if (modelDrivers.length > 0 && modelsInvolved.size > 1) {
|
|
7018
|
+
console.log();
|
|
7019
|
+
console.log(` ${c.dim(t.profile.againstByModel())}`);
|
|
7020
|
+
for (const d of modelDrivers.slice(0, 3)) {
|
|
7021
|
+
const line = driverLine(d, d.key);
|
|
7022
|
+
console.log(` ${d.delta > 0 ? c.yellow(line) : c.dim(line)}`);
|
|
7023
|
+
}
|
|
7024
|
+
}
|
|
7025
|
+
/**
|
|
7026
|
+
* What the comparison stopped being able to see.
|
|
7027
|
+
*
|
|
7028
|
+
* Every figure above is dollars, and dollars cannot tell a finding that
|
|
7029
|
+
* was fixed from a finding whose field the log stopped recording — both
|
|
7030
|
+
* are silence. This is the only section that can, so it is loud: a
|
|
7031
|
+
* collapse in coverage invalidates whichever findings depended on it,
|
|
7032
|
+
* and reading the drop as good news is the specific mistake it exists
|
|
7033
|
+
* to prevent.
|
|
7034
|
+
*/
|
|
7035
|
+
const drifts = coverageDrift(previous.fieldCoverage, report.fieldCoverage);
|
|
7036
|
+
if (drifts.length > 0) {
|
|
7037
|
+
console.log();
|
|
7038
|
+
for (const drift of drifts) {
|
|
7039
|
+
const line = t.profile.coverageDrift(t.profile.coverageField(drift.field), pct(drift.was), pct(drift.now));
|
|
7040
|
+
console.log(drift.delta < 0
|
|
7041
|
+
? ` ${c.yellow('!')} ${c.bold(wrap(line, 74, ' '))}`
|
|
7042
|
+
: ` ${c.dim(wrap(line, 74, ' '))}`);
|
|
7043
|
+
/**
|
|
7044
|
+
* Which findings went with it, named. "Some findings are silent" is
|
|
7045
|
+
* not something a reader can act on; knowing that conversation
|
|
7046
|
+
* growth and the cache-TTL fit are now silence rather than absence
|
|
7047
|
+
* tells them exactly which sections of this report to distrust.
|
|
7048
|
+
*/
|
|
7049
|
+
if (drift.delta < 0) {
|
|
7050
|
+
const silenced = t.profile.coverageSilenced(drift.field);
|
|
7051
|
+
if (silenced !== '')
|
|
7052
|
+
console.log(` ${c.dim(wrap(silenced, 72, ' '))}`);
|
|
7053
|
+
}
|
|
7054
|
+
}
|
|
7055
|
+
if (drifts.some((d) => d.delta < 0)) {
|
|
7056
|
+
console.log(` ${c.dim(wrap(t.profile.coverageDriftWhy(), 74, ' '))}`);
|
|
7057
|
+
}
|
|
7058
|
+
}
|
|
7059
|
+
}
|
|
7060
|
+
}
|
|
7061
|
+
}
|
|
7062
|
+
/**
|
|
7063
|
+
* The same request, sent again a moment later.
|
|
7064
|
+
*
|
|
7065
|
+
* A conversation's input grows with every turn, so two calls of the same size
|
|
7066
|
+
* seconds apart are not a conversation: they are a retry, a duplicated step, a
|
|
7067
|
+
* loop. Loud, because the money bought nothing, and hedged, because this reads
|
|
7068
|
+
* counts and cannot see content.
|
|
7069
|
+
*/
|
|
7070
|
+
function reportRepeatedCalls(report, t) {
|
|
7071
|
+
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
7072
|
+
const pct = (share) => `${(share * 100).toFixed(1)}%`;
|
|
7073
|
+
/**
|
|
7074
|
+
* The same request, sent again a moment later.
|
|
7075
|
+
*
|
|
7076
|
+
* A conversation's input grows with every turn, so two consecutive calls in
|
|
7077
|
+
* one conversation carrying the same size seconds apart is a thing going
|
|
7078
|
+
* wrong rather than a thing working — a retry after a timeout, an agent
|
|
7079
|
+
* step repeating, a loop. Loud, because the money bought nothing, and
|
|
7080
|
+
* hedged, because this reads counts and cannot see content: the sentence
|
|
7081
|
+
* says the pattern is *usually* a retry, never that it is one.
|
|
7082
|
+
*/
|
|
7083
|
+
if (report.repeatedTurns.length > 0) {
|
|
7084
|
+
console.log();
|
|
7085
|
+
console.log(sectionHeading(t.profile.repeatsHeading()));
|
|
7086
|
+
for (const row of report.repeatedTurns.slice(0, 3)) {
|
|
7087
|
+
const label = row.label === UNLABELLED ? t.profile.unlabelled() : row.label;
|
|
7088
|
+
console.log();
|
|
7089
|
+
console.log(` ${c.yellow('!')} ${c.bold(wrap(t.profile.repeatsFound(label, row.modelName, n(row.repeats), n(row.checkedCalls), n(Math.round(row.withinMs / 1000)), formatUsd(row.usd)), 74, ' '))}`);
|
|
7090
|
+
console.log(` ${c.dim(wrap(t.profile.repeatsAdvice(), 74, ' '))}`);
|
|
7091
|
+
}
|
|
7092
|
+
}
|
|
7093
|
+
/**
|
|
7094
|
+
* Output spend that bought answers cut off mid-generation — the one slice of
|
|
7095
|
+
* a bill that is waste without a counterpart. Paid in full, frequently
|
|
7096
|
+
* retried and billed again, and the truncated attempt bought nothing.
|
|
7097
|
+
*
|
|
7098
|
+
* Three states, kept apart on purpose: waste found, none found on a log that
|
|
7099
|
+
* measured, and a log that never recorded a stop reason at all — which gets
|
|
7100
|
+
* the missing-field message, because silence there would read as a clean bill
|
|
7101
|
+
* of health on a question the log never asked.
|
|
7102
|
+
*/
|
|
7103
|
+
if (report.total.truncatedCalls > 0 && report.total.outputUsd > 0) {
|
|
7104
|
+
console.log();
|
|
7105
|
+
console.log(` ${c.yellow('!')} ${c.bold(wrap(t.profile.truncatedWaste(t.profile.calls(report.total.truncatedCalls), formatUsd(report.total.truncatedOutputUsd), pct(report.total.truncatedOutputUsd / report.total.outputUsd)), 74, ' '))}`);
|
|
7106
|
+
/**
|
|
7107
|
+
* Which workloads are paying for it, and at what rate — the actionable
|
|
7108
|
+
* half the total hides. A 40% truncation rate is a max_tokens setting
|
|
7109
|
+
* that is simply wrong; 1% is a long tail, and the two call for opposite
|
|
7110
|
+
* responses.
|
|
7111
|
+
*
|
|
7112
|
+
* The rate is over calls that **recorded a stop reason**, never over all
|
|
7113
|
+
* calls: a workload that logs the field on half its traffic must not be
|
|
7114
|
+
* reported as though the unmeasured half completed. Both numbers print,
|
|
7115
|
+
* so the denominator is visible rather than implied.
|
|
7116
|
+
*/
|
|
7117
|
+
const truncatedLabels = report.byLabel
|
|
7118
|
+
.filter((row) => row.breakdown.truncatedCalls > 0)
|
|
7119
|
+
.sort((a, b) => b.breakdown.truncatedOutputUsd - a.breakdown.truncatedOutputUsd);
|
|
7120
|
+
if (truncatedLabels.length > 0 && report.byLabel.length > 1) {
|
|
7121
|
+
for (const row of truncatedLabels.slice(0, 3)) {
|
|
7122
|
+
const name = row.label === UNLABELLED ? t.profile.unlabelled() : row.label;
|
|
7123
|
+
console.log(` ${c.dim(wrap(t.profile.truncatedBy(name, n(row.breakdown.truncatedCalls), n(row.breakdown.stopReasonCalls), pct(row.breakdown.truncatedCalls / row.breakdown.stopReasonCalls), formatUsd(row.breakdown.truncatedOutputUsd)), 74, ' '))}`);
|
|
7124
|
+
}
|
|
7125
|
+
}
|
|
7126
|
+
/**
|
|
7127
|
+
* The ceiling the completed answers actually needed, when the output
|
|
7128
|
+
* shapes measured it: "95% of the answers that finished fit within N
|
|
7129
|
+
* tokens" is the number a max_tokens cap wants, and it sits next to the
|
|
7130
|
+
* evidence that the current cap is too low. Measured on these calls,
|
|
7131
|
+
* promised for nothing.
|
|
7132
|
+
*/
|
|
7133
|
+
const ceiling = report.outputShapes.find((shape) => shape.p95WithinTokens !== null);
|
|
7134
|
+
if (ceiling !== undefined) {
|
|
7135
|
+
console.log(` ${c.dim(wrap(t.profile.truncatedCeiling(n(ceiling.p95WithinTokens)), 74, ' '))}`);
|
|
7136
|
+
}
|
|
7137
|
+
/**
|
|
7138
|
+
* The "billed again" half, measured. The sentence above has always said
|
|
7139
|
+
* truncated answers are frequently retried; this is the count and the
|
|
7140
|
+
* money, when the log can carry it — a pattern, never a certainty, and
|
|
7141
|
+
* attributed to the truncated call's slice, where the ceiling that
|
|
7142
|
+
* caused it lives.
|
|
7143
|
+
*/
|
|
7144
|
+
for (const row of report.truncationRetries.slice(0, 3)) {
|
|
7145
|
+
const name = row.label === UNLABELLED ? t.profile.unlabelled() : row.label;
|
|
7146
|
+
console.log(` ${c.yellow('!')} ${c.bold(wrap(t.profile.truncationRetryLine(name, row.modelName, n(row.retried), n(row.truncatedCalls), n(Math.round(row.withinMs / 1000)), formatUsd(row.wastedUsd), formatUsd(row.retryUsd)), 74, ' '))}`);
|
|
7147
|
+
}
|
|
7148
|
+
if (report.truncationRetries.length > 0) {
|
|
7149
|
+
console.log(` ${c.dim(wrap(t.profile.truncationRetryNote(), 74, ' '))}`);
|
|
7150
|
+
}
|
|
7151
|
+
}
|
|
7152
|
+
else if (report.total.stopReasonCalls === 0) {
|
|
7153
|
+
console.log();
|
|
7154
|
+
console.log(` ${c.dim(wrap(t.profile.truncatedNotRecorded(), 74, ' '))}`);
|
|
7155
|
+
}
|
|
7156
|
+
}
|
|
7157
|
+
/**
|
|
7158
|
+
* The mix moving inside one log: the drift `--against` needs two files to see.
|
|
7159
|
+
*
|
|
7160
|
+
* Spoken only past fifteen points, and hedged in the copy while the data states
|
|
7161
|
+
* exact shares, because a mix that moved is a question rather than a finding.
|
|
7162
|
+
*/
|
|
7163
|
+
function reportMixDrift(report, t) {
|
|
7164
|
+
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
7165
|
+
const pct = (share) => `${(share * 100).toFixed(1)}%`;
|
|
7166
|
+
/**
|
|
7167
|
+
* The mix moving inside one log — the drift `--against` needs a second log
|
|
7168
|
+
* to see. Spoken only past fifteen points, a presentation threshold stated
|
|
7169
|
+
* in the copy; the data states exact shares and the JSON carries them all.
|
|
7170
|
+
*/
|
|
7171
|
+
if (report.modelMixDrift !== null) {
|
|
7172
|
+
const moved = report.modelMixDrift.models.filter((m) => Math.abs(m.lastShare - m.firstShare) >= 0.15);
|
|
7173
|
+
if (moved.length > 0) {
|
|
7174
|
+
console.log();
|
|
7175
|
+
console.log(sectionHeading(t.profile.mixDriftHeading()));
|
|
7176
|
+
for (const m of moved.slice(0, 3)) {
|
|
7177
|
+
console.log();
|
|
7178
|
+
console.log(` ${c.yellow('!')} ${c.bold(wrap(t.profile.mixDriftLine(m.model, pct(m.firstShare), pct(m.lastShare), n(report.modelMixDrift.firstDays), n(report.modelMixDrift.lastDays), formatUsd(m.lastUsd)), 74, ' '))}`);
|
|
7179
|
+
}
|
|
7180
|
+
console.log(` ${c.dim(wrap(t.profile.mixDriftNote(), 74, ' '))}`);
|
|
7181
|
+
}
|
|
7182
|
+
}
|
|
7183
|
+
}
|
|
7184
|
+
/**
|
|
7185
|
+
* How close the largest call is to the model's ceiling.
|
|
7186
|
+
*
|
|
7187
|
+
* A window is a wall, not a budget: the call that hits it fails rather than costing
|
|
7188
|
+
* more, so this is a different warning from every dollar above it and is kept a
|
|
7189
|
+
* different shape.
|
|
7190
|
+
*/
|
|
7191
|
+
function reportContextPressure(pressures, t) {
|
|
7192
|
+
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
7193
|
+
const pct = (share) => `${(share * 100).toFixed(1)}%`;
|
|
7194
|
+
/**
|
|
7195
|
+
* How close the largest call is to the model's ceiling.
|
|
7196
|
+
*
|
|
7197
|
+
* The failure a bill cannot show: input grows turn by turn or document by
|
|
7198
|
+
* document, costs nothing extra to grow, and then one call crosses the
|
|
7199
|
+
* context window and the API refuses it. Loud from 85% — close enough
|
|
7200
|
+
* that the next retrieval bump or long conversation plausibly crosses —
|
|
7201
|
+
* and quiet above half, so the reader sees it coming either way. No
|
|
7202
|
+
* prediction of *when*: a straight line through two points is a guess
|
|
7203
|
+
* wearing arithmetic's clothes.
|
|
7204
|
+
*/
|
|
7205
|
+
{
|
|
7206
|
+
if (pressures.length > 0) {
|
|
7207
|
+
console.log();
|
|
7208
|
+
console.log(sectionHeading(t.profile.pressureHeading()));
|
|
7209
|
+
for (const row of pressures.slice(0, 3)) {
|
|
7210
|
+
const label = row.label === UNLABELLED ? t.profile.unlabelled() : row.label;
|
|
7211
|
+
const line = t.profile.pressureLine(label, row.modelName, n(row.maxCallInputTokens), n(row.contextWindow), pct(row.share));
|
|
7212
|
+
console.log();
|
|
7213
|
+
if (row.share >= 0.85) {
|
|
7214
|
+
console.log(` ${c.yellow('!')} ${c.bold(wrap(line, 74, ' '))}`);
|
|
7215
|
+
console.log(` ${c.dim(wrap(t.profile.pressureAdvice(), 74, ' '))}`);
|
|
7216
|
+
}
|
|
7217
|
+
else {
|
|
7218
|
+
console.log(` ${c.dim(wrap(line, 74, ' '))}`);
|
|
7219
|
+
}
|
|
7220
|
+
}
|
|
7221
|
+
}
|
|
7222
|
+
}
|
|
7223
|
+
}
|
|
7224
|
+
/**
|
|
7225
|
+
* How big the calls themselves are, which is the other half of the bill.
|
|
7226
|
+
*
|
|
7227
|
+
* The totals say what was spent; this says whether it went on many small calls or a
|
|
7228
|
+
* few enormous ones, and those two bills are fixed by different things.
|
|
7229
|
+
*/
|
|
7230
|
+
function reportInputShape(report, t) {
|
|
7231
|
+
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
7232
|
+
const pct = (share) => `${(share * 100).toFixed(1)}%`;
|
|
7233
|
+
/**
|
|
7234
|
+
* How big the calls themselves are — the other half of the bill.
|
|
7235
|
+
*
|
|
7236
|
+
* The section above describes output; on a RAG or agent workload input is
|
|
7237
|
+
* most of the invoice, and a total could only ever say "input is 63% of
|
|
7238
|
+
* this bill", which nobody can act on. The actionable question is whether
|
|
7239
|
+
* the ordinary call is large or a few calls are enormous, and those two
|
|
7240
|
+
* shapes want opposite responses: a cap on something, or a shorter prompt.
|
|
7241
|
+
*
|
|
7242
|
+
* Loud past **four times** the median — far enough from an even
|
|
7243
|
+
* distribution that a bucket boundary cannot flip the message, and stated
|
|
7244
|
+
* in the sentence rather than hidden here. Both figures are bucket
|
|
7245
|
+
* ceilings, so the ratio is coarse by construction and the copy says so.
|
|
7246
|
+
*/
|
|
7247
|
+
if (report.inputShapes.length > 0) {
|
|
7248
|
+
console.log();
|
|
7249
|
+
console.log(sectionHeading(t.profile.inputShapeHeading()));
|
|
7250
|
+
for (const shape of report.inputShapes.slice(0, 3)) {
|
|
7251
|
+
const label = shape.label === UNLABELLED ? t.profile.unlabelled() : shape.label;
|
|
7252
|
+
console.log();
|
|
7253
|
+
if (shape.medianWithinTokens === null || shape.p95WithinTokens === null || shape.p95OverMedian === null) {
|
|
7254
|
+
/**
|
|
7255
|
+
* The covering bucket is the open-ended last one, so there is no
|
|
7256
|
+
* ceiling to name. Said rather than skipped: a slice whose calls are
|
|
7257
|
+
* all above a million tokens is a finding, and silence would drop it.
|
|
7258
|
+
*/
|
|
7259
|
+
console.log(` ${c.bold(wrap(t.profile.inputHuge(label, shape.modelName, t.profile.calls(shape.calls), formatUsd(shape.inputUsd)), 74, ' '))}`);
|
|
7260
|
+
continue;
|
|
7261
|
+
}
|
|
7262
|
+
const skewed = shape.p95OverMedian >= 4;
|
|
7263
|
+
const line = skewed
|
|
7264
|
+
? t.profile.inputSkewed(label, shape.modelName, n(shape.medianWithinTokens), n(shape.p95WithinTokens), shape.p95OverMedian.toFixed(1), formatUsd(shape.inputUsd))
|
|
7265
|
+
: t.profile.inputEven(label, shape.modelName, n(shape.medianWithinTokens), n(shape.p95WithinTokens), formatUsd(shape.inputUsd));
|
|
7266
|
+
console.log(` ${c.bold(wrap(line, 74, ' '))}`);
|
|
7267
|
+
console.log(` ${c.dim(wrap(skewed ? t.profile.inputSkewedAdvice() : t.profile.inputEvenAdvice(), 74, ' '))}`);
|
|
7268
|
+
/**
|
|
7269
|
+
* What that size actually costs. A cache read is a tenth of input on
|
|
7270
|
+
* Anthropic, so a large slice reading almost everything from cache is a
|
|
7271
|
+
* very different bill from one paying full rate — and the token counts
|
|
7272
|
+
* alone cannot tell them apart.
|
|
7273
|
+
*/
|
|
7274
|
+
if (shape.cachedShare >= 0.5) {
|
|
7275
|
+
console.log(` ${c.dim(wrap(t.profile.inputMostlyCached(pct(shape.cachedShare)), 74, ' '))}`);
|
|
7276
|
+
}
|
|
7277
|
+
else if (shape.cachedShare < 0.1) {
|
|
7278
|
+
console.log(` ${c.dim(wrap(t.profile.inputFullRate(), 74, ' '))}`);
|
|
7279
|
+
}
|
|
7280
|
+
}
|
|
7281
|
+
}
|
|
7282
|
+
}
|
|
7283
|
+
/**
|
|
7284
|
+
* Where the output spend concentrates.
|
|
7285
|
+
*
|
|
7286
|
+
* Output is usually most of the bill, so the shape of it is usually the actionable
|
|
7287
|
+
* half of the report rather than a footnote to the totals.
|
|
7288
|
+
*/
|
|
7289
|
+
function reportOutputShape(report, t) {
|
|
7290
|
+
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
7291
|
+
const pct = (share) => `${(share * 100).toFixed(1)}%`;
|
|
7292
|
+
/**
|
|
7293
|
+
* Where the output spend concentrates — the actionable half of "output
|
|
7294
|
+
* dominates", which the headline above could only state as a total.
|
|
7295
|
+
*
|
|
7296
|
+
* Two bills with identical output spend want opposite responses. Six per cent
|
|
7297
|
+
* of calls holding half of it is a tail, and a tail has a cause worth a
|
|
7298
|
+
* morning; forty-five per cent is what evenly spread looks like, and the only
|
|
7299
|
+
* lever there is asking every answer to be shorter. The threshold between the
|
|
7300
|
+
* two is a quarter of the calls — far enough from both shapes that rounding
|
|
7301
|
+
* cannot flip the message, and stated here because it is a presentation choice,
|
|
7302
|
+
* not a measurement.
|
|
7303
|
+
*/
|
|
7304
|
+
if (report.outputShapes.length > 0) {
|
|
7305
|
+
console.log();
|
|
7306
|
+
console.log(sectionHeading(t.profile.outputShapeHeading()));
|
|
7307
|
+
for (const shape of report.outputShapes.slice(0, 3)) {
|
|
7308
|
+
const label = shape.label === UNLABELLED ? t.profile.unlabelled() : shape.label;
|
|
7309
|
+
const isTail = shape.heavyCallShare < 0.25;
|
|
7310
|
+
console.log();
|
|
7311
|
+
if (isTail) {
|
|
7312
|
+
console.log(` ${c.bold(wrap(t.profile.outputTail(label, shape.modelName, pct(shape.heavyCallShare), pct(shape.heavySpendShare), n(shape.aboveTokens), formatUsd(shape.outputUsd)), 74, ' '))}`);
|
|
7313
|
+
console.log(` ${c.dim(wrap(t.profile.outputTailAdvice(), 74, ' '))}`);
|
|
7314
|
+
}
|
|
7315
|
+
else {
|
|
7316
|
+
console.log(` ${c.bold(wrap(t.profile.outputFlat(label, shape.modelName, pct(shape.heavyCallShare), pct(shape.heavySpendShare), formatUsd(shape.outputUsd)), 74, ' '))}`);
|
|
7317
|
+
console.log(` ${c.dim(wrap(t.profile.outputFlatAdvice(), 74, ' '))}`);
|
|
7318
|
+
}
|
|
7319
|
+
/**
|
|
7320
|
+
* The ceilings a max_tokens cap actually wants, exact over the
|
|
7321
|
+
* histogram: every measured answer at or under the named number is
|
|
7322
|
+
* counted, none interpolated. Omitted when the covering bucket is the
|
|
7323
|
+
* open-ended last one, which has no ceiling to name honestly.
|
|
7324
|
+
*/
|
|
7325
|
+
if (shape.medianWithinTokens !== null && shape.p95WithinTokens !== null) {
|
|
7326
|
+
console.log(` ${c.dim(wrap(t.profile.outputPercentiles(n(shape.medianWithinTokens), n(shape.p95WithinTokens)), 74, ' '))}`);
|
|
7327
|
+
}
|
|
7328
|
+
}
|
|
7329
|
+
}
|
|
7330
|
+
}
|
|
7331
|
+
/**
|
|
7332
|
+
* What re-sending the conversation costs.
|
|
7333
|
+
*
|
|
7334
|
+
* A chat or agent workload sends the whole conversation every turn, so the history
|
|
7335
|
+
* is paid for again on each call. It is one of the four levers with real money on
|
|
7336
|
+
* it and the one least likely to be noticed, because nothing in the bill is
|
|
7337
|
+
* labelled "history".
|
|
7338
|
+
*/
|
|
7339
|
+
function reportConversationHistory(report, t) {
|
|
7340
|
+
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
7341
|
+
const pct = (share) => `${(share * 100).toFixed(1)}%`;
|
|
7342
|
+
/**
|
|
7343
|
+
* What re-sending the conversation costs — the line nothing here could see.
|
|
7344
|
+
*
|
|
7345
|
+
* A chat or agent workload sends the whole conversation back on every turn, so
|
|
7346
|
+
* the input grows with the turn count and that growth is routinely the largest
|
|
7347
|
+
* item on the bill. A prompt file shows the system prompt and not the history; a
|
|
7348
|
+
* total shows the sum and not the shape.
|
|
7349
|
+
*
|
|
7350
|
+
* Reported as a **ceiling**, because part of the growth is the user's own new
|
|
7351
|
+
* messages and this reads counts rather than content, so it cannot separate the
|
|
7352
|
+
* two. Saying nothing because the exact split is unknowable would be worse: the
|
|
7353
|
+
* bound is exact, and the reader can act on it.
|
|
7354
|
+
*/
|
|
7355
|
+
if (report.conversations.length > 0) {
|
|
7356
|
+
console.log();
|
|
7357
|
+
console.log(sectionHeading(t.profile.historyHeading()));
|
|
7358
|
+
for (const growth of report.conversations.slice(0, 3)) {
|
|
7359
|
+
const label = growth.label === UNLABELLED ? t.profile.unlabelled() : growth.label;
|
|
7360
|
+
console.log();
|
|
7361
|
+
console.log(` ${c.bold(wrap(t.profile.historyGrowth(label, growth.modelName, n(Math.round(growth.minTurnTokens)), n(Math.round(growth.maxTurnTokens)), n(growth.longestSession)), 74, ' '))}`);
|
|
7362
|
+
console.log(` ${c.dim(wrap(t.profile.historyCeiling(formatUsd(growth.growthUsd), pct(growth.shareOfBill), formatUsd(growth.flatUsd), formatUsd(growth.inputUsd)), 74, ' '))}`);
|
|
7363
|
+
}
|
|
7364
|
+
}
|
|
7365
|
+
else if (!report.hasSessions) {
|
|
7366
|
+
/**
|
|
7367
|
+
* Not the same as "no growth". A log without a session field cannot be asked
|
|
7368
|
+
* the question at all, and silence there would read as a clean bill of health
|
|
7369
|
+
* on the line most likely to be the biggest.
|
|
7370
|
+
*/
|
|
7371
|
+
console.log();
|
|
7372
|
+
console.log(sectionHeading(t.profile.historyHeading()));
|
|
7373
|
+
console.log(` ${c.dim(wrap(t.profile.historyNoSessions(), 74, ' '))}`);
|
|
7374
|
+
}
|
|
7375
|
+
}
|
|
7376
|
+
/**
|
|
7377
|
+
* `--what-if <model>`: these exact calls, at another model's prices.
|
|
7378
|
+
*
|
|
7379
|
+
* Priced from the tokens that were actually billed rather than from an estimate,
|
|
7380
|
+
* which is the only version of this question worth answering: a migration priced
|
|
7381
|
+
* off a guess is a forecast, and this product does not forecast.
|
|
7382
|
+
*/
|
|
7383
|
+
function reportWhatIf(whatIf, report, t) {
|
|
7384
|
+
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
7385
|
+
const pct = (share) => `${(share * 100).toFixed(1)}%`;
|
|
7386
|
+
/**
|
|
7387
|
+
* `--what-if <model>`: these exact calls, at another model's rates.
|
|
6736
7388
|
*
|
|
6737
|
-
*
|
|
6738
|
-
*
|
|
7389
|
+
* The levers above pick their own candidate; this answers the question the
|
|
7390
|
+
* reader arrived with. It is multiplication, not advice, and every part of
|
|
7391
|
+
* this section is built so it cannot be read as advice:
|
|
7392
|
+
*
|
|
7393
|
+
* - the caveat line prints **before** the figure, not after it;
|
|
7394
|
+
* - calls the target's context window could not have accepted are named as
|
|
7395
|
+
* impossible rather than priced as cheap, and their money is in none of
|
|
7396
|
+
* the totals;
|
|
7397
|
+
* - spend already on the target is stated separately, because a difference
|
|
7398
|
+
* computed over money that cannot move is a percentage of the wrong
|
|
7399
|
+
* denominator.
|
|
6739
7400
|
*/
|
|
6740
|
-
if (
|
|
7401
|
+
if (whatIf !== null) {
|
|
6741
7402
|
console.log();
|
|
6742
|
-
console.log(
|
|
6743
|
-
|
|
6744
|
-
|
|
7403
|
+
console.log(sectionHeading(t.profile.whatIfHeading(whatIf.target.displayName)));
|
|
7404
|
+
console.log(` ${c.dim(wrap(t.profile.whatIfAssumption(), 74, ' '))}`);
|
|
7405
|
+
console.log();
|
|
7406
|
+
if (whatIf.slices.length === 0) {
|
|
7407
|
+
console.log(` ${c.dim(wrap(t.profile.whatIfNothingToMove(), 74, ' '))}`);
|
|
7408
|
+
}
|
|
7409
|
+
else {
|
|
7410
|
+
const cheaper = whatIf.deltaUsd < 0;
|
|
7411
|
+
const line = t.profile.whatIfTotal(formatUsd(whatIf.currentUsd), formatUsd(whatIf.targetUsd), formatUsd(Math.abs(whatIf.deltaUsd)));
|
|
7412
|
+
console.log(` ${cheaper ? c.green('→') : c.yellow('!')} ${c.bold(wrap(line, 74, ' '))}`);
|
|
7413
|
+
console.log(` ${c.dim(wrap(cheaper ? t.profile.whatIfCheaper() : t.profile.whatIfDearer(), 74, ' '))}`);
|
|
7414
|
+
/**
|
|
7415
|
+
* The other half of the whole decision: the same move with the target's
|
|
7416
|
+
* Batch API on top. Computed against the target's rates, never by
|
|
7417
|
+
* adding two savings, and hedged the only honest way — whether these
|
|
7418
|
+
* calls can wait is not in the log.
|
|
7419
|
+
*/
|
|
7420
|
+
if (whatIf.batchOnTarget !== null) {
|
|
7421
|
+
console.log(` ${c.dim(wrap(t.profile.whatIfBatchOnTarget(formatUsd(whatIf.batchOnTarget.targetUsd), formatUsd(whatIf.targetUsd)), 74, ' '))}`);
|
|
7422
|
+
}
|
|
7423
|
+
for (const slice of whatIf.slices.slice(0, 5)) {
|
|
7424
|
+
const label = slice.label === UNLABELLED ? t.profile.unlabelled() : slice.label;
|
|
7425
|
+
console.log(` ${c.dim('·')} ${c.dim(wrap(t.profile.whatIfSlice(label, slice.model, formatUsd(slice.currentUsd), formatUsd(slice.targetUsd)), 74, ' '))}`);
|
|
7426
|
+
/**
|
|
7427
|
+
* Cache traffic the target could not grant, said loudly and beside
|
|
7428
|
+
* the figure it corrects. The standard row prices cache entries the
|
|
7429
|
+
* target's minimum would refuse to create — an error that flatters
|
|
7430
|
+
* the move, which is the direction this repository refuses.
|
|
7431
|
+
*/
|
|
7432
|
+
if (slice.cacheBeyondTarget !== null) {
|
|
7433
|
+
console.log(` ${c.yellow('!')} ${c.dim(wrap(t.profile.whatIfCacheBeyond(n(slice.maxCallInputTokens), n(slice.cacheBeyondTarget.minTokens), formatUsd(slice.cacheBeyondTarget.noCacheUsd)), 74, ' '))}`);
|
|
7434
|
+
}
|
|
7435
|
+
}
|
|
7436
|
+
}
|
|
7437
|
+
/**
|
|
7438
|
+
* The refusal, and it is loud. A call larger than the target's window is
|
|
7439
|
+
* not a cheaper call, and a comparison that priced it anyway would report
|
|
7440
|
+
* a saving for traffic that would have failed outright.
|
|
7441
|
+
*/
|
|
7442
|
+
for (const slice of whatIf.overContext.slice(0, 3)) {
|
|
7443
|
+
const label = slice.label === UNLABELLED ? t.profile.unlabelled() : slice.label;
|
|
7444
|
+
console.log(` ${c.yellow('!')} ${c.bold(wrap(t.profile.whatIfOverContext(label, n(slice.maxCallInputTokens), n(whatIf.target.contextWindow), formatUsd(slice.currentUsd)), 74, ' '))}`);
|
|
7445
|
+
}
|
|
7446
|
+
// Money that is already there cannot move, and leaving it out of the
|
|
7447
|
+
// totals above is only honest if the reader is told it exists.
|
|
7448
|
+
if (whatIf.alreadyOnTarget.calls > 0) {
|
|
7449
|
+
console.log(` ${c.dim(wrap(t.profile.whatIfAlreadyThere(t.profile.calls(whatIf.alreadyOnTarget.calls), formatUsd(whatIf.alreadyOnTarget.usd)), 74, ' '))}`);
|
|
7450
|
+
}
|
|
7451
|
+
// Models with no current price have no difference to state — their target
|
|
7452
|
+
// cost is knowable and the subtraction is not.
|
|
7453
|
+
if (whatIf.unpricedCalls > 0) {
|
|
7454
|
+
console.log(` ${c.dim(wrap(t.profile.whatIfUnpriced(t.profile.calls(whatIf.unpricedCalls), whatIf.unpricedModels.join(', ')), 74, ' '))}`);
|
|
7455
|
+
}
|
|
6745
7456
|
}
|
|
6746
|
-
|
|
6747
|
-
|
|
6748
|
-
|
|
6749
|
-
|
|
6750
|
-
|
|
6751
|
-
|
|
6752
|
-
|
|
7457
|
+
}
|
|
7458
|
+
/**
|
|
7459
|
+
* The section this command exists for: what would actually move this bill.
|
|
7460
|
+
*
|
|
7461
|
+
* Shortening the prompt is the smallest line item there is. Which model a call goes
|
|
7462
|
+
* to, the Batch API, caching and re-sent conversation history are the levers with
|
|
7463
|
+
* real money on them, and all four are priced here from what the provider charged
|
|
7464
|
+
* rather than from what a file would cost.
|
|
7465
|
+
*/
|
|
7466
|
+
function reportLevers(report, pricing, t) {
|
|
7467
|
+
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
7468
|
+
const pct = (share) => `${(share * 100).toFixed(1)}%`;
|
|
7469
|
+
/**
|
|
7470
|
+
* The section this command is for, and the answer to the fairest complaint the
|
|
7471
|
+
* product has had: on a bill of twenty thousand, the rules recover two hundred.
|
|
7472
|
+
*
|
|
7473
|
+
* That figure is right — measured, three tokens out of three hundred and six on
|
|
7474
|
+
* an ordinary support prompt. The conclusion is not that the tool is worthless
|
|
7475
|
+
* but that it had been looking at the smallest line item. Which model a call
|
|
7476
|
+
* goes to moves 60% to 80%. The Batch API moves 50% flat. Both are priced here
|
|
7477
|
+
* from the reader's own tokens, at published rates, with no modelling in
|
|
7478
|
+
* between — and printed above the breakdowns, because a lever nobody scrolls to
|
|
7479
|
+
* is a lever nobody pulls.
|
|
7480
|
+
*
|
|
7481
|
+
* The ceiling on prompt shortening prints underneath them on purpose. A 1% win
|
|
7482
|
+
* reported without saying 1% of what is not information, and this repository
|
|
7483
|
+
* would rather say the uncomfortable number itself than let somebody else
|
|
7484
|
+
* discover it.
|
|
7485
|
+
*/
|
|
7486
|
+
const levers = billLevers(report, { catalogue: pricing });
|
|
6753
7487
|
console.log();
|
|
6754
|
-
console.log(sectionHeading(t.profile.
|
|
6755
|
-
console.log(` ${t.profile.spent(t.profile.calls(report.total.calls), formatUsd(report.total.totalUsd))}`);
|
|
7488
|
+
console.log(sectionHeading(t.profile.leversHeading()));
|
|
6756
7489
|
/**
|
|
6757
|
-
*
|
|
6758
|
-
*
|
|
6759
|
-
*
|
|
6760
|
-
*
|
|
6761
|
-
*
|
|
6762
|
-
*
|
|
7490
|
+
* Every lever below describes a mixture when nothing carries a label.
|
|
7491
|
+
*
|
|
7492
|
+
* A 2,000-call classifier and a 400-call RAG pipeline merge into one slice, and
|
|
7493
|
+
* the section then offers a single route for two workloads that need different
|
|
7494
|
+
* answers — and `trazum route` would measure one prompt against a figure
|
|
7495
|
+
* covering both. The session case already tells the reader to add the field;
|
|
7496
|
+
* this one named the row `unlabelled` and said nothing, as though that were a
|
|
7497
|
+
* workload.
|
|
6763
7498
|
*/
|
|
6764
|
-
|
|
6765
|
-
|
|
6766
|
-
|
|
6767
|
-
? ` ${t.profile.spanPartial(n(report.span.calls), n(totalParsed))}`
|
|
6768
|
-
: '';
|
|
6769
|
-
console.log(` ${c.dim(wrap(`${t.profile.spanLine(dayOf(report.span.fromMs), dayOf(report.span.toMs), spanDays(report.span.fromMs, report.span.toMs))}${partial}`, 74, ' '))}`);
|
|
7499
|
+
const unlabelledOnly = report.byLabel.length === 1 && report.byLabel[0].label === UNLABELLED;
|
|
7500
|
+
if (unlabelledOnly && levers.slices.length > 0) {
|
|
7501
|
+
console.log(` ${c.yellow('!')} ${c.dim(wrap(t.profile.leversUnlabelled(), 74, ' '))}`);
|
|
6770
7502
|
}
|
|
6771
|
-
|
|
6772
|
-
|
|
6773
|
-
|
|
6774
|
-
|
|
7503
|
+
if (levers.slices.length === 0) {
|
|
7504
|
+
console.log(` ${c.dim(wrap(t.profile.leversNone(), 74, ' '))}`);
|
|
7505
|
+
}
|
|
7506
|
+
else {
|
|
7507
|
+
for (const slice of levers.slices.slice(0, 5)) {
|
|
7508
|
+
const label = slice.label === UNLABELLED ? t.profile.unlabelled() : slice.label;
|
|
7509
|
+
console.log();
|
|
7510
|
+
/**
|
|
7511
|
+
* The headline is the **combined** figure, and the options underneath are
|
|
7512
|
+
* the ways to reach it — not rows to add up. Batching a routed call
|
|
7513
|
+
* discounts the cheaper model's price, so listing them separately printed
|
|
7514
|
+
* $12.60 and $10.50 against a slice that had spent $21.00: a saving larger
|
|
7515
|
+
* than the bill it came from, in the flattering direction.
|
|
7516
|
+
*/
|
|
7517
|
+
console.log(` ${c.green('→')} ${c.bold(wrap(t.profile.leverSlice(label, slice.modelName, formatUsd(slice.combinedUsd), pct(slice.shareOfBill)), 74, ' '))}`);
|
|
7518
|
+
console.log(` ${c.dim(t.profile.leverCalls(t.profile.calls(slice.calls), formatUsd(slice.spentUsd)))}`);
|
|
7519
|
+
if (slice.route) {
|
|
7520
|
+
console.log(` ${c.dim('·')} ${c.dim(wrap(t.profile.leverRoute(slice.route.candidate.displayName, formatUsd(slice.route.savingUsd)), 74, ' '))}`);
|
|
7521
|
+
}
|
|
7522
|
+
if (slice.batch) {
|
|
7523
|
+
console.log(` ${c.dim('·')} ${c.dim(wrap(t.profile.leverBatch(formatUsd(slice.batch.savingUsd)), 74, ' '))}`);
|
|
7524
|
+
}
|
|
7525
|
+
// The arithmetic is exact and the quality question is untouched by it.
|
|
7526
|
+
// Naming the command is the difference between a saving and a gamble.
|
|
7527
|
+
if (slice.route) {
|
|
7528
|
+
console.log(` ${c.dim(wrap(t.profile.leverRouteVerify(slice.route.candidate.id), 74, ' '))}`);
|
|
7529
|
+
}
|
|
7530
|
+
}
|
|
7531
|
+
}
|
|
7532
|
+
console.log();
|
|
7533
|
+
console.log(` ${c.dim(wrap(t.profile.leverPromptCeiling(formatUsd(levers.promptCeilingUsd), pct(levers.promptCeilingShare)), 74, ' '))}`);
|
|
7534
|
+
}
|
|
7535
|
+
/**
|
|
7536
|
+
* What the labels, the budgets and the cache TTLs say about this bill.
|
|
7537
|
+
*
|
|
7538
|
+
* Why a label's caching is failing, read from the prompt file itself: the log
|
|
7539
|
+
* carries counts, so `profile` can say *that* caching loses money on a label and
|
|
7540
|
+
* nothing more, and `labels` in the config maps a label to the file it sends.
|
|
7541
|
+
* Every sentence carries "as it is today", because the file is whatever the
|
|
7542
|
+
* repository holds now and may not be what produced the log.
|
|
7543
|
+
*
|
|
7544
|
+
* Then the budgets, the TTL fits, the single-turn writes and the shape of the
|
|
7545
|
+
* sessions: two hundred lines of one topic, what the configuration and the
|
|
7546
|
+
* conversation did to the money.
|
|
7547
|
+
*/
|
|
7548
|
+
async function reportLabelsBudgetsAndCache(report, config, pricing, t) {
|
|
7549
|
+
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
7550
|
+
const pct = (share) => `${(share * 100).toFixed(1)}%`;
|
|
7551
|
+
/**
|
|
7552
|
+
* Why, read from the prompt file itself — the loop `profile` could not close.
|
|
7553
|
+
*
|
|
7554
|
+
* The log carries counts, so this command can say *that* caching loses money
|
|
7555
|
+
* on a label and nothing more. `labels` in the config maps a label to the
|
|
7556
|
+
* prompt file it sends, and for each mapped label whose cache is failing —
|
|
7557
|
+
* losing money, or never attempted while money sat in cacheable input — the
|
|
7558
|
+
* file is read and the reason named: a prefix under the model's minimum,
|
|
7559
|
+
* stable tokens stranded behind the first placeholder, or a healthy file
|
|
7560
|
+
* whose problem is byte-identity between calls.
|
|
7561
|
+
*
|
|
7562
|
+
* Every sentence carries "as it is today": the file is whatever the
|
|
7563
|
+
* repository holds now, which may not be what produced the log, and a fresh
|
|
7564
|
+
* file presented as the history's explanation would be a figure attributed to
|
|
7565
|
+
* something it does not describe.
|
|
7566
|
+
*/
|
|
7567
|
+
const labelMap = config.labels ?? {};
|
|
7568
|
+
for (const { label, model: modelId, breakdown } of report.byLabelAndModel) {
|
|
7569
|
+
const file = labelMap[label];
|
|
7570
|
+
if (file === undefined)
|
|
7571
|
+
continue;
|
|
7572
|
+
const labelCache = cacheEconomics(breakdown);
|
|
7573
|
+
const failing = labelCache.verdict === 'lost-money' ||
|
|
7574
|
+
(labelCache.verdict === 'not-attempted' && breakdown.inputUsd > 0);
|
|
7575
|
+
if (!failing)
|
|
7576
|
+
continue;
|
|
7577
|
+
let text;
|
|
7578
|
+
try {
|
|
7579
|
+
text = await readFile(file, 'utf8');
|
|
7580
|
+
}
|
|
7581
|
+
catch {
|
|
7582
|
+
console.log(` ${c.dim(wrap(t.profile.labelFileMissing(label, file), 74, ' '))}`);
|
|
7583
|
+
continue;
|
|
7584
|
+
}
|
|
7585
|
+
const model = pricing.byId.get(modelId);
|
|
7586
|
+
if (!model)
|
|
7587
|
+
continue;
|
|
7588
|
+
const analysis = analyzeCachePrefix(text, estimateTokens);
|
|
7589
|
+
const minimum = model.cacheMinTokens;
|
|
7590
|
+
console.log();
|
|
7591
|
+
if (minimum !== null && analysis.stablePrefixTokens < minimum) {
|
|
7592
|
+
console.log(` ${c.dim(wrap(t.profile.labelPrefixBelowMinimum(file, n(analysis.stablePrefixTokens), n(minimum), model.displayName), 74, ' '))}`);
|
|
7593
|
+
}
|
|
7594
|
+
else if (analysis.staticTokensAfter >= 200) {
|
|
7595
|
+
console.log(` ${c.dim(wrap(t.profile.labelPrefixMovable(file, n(analysis.staticTokensAfter), n(analysis.stablePrefixTokens)), 74, ' '))}`);
|
|
7596
|
+
}
|
|
7597
|
+
else {
|
|
7598
|
+
console.log(` ${c.dim(wrap(t.profile.labelPrefixHealthy(file, n(analysis.stablePrefixTokens), n(minimum ?? 0)), 74, ' '))}`);
|
|
7599
|
+
}
|
|
6775
7600
|
}
|
|
6776
7601
|
/**
|
|
6777
|
-
*
|
|
7602
|
+
* The token budget against what actually goes up the wire.
|
|
6778
7603
|
*
|
|
6779
|
-
*
|
|
6780
|
-
*
|
|
6781
|
-
*
|
|
6782
|
-
*
|
|
6783
|
-
*
|
|
6784
|
-
*
|
|
7604
|
+
* `budgets` gates a prompt *file*; the log records what the *call* carried —
|
|
7605
|
+
* system prompt, retrieved context, conversation history, tool results. The
|
|
7606
|
+
* two are related only through `labels`, and when the gap is large the gate
|
|
7607
|
+
* is real but tiny: a 2,000-token budget on a workload sending 47,000
|
|
7608
|
+
* tokens a call governs four per cent of what is sent, and nobody looking
|
|
7609
|
+
* at a green build would know it.
|
|
7610
|
+
*
|
|
7611
|
+
* Only stated when both ends are known — a label mapped to a file, and a
|
|
7612
|
+
* budget covering that file — and the share is named as approximate,
|
|
7613
|
+
* because the budget counts the file's tokens with the estimator while the
|
|
7614
|
+
* log counts what the provider billed. It says which part of the bill the
|
|
7615
|
+
* gate can see, and never that the budget is wrong.
|
|
6785
7616
|
*/
|
|
6786
|
-
|
|
6787
|
-
|
|
7617
|
+
const budgetPatterns = Object.keys(config.budgets ?? {});
|
|
7618
|
+
if (budgetPatterns.length > 0) {
|
|
7619
|
+
for (const row of report.byLabel) {
|
|
7620
|
+
const file = labelMap[row.label];
|
|
7621
|
+
if (file === undefined || row.breakdown.calls === 0)
|
|
7622
|
+
continue;
|
|
7623
|
+
const pattern = mostSpecificMatch(budgetPatterns, file);
|
|
7624
|
+
if (pattern === null)
|
|
7625
|
+
continue;
|
|
7626
|
+
const budget = config.budgets[pattern];
|
|
7627
|
+
if (budget <= 0)
|
|
7628
|
+
continue;
|
|
7629
|
+
/**
|
|
7630
|
+
* Input tokens per call over this label — every class that is billed
|
|
7631
|
+
* as input, because a cached token was still sent and still counted
|
|
7632
|
+
* against the model's window.
|
|
7633
|
+
*/
|
|
7634
|
+
const perCall = (row.breakdown.inputTokens + row.breakdown.cacheReadTokens + row.breakdown.cacheWriteTokens) /
|
|
7635
|
+
row.breakdown.calls;
|
|
7636
|
+
if (perCall <= 0)
|
|
7637
|
+
continue;
|
|
7638
|
+
const share = budget / perCall;
|
|
7639
|
+
// Only when the gap is wide enough to change what somebody believes.
|
|
7640
|
+
// A budget covering most of the call is doing its job quietly.
|
|
7641
|
+
if (share >= 0.5)
|
|
7642
|
+
continue;
|
|
7643
|
+
console.log();
|
|
7644
|
+
console.log(` ${c.yellow('!')} ${c.dim(wrap(t.profile.budgetVsWire(row.label === UNLABELLED ? t.profile.unlabelled() : row.label, file, n(budget), n(Math.round(perCall)), pct(share)), 74, ' '))}`);
|
|
7645
|
+
}
|
|
6788
7646
|
}
|
|
6789
7647
|
/**
|
|
6790
|
-
*
|
|
6791
|
-
*
|
|
6792
|
-
*
|
|
6793
|
-
*
|
|
6794
|
-
*
|
|
7648
|
+
* Whether the TTL fits how fast the turns arrive — the mechanism behind the
|
|
7649
|
+
* verdict above, readable only when the log carries a clock and a session.
|
|
7650
|
+
*
|
|
7651
|
+
* Rendered as four verdicts plus "could not be measured", the same
|
|
7652
|
+
* three-state discipline truncation uses: a workload with cache writes and no
|
|
7653
|
+
* clock has not been cleared, and silence here would read as fine.
|
|
6795
7654
|
*/
|
|
6796
|
-
|
|
6797
|
-
|
|
6798
|
-
|
|
6799
|
-
|
|
7655
|
+
const TTL_SHOWN = 3;
|
|
7656
|
+
for (const fit of report.cacheTtlFit.slice(0, TTL_SHOWN)) {
|
|
7657
|
+
const name = fit.label === UNLABELLED ? t.profile.unlabelled() : fit.label;
|
|
7658
|
+
const gap = formatGap(fit.medianGapMs);
|
|
7659
|
+
if (fit.verdict === 'expires-before-reuse') {
|
|
7660
|
+
const line = fit.medianGapMs > TTL_1H_MS
|
|
7661
|
+
? t.profile.ttlFitExpiresBoth(name, fit.modelName, gap)
|
|
7662
|
+
: t.profile.ttlFitExpires(name, fit.modelName, gap);
|
|
7663
|
+
console.log(` ${c.yellow('!')} ${c.bold(wrap(line, 74, ' '))}`);
|
|
6800
7664
|
}
|
|
6801
|
-
if (
|
|
6802
|
-
console.log(` ${c.yellow(wrap(t.profile.
|
|
7665
|
+
else if (fit.verdict === 'overlong-ttl') {
|
|
7666
|
+
console.log(` ${c.yellow('!')} ${c.bold(wrap(t.profile.ttlFitOverlong(name, fit.modelName, gap, formatUsd(fit.overpayUsd)), 74, ' '))}`);
|
|
7667
|
+
}
|
|
7668
|
+
else if (fit.verdict === 'unsettled') {
|
|
7669
|
+
console.log(` ${c.dim(wrap(t.profile.ttlFitUnsettledGap(name, fit.modelName, gap), 74, ' '))}`);
|
|
7670
|
+
}
|
|
7671
|
+
else {
|
|
7672
|
+
console.log(` ${c.dim(wrap(t.profile.ttlFitFits(name, fit.modelName, gap), 74, ' '))}`);
|
|
6803
7673
|
}
|
|
6804
7674
|
}
|
|
6805
|
-
|
|
6806
|
-
|
|
6807
|
-
// as a row somebody forgot, and "you are not caching at all" is a finding.
|
|
6808
|
-
for (const [name, usd, share, tokens] of parts) {
|
|
6809
|
-
console.log(` ${bar(share)} ${t.profile.part(name, formatUsd(usd), pct(share), n(tokens))}`);
|
|
7675
|
+
if (report.total.cacheWriteTokens > 0 && report.cacheTtlFit.length === 0) {
|
|
7676
|
+
console.log(` ${c.dim(wrap(t.profile.ttlFitUnmeasured(), 74, ' '))}`);
|
|
6810
7677
|
}
|
|
6811
7678
|
/**
|
|
6812
|
-
*
|
|
7679
|
+
* Cache writes by conversations that never came back.
|
|
6813
7680
|
*
|
|
6814
|
-
*
|
|
6815
|
-
*
|
|
6816
|
-
*
|
|
7681
|
+
* Two sentences for the same tokens, and which one prints is decided by the
|
|
7682
|
+
* slice's own reads: with zero cache reads anywhere in the slice, nothing
|
|
7683
|
+
* read those writes — within the session, across sessions, at all — and the
|
|
7684
|
+
* ceiling collapses into a fact said loudly. With reads present, another
|
|
7685
|
+
* conversation sharing the prefix may have read them, the log cannot see
|
|
7686
|
+
* whose write a read hit, and the figure prints as the ceiling it is.
|
|
6817
7687
|
*/
|
|
6818
|
-
const
|
|
6819
|
-
|
|
6820
|
-
|
|
6821
|
-
|
|
6822
|
-
|
|
6823
|
-
|
|
6824
|
-
|
|
6825
|
-
|
|
7688
|
+
const LEDGER_SHOWN = 3;
|
|
7689
|
+
if (report.singleTurnCacheWrites.length > 0) {
|
|
7690
|
+
const readsBySlice = new Map(report.byLabelAndModel.map((r) => [`${r.label}\n${r.model}`, r.breakdown.cacheReadTokens]));
|
|
7691
|
+
for (const row of report.singleTurnCacheWrites.slice(0, LEDGER_SHOWN)) {
|
|
7692
|
+
const name = row.label === UNLABELLED ? t.profile.unlabelled() : row.label;
|
|
7693
|
+
const reads = readsBySlice.get(`${row.label}\n${row.model}`) ?? 0;
|
|
7694
|
+
if (reads === 0) {
|
|
7695
|
+
console.log(` ${c.yellow('!')} ${c.bold(wrap(t.profile.singleTurnConfirmed(name, row.modelName, n(row.singleTurnSessions), n(row.sessions), formatUsd(row.singleTurnWriteUsd)), 74, ' '))}`);
|
|
7696
|
+
}
|
|
7697
|
+
else {
|
|
7698
|
+
console.log(` ${c.dim(wrap(t.profile.singleTurnCeiling(name, row.modelName, n(row.singleTurnSessions), n(row.sessions), formatUsd(row.singleTurnWriteUsd)), 74, ' '))}`);
|
|
7699
|
+
}
|
|
7700
|
+
}
|
|
6826
7701
|
}
|
|
6827
7702
|
/**
|
|
6828
|
-
*
|
|
6829
|
-
*
|
|
6830
|
-
*
|
|
6831
|
-
*
|
|
6832
|
-
* for opposite responses. Rendered against the **median** day — a mean would
|
|
6833
|
-
* let the spike inflate its own yardstick — and loud only when it clears twice
|
|
6834
|
-
* the median, a threshold stated in the sentence rather than hidden in code.
|
|
7703
|
+
* What one conversation costs — the question a total cannot answer, and the
|
|
7704
|
+
* one a per-seat price or a quota is set from. Median against p95, never a
|
|
7705
|
+
* mean: one runaway agent loop would drag a mean up and hide the ordinary
|
|
7706
|
+
* case, which is the figure somebody is actually pricing.
|
|
6835
7707
|
*/
|
|
6836
|
-
|
|
6837
|
-
const
|
|
6838
|
-
|
|
6839
|
-
|
|
6840
|
-
|
|
6841
|
-
|
|
6842
|
-
|
|
6843
|
-
|
|
6844
|
-
|
|
6845
|
-
|
|
6846
|
-
|
|
6847
|
-
console.log(` ${
|
|
7708
|
+
for (const shape of report.sessionCosts.slice(0, 3)) {
|
|
7709
|
+
const name = shape.label === UNLABELLED ? t.profile.unlabelled() : shape.label;
|
|
7710
|
+
console.log();
|
|
7711
|
+
console.log(` ${c.dim(wrap(t.profile.sessionCost(name, shape.modelName, n(shape.sessions), formatUsd(shape.medianUsd), n(shape.medianTurns), formatUsd(shape.p95Usd), formatUsd(shape.maxUsd)), 74, ' '))}`);
|
|
7712
|
+
/**
|
|
7713
|
+
* The tail, when there is one. A p95 far above the median is a shape a
|
|
7714
|
+
* quota can fix; a p95 beside it is a workload that is simply expensive,
|
|
7715
|
+
* and saying "hunt the tail" there would send somebody after nothing.
|
|
7716
|
+
* The threshold is in the sentence rather than hidden here.
|
|
7717
|
+
*/
|
|
7718
|
+
if (shape.medianUsd > 0 && shape.p95Usd > 10 * shape.medianUsd) {
|
|
7719
|
+
console.log(` ${c.yellow('!')} ${c.dim(wrap(t.profile.sessionCostTail((shape.p95Usd / shape.medianUsd).toFixed(0)), 74, ' '))}`);
|
|
6848
7720
|
}
|
|
6849
7721
|
}
|
|
6850
7722
|
/**
|
|
6851
|
-
* The
|
|
6852
|
-
*
|
|
6853
|
-
*
|
|
6854
|
-
*
|
|
6855
|
-
*
|
|
6856
|
-
* measure is exact and needs no threshold to state: the **fewest hours that
|
|
6857
|
-
* hold 80% of the spend**. Two or three means concentrated; sixteen means
|
|
6858
|
-
* flat.
|
|
6859
|
-
*
|
|
6860
|
-
* It says what the shape is and stops. Whether a workload can wait is a
|
|
6861
|
-
* product decision Trazum cannot make from counts, so the sentence names
|
|
6862
|
-
* the lever and never claims the saving — the batch figure the levers
|
|
6863
|
-
* section already prints is the one with money attached.
|
|
7723
|
+
* The figure that survives a small log. `sessionCosts` refuses slices too
|
|
7724
|
+
* thin for a percentile, and rightly — but a log of four conversations
|
|
7725
|
+
* still has a most expensive one, and that maximum is a fact at any count.
|
|
7726
|
+
* It is also exactly the number `--max-session-usd` judges, so the report
|
|
7727
|
+
* states it rather than going silent where the gate would speak.
|
|
6864
7728
|
*/
|
|
6865
|
-
if (report.
|
|
6866
|
-
const sorted = [...report.spendByHour].sort((a, b) => b.usd - a.usd);
|
|
6867
|
-
let covered = 0;
|
|
6868
|
-
let hoursForMost = 0;
|
|
6869
|
-
for (const hour of sorted) {
|
|
6870
|
-
covered += hour.usd;
|
|
6871
|
-
hoursForMost += 1;
|
|
6872
|
-
if (covered >= 0.8 * report.total.totalUsd)
|
|
6873
|
-
break;
|
|
6874
|
-
}
|
|
6875
|
-
const busiest = sorted
|
|
6876
|
-
.slice(0, hoursForMost)
|
|
6877
|
-
.map((hour) => hour.hour)
|
|
6878
|
-
.sort((a, b) => a - b)
|
|
6879
|
-
.map((hour) => `${String(hour).padStart(2, '0')}:00`)
|
|
6880
|
-
.join(', ');
|
|
7729
|
+
if (report.sessionCosts.length === 0 && report.sessionSpend !== null) {
|
|
6881
7730
|
console.log();
|
|
6882
|
-
console.log(` ${c.dim(wrap(
|
|
7731
|
+
console.log(` ${c.dim(wrap(t.profile.sessionSpendOnly(n(report.sessionSpend.sessions), formatUsd(report.sessionSpend.maxUsd)), 74, ' '))}`);
|
|
7732
|
+
}
|
|
7733
|
+
/**
|
|
7734
|
+
* A total that assumed a cache-write rate is a floor, and says so.
|
|
7735
|
+
*
|
|
7736
|
+
* Anthropic's 1-hour entry costs 2x input against the 5-minute entry's 1.25x. A
|
|
7737
|
+
* log carrying only the flat `cache_creation_input_tokens` cannot say which, so
|
|
7738
|
+
* the cheaper one is used — and the flattering direction is exactly the one this
|
|
7739
|
+
* tool refuses to take quietly.
|
|
7740
|
+
*/
|
|
7741
|
+
if (report.total.assumedWriteTtlCalls > 0) {
|
|
7742
|
+
console.log(` ${c.dim(wrap(t.profile.assumedWriteTtl(report.total.assumedWriteTtlCalls), 74, ' '))}`);
|
|
6883
7743
|
}
|
|
7744
|
+
}
|
|
7745
|
+
/**
|
|
7746
|
+
* What caching did to this bill, and what it is about to do.
|
|
7747
|
+
*
|
|
7748
|
+
* The hit rate, the money the cache made or lost, the labels bleeding on it, and
|
|
7749
|
+
* the TTL question the hit rate cannot answer. One topic, and it needs the report
|
|
7750
|
+
* and the messages: it was a hundred and thirty lines inside a function with
|
|
7751
|
+
* twenty-three values in scope, and the only way to learn it used two of them was
|
|
7752
|
+
* to read it.
|
|
7753
|
+
*/
|
|
7754
|
+
function reportCacheVerdict(report, t) {
|
|
7755
|
+
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
7756
|
+
const pct = (share) => `${(share * 100).toFixed(1)}%`;
|
|
6884
7757
|
/**
|
|
6885
7758
|
* The hit rate, and then the question the hit rate does not answer.
|
|
6886
7759
|
*
|
|
@@ -6999,682 +7872,641 @@ async function commandProfile(args, config, configDir, pricing, t) {
|
|
|
6999
7872
|
if (maybeLostLabels.length > 0) {
|
|
7000
7873
|
console.log(` ${c.dim(wrap(t.profile.cacheTtlUnsettledLabels(listNames(maybeLostLabels)), 74, ' '))}`);
|
|
7001
7874
|
}
|
|
7002
|
-
|
|
7003
|
-
|
|
7004
|
-
|
|
7005
|
-
|
|
7006
|
-
|
|
7007
|
-
|
|
7008
|
-
|
|
7009
|
-
|
|
7010
|
-
|
|
7011
|
-
|
|
7012
|
-
|
|
7013
|
-
|
|
7014
|
-
|
|
7015
|
-
|
|
7016
|
-
|
|
7875
|
+
}
|
|
7876
|
+
/**
|
|
7877
|
+
* `trazum plan <log>` — not a list of findings, a ranked plan of what to do.
|
|
7878
|
+
*
|
|
7879
|
+
* The composition (route and batch on one slice never summed) happens in
|
|
7880
|
+
* core's `buildPlan`; this command owns the I/O and the rendering. The plan
|
|
7881
|
+
* saves as a dated JSON file on request, which is what makes verifying it
|
|
7882
|
+
* against a later log possible at all — a prediction nobody wrote down is a
|
|
7883
|
+
* prediction nobody can be held to.
|
|
7884
|
+
*/
|
|
7885
|
+
async function commandPlan(args, pricing, t) {
|
|
7886
|
+
const path = args.positional[0];
|
|
7887
|
+
if (path === undefined)
|
|
7888
|
+
throw new Error(t.plan.noTarget());
|
|
7889
|
+
const GZ = LOG_EXTENSIONS.map((ext) => `${ext}.gz`);
|
|
7890
|
+
const READABLE = [...LOG_EXTENSIONS, ...GZ];
|
|
7891
|
+
const target = await stat(path).catch(() => null);
|
|
7892
|
+
let files = [path];
|
|
7893
|
+
if (target?.isDirectory()) {
|
|
7894
|
+
const entries = await readdir(path, { withFileTypes: true });
|
|
7895
|
+
files = entries
|
|
7896
|
+
.filter((entry) => entry.isFile() && READABLE.some((ext) => entry.name.endsWith(ext)))
|
|
7897
|
+
.map((entry) => join(path, entry.name))
|
|
7898
|
+
.sort((a, b) => a.localeCompare(b));
|
|
7899
|
+
if (files.length === 0)
|
|
7900
|
+
throw new Error(t.profile.noLogsInDirectory(path, READABLE.join(', ')));
|
|
7901
|
+
}
|
|
7902
|
+
const texts = await Promise.all(files.map((file) => readUsageLog(file, t)));
|
|
7903
|
+
const raw = texts.map((text) => (text.endsWith('\n') ? text : `${text}\n`)).join('');
|
|
7904
|
+
const report = profileUsage(raw, { catalogue: pricing });
|
|
7905
|
+
if (report.total.calls === 0)
|
|
7906
|
+
throw new Error(t.plan.nothingPriced());
|
|
7907
|
+
const levers = billLevers(report, { catalogue: pricing });
|
|
7908
|
+
const plan = buildPlan(report, levers, pricing.lastReviewed);
|
|
7909
|
+
const minUsd = typeof args.flags.get('min-usd') === 'string' ? numberFlag(args, 'min-usd', 0, t) : 0;
|
|
7910
|
+
const actions = plan.actions.filter((a) => (a.savingUsd ?? a.stakeUsd ?? 0) >= minUsd);
|
|
7911
|
+
const filtered = plan.actions.length - actions.length;
|
|
7912
|
+
const droppedUsd = plan.actions
|
|
7913
|
+
.filter((a) => (a.savingUsd ?? a.stakeUsd ?? 0) < minUsd)
|
|
7914
|
+
.reduce((sum, a) => sum + (a.savingUsd ?? a.stakeUsd ?? 0), 0);
|
|
7915
|
+
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
7916
|
+
/**
|
|
7917
|
+
* The document's totals cover the actions the document holds — a filtered
|
|
7918
|
+
* plan whose totals still counted the filtered actions would be a file
|
|
7919
|
+
* that contradicts itself, and 1.39's verify would hold it to money it
|
|
7920
|
+
* cannot see. What --min-usd dropped is stated with its worth, never
|
|
7921
|
+
* silently.
|
|
7017
7922
|
*/
|
|
7018
|
-
const
|
|
7019
|
-
|
|
7020
|
-
|
|
7021
|
-
|
|
7022
|
-
|
|
7023
|
-
|
|
7024
|
-
|
|
7025
|
-
|
|
7026
|
-
|
|
7027
|
-
|
|
7028
|
-
|
|
7029
|
-
|
|
7030
|
-
|
|
7923
|
+
const stamped = {
|
|
7924
|
+
...plan,
|
|
7925
|
+
actions,
|
|
7926
|
+
projectedSavingUsd: actions.reduce((sum, a) => sum + (a.savingUsd ?? 0), 0),
|
|
7927
|
+
measuredStakeUsd: actions.reduce((sum, a) => sum + (a.stakeUsd ?? 0), 0),
|
|
7928
|
+
createdAt: new Date().toISOString(),
|
|
7929
|
+
};
|
|
7930
|
+
const outPath = stringFlag(args, 'out');
|
|
7931
|
+
if (outPath !== undefined) {
|
|
7932
|
+
await writeFile(outPath, `${JSON.stringify(stamped, null, 2)}\n`);
|
|
7933
|
+
}
|
|
7934
|
+
await writeMarkdown(args, () => {
|
|
7935
|
+
const lines = [];
|
|
7936
|
+
lines.push(`## ${t.plan.heading(n(actions.length), formatUsd(plan.totalUsd))}`);
|
|
7937
|
+
lines.push('');
|
|
7938
|
+
lines.push(t.plan.totals(formatUsd(stamped.projectedSavingUsd), formatUsd(stamped.measuredStakeUsd)));
|
|
7939
|
+
if (plan.span === null) {
|
|
7940
|
+
lines.push('');
|
|
7941
|
+
lines.push(`_${t.plan.noClock()}_`);
|
|
7031
7942
|
}
|
|
7032
|
-
|
|
7033
|
-
|
|
7034
|
-
|
|
7943
|
+
for (const action of actions) {
|
|
7944
|
+
const name = action.label === UNLABELLED ? t.profile.unlabelled() : action.label;
|
|
7945
|
+
const money = action.savingUsd !== null
|
|
7946
|
+
? t.plan.projected(formatUsd(action.savingUsd))
|
|
7947
|
+
: t.plan.staked(formatUsd(action.stakeUsd ?? 0));
|
|
7948
|
+
lines.push('');
|
|
7949
|
+
lines.push(`### ${t.plan.action(action.kind, name, action.model)} — ${money}`);
|
|
7950
|
+
if (action.detail.routeTo !== undefined)
|
|
7951
|
+
lines.push(`- ${t.plan.routeTo(action.detail.routeTo.displayName)}`);
|
|
7952
|
+
for (const assumption of action.assumes)
|
|
7953
|
+
lines.push(`- ${t.plan.assume(assumption)}`);
|
|
7954
|
+
if (action.check !== null)
|
|
7955
|
+
lines.push(`- ${t.plan.check(action.check)}`);
|
|
7035
7956
|
}
|
|
7036
|
-
|
|
7037
|
-
|
|
7038
|
-
|
|
7039
|
-
|
|
7040
|
-
|
|
7957
|
+
if (filtered > 0) {
|
|
7958
|
+
lines.push('');
|
|
7959
|
+
lines.push(`_${t.plan.filtered(n(filtered), formatUsd(minUsd), formatUsd(droppedUsd))}_`);
|
|
7960
|
+
}
|
|
7961
|
+
lines.push('');
|
|
7962
|
+
lines.push(`_${t.plan.footer()}_`);
|
|
7963
|
+
return lines.join('\n');
|
|
7964
|
+
});
|
|
7965
|
+
if (boolFlag(args, 'json')) {
|
|
7966
|
+
console.log(JSON.stringify(stamped, null, 2));
|
|
7967
|
+
return;
|
|
7968
|
+
}
|
|
7969
|
+
console.log(sectionHeading(t.plan.heading(n(actions.length), formatUsd(plan.totalUsd))));
|
|
7970
|
+
console.log(` ${wrap(t.plan.totals(formatUsd(stamped.projectedSavingUsd), formatUsd(stamped.measuredStakeUsd)), 74, ' ')}`);
|
|
7971
|
+
if (plan.span === null) {
|
|
7972
|
+
console.log(` ${c.dim(wrap(t.plan.noClock(), 74, ' '))}`);
|
|
7973
|
+
}
|
|
7974
|
+
for (const action of actions) {
|
|
7975
|
+
const name = action.label === UNLABELLED ? t.profile.unlabelled() : action.label;
|
|
7976
|
+
const money = action.savingUsd !== null
|
|
7977
|
+
? t.plan.projected(formatUsd(action.savingUsd))
|
|
7978
|
+
: t.plan.staked(formatUsd(action.stakeUsd ?? 0));
|
|
7041
7979
|
console.log();
|
|
7042
|
-
|
|
7043
|
-
|
|
7980
|
+
console.log(` ${c.green('→')} ${c.bold(t.plan.action(action.kind, name, action.model))} ${money}`);
|
|
7981
|
+
if (action.detail.routeTo !== undefined) {
|
|
7982
|
+
console.log(` ${c.dim(t.plan.routeTo(action.detail.routeTo.displayName))}`);
|
|
7044
7983
|
}
|
|
7045
|
-
|
|
7046
|
-
console.log(`
|
|
7984
|
+
for (const assumption of action.assumes) {
|
|
7985
|
+
console.log(` ${c.yellow('?')} ${c.dim(wrap(t.plan.assume(assumption), 72, ' '))}`);
|
|
7047
7986
|
}
|
|
7048
|
-
|
|
7049
|
-
console.log(`
|
|
7987
|
+
if (action.check !== null) {
|
|
7988
|
+
console.log(` ${c.dim(wrap(t.plan.check(action.check), 72, ' '))}`);
|
|
7050
7989
|
}
|
|
7051
7990
|
}
|
|
7991
|
+
if (filtered > 0) {
|
|
7992
|
+
console.log();
|
|
7993
|
+
console.log(` ${c.dim(wrap(t.plan.filtered(n(filtered), formatUsd(minUsd), formatUsd(droppedUsd)), 74, ' '))}`);
|
|
7994
|
+
}
|
|
7995
|
+
console.log();
|
|
7996
|
+
console.log(` ${c.dim(wrap(t.plan.footer(), 74, ' '))}`);
|
|
7997
|
+
if (outPath !== undefined)
|
|
7998
|
+
console.log(c.dim(wrap(t.plan.wrote(outPath), 74, '')));
|
|
7999
|
+
}
|
|
8000
|
+
async function commandProfile(args, config, configDir, pricing, t) {
|
|
8001
|
+
const path = args.positional[0];
|
|
8002
|
+
if (path === undefined) {
|
|
8003
|
+
console.log();
|
|
8004
|
+
console.log(c.dim(wrap(t.profile.noTarget(), 74, ' ')));
|
|
8005
|
+
console.log();
|
|
8006
|
+
return;
|
|
8007
|
+
}
|
|
7052
8008
|
/**
|
|
7053
|
-
*
|
|
8009
|
+
* A log, or a directory of them.
|
|
7054
8010
|
*
|
|
7055
|
-
*
|
|
7056
|
-
*
|
|
7057
|
-
*
|
|
7058
|
-
* is
|
|
7059
|
-
* tokens a call governs four per cent of what is sent, and nobody looking
|
|
7060
|
-
* at a green build would know it.
|
|
8011
|
+
* Usage logs rotate: `logs/2026-08-01.jsonl`, `logs/2026-08-02.jsonl`, one
|
|
8012
|
+
* per day for a month. Making somebody `cat` them together before a profile
|
|
8013
|
+
* will read them is a setup cost that gets a tool skipped, and doing it for
|
|
8014
|
+
* them is a directory listing.
|
|
7061
8015
|
*
|
|
7062
|
-
*
|
|
7063
|
-
*
|
|
7064
|
-
*
|
|
7065
|
-
*
|
|
7066
|
-
*
|
|
8016
|
+
* Files are read in name order — which for dated names is time order — and
|
|
8017
|
+
* how many were read is stated, because a report over "the logs" that
|
|
8018
|
+
* silently skipped one is a total that is wrong by an unknown amount. A
|
|
8019
|
+
* directory holding nothing readable is an error naming what it looked for,
|
|
8020
|
+
* not an empty report.
|
|
7067
8021
|
*/
|
|
7068
|
-
const budgetPatterns = Object.keys(config.budgets ?? {});
|
|
7069
|
-
if (budgetPatterns.length > 0) {
|
|
7070
|
-
for (const row of report.byLabel) {
|
|
7071
|
-
const file = labelMap[row.label];
|
|
7072
|
-
if (file === undefined || row.breakdown.calls === 0)
|
|
7073
|
-
continue;
|
|
7074
|
-
const pattern = mostSpecificMatch(budgetPatterns, file);
|
|
7075
|
-
if (pattern === null)
|
|
7076
|
-
continue;
|
|
7077
|
-
const budget = config.budgets[pattern];
|
|
7078
|
-
if (budget <= 0)
|
|
7079
|
-
continue;
|
|
7080
|
-
/**
|
|
7081
|
-
* Input tokens per call over this label — every class that is billed
|
|
7082
|
-
* as input, because a cached token was still sent and still counted
|
|
7083
|
-
* against the model's window.
|
|
7084
|
-
*/
|
|
7085
|
-
const perCall = (row.breakdown.inputTokens + row.breakdown.cacheReadTokens + row.breakdown.cacheWriteTokens) /
|
|
7086
|
-
row.breakdown.calls;
|
|
7087
|
-
if (perCall <= 0)
|
|
7088
|
-
continue;
|
|
7089
|
-
const share = budget / perCall;
|
|
7090
|
-
// Only when the gap is wide enough to change what somebody believes.
|
|
7091
|
-
// A budget covering most of the call is doing its job quietly.
|
|
7092
|
-
if (share >= 0.5)
|
|
7093
|
-
continue;
|
|
7094
|
-
console.log();
|
|
7095
|
-
console.log(` ${c.yellow('!')} ${c.dim(wrap(t.profile.budgetVsWire(row.label === UNLABELLED ? t.profile.unlabelled() : row.label, file, n(budget), n(Math.round(perCall)), pct(share)), 74, ' '))}`);
|
|
7096
|
-
}
|
|
7097
|
-
}
|
|
7098
8022
|
/**
|
|
7099
|
-
*
|
|
7100
|
-
*
|
|
8023
|
+
* The same names, gzipped — which is what a rotated log actually looks like
|
|
8024
|
+
* a day after it rotates.
|
|
7101
8025
|
*
|
|
7102
|
-
*
|
|
7103
|
-
*
|
|
7104
|
-
*
|
|
8026
|
+
* `logrotate`, Docker's json-file driver and every cloud log export compress
|
|
8027
|
+
* yesterday's file, so a directory of a month's logs is one plain file and
|
|
8028
|
+
* twenty-nine `.gz` ones. Reading only the plain one and saying nothing
|
|
8029
|
+
* would report a month's bill from a day of it, in the flattering
|
|
8030
|
+
* direction, which is exactly the failure directory mode was added to
|
|
8031
|
+
* prevent.
|
|
7105
8032
|
*/
|
|
7106
|
-
const
|
|
7107
|
-
|
|
7108
|
-
|
|
7109
|
-
|
|
7110
|
-
|
|
7111
|
-
|
|
7112
|
-
|
|
7113
|
-
|
|
7114
|
-
|
|
7115
|
-
|
|
7116
|
-
|
|
7117
|
-
|
|
7118
|
-
}
|
|
7119
|
-
|
|
7120
|
-
|
|
7121
|
-
|
|
7122
|
-
|
|
7123
|
-
|
|
8033
|
+
const GZ_EXTENSIONS = LOG_EXTENSIONS.map((ext) => `${ext}.gz`);
|
|
8034
|
+
const READABLE = [...LOG_EXTENSIONS, ...GZ_EXTENSIONS];
|
|
8035
|
+
const target = await stat(path).catch(() => null);
|
|
8036
|
+
let logFiles = [path];
|
|
8037
|
+
if (target?.isDirectory()) {
|
|
8038
|
+
/**
|
|
8039
|
+
* Recursive under `--by-source`, flat otherwise. The fleet's whole point
|
|
8040
|
+
* is one directory per service, so the walk must descend; the flat mode
|
|
8041
|
+
* keeps its long-standing behaviour because a directory of rotated logs
|
|
8042
|
+
* with an unrelated subfolder should not quietly absorb it.
|
|
8043
|
+
*/
|
|
8044
|
+
const bySourceMode = boolFlag(args, 'by-source');
|
|
8045
|
+
const entries = await readdir(path, { withFileTypes: true, recursive: bySourceMode });
|
|
8046
|
+
logFiles = entries
|
|
8047
|
+
.filter((entry) => entry.isFile() && READABLE.some((ext) => entry.name.endsWith(ext)))
|
|
8048
|
+
.map((entry) => join(entry.parentPath ?? path, entry.name))
|
|
8049
|
+
.sort((a, b) => a.localeCompare(b));
|
|
8050
|
+
if (logFiles.length === 0) {
|
|
8051
|
+
throw new Error(t.profile.noLogsInDirectory(path, READABLE.join(', ')));
|
|
7124
8052
|
}
|
|
7125
8053
|
}
|
|
7126
|
-
if (report.total.cacheWriteTokens > 0 && report.cacheTtlFit.length === 0) {
|
|
7127
|
-
console.log(` ${c.dim(wrap(t.profile.ttlFitUnmeasured(), 74, ' '))}`);
|
|
7128
|
-
}
|
|
7129
8054
|
/**
|
|
7130
|
-
*
|
|
8055
|
+
* Gzipped files are decompressed in memory; everything else is read as text.
|
|
7131
8056
|
*
|
|
7132
|
-
*
|
|
7133
|
-
*
|
|
7134
|
-
*
|
|
7135
|
-
*
|
|
7136
|
-
*
|
|
7137
|
-
*
|
|
8057
|
+
* Decided by **extension**, not by sniffing the first two bytes: a file
|
|
8058
|
+
* named `.jsonl` whose contents happen to start with 0x1f8b is far more
|
|
8059
|
+
* likely to be a corrupt log than a mislabelled archive, and silently
|
|
8060
|
+
* treating it as one would turn a diagnosable error into an empty report.
|
|
8061
|
+
*
|
|
8062
|
+
* A `.gz` that will not decompress is an error naming the file. The
|
|
8063
|
+
* alternative — skipping it — is a total quietly missing a day, which is
|
|
8064
|
+
* the failure this repository refuses in every other place it can occur.
|
|
7138
8065
|
*/
|
|
7139
|
-
const
|
|
7140
|
-
|
|
7141
|
-
|
|
7142
|
-
|
|
7143
|
-
const name = row.label === UNLABELLED ? t.profile.unlabelled() : row.label;
|
|
7144
|
-
const reads = readsBySlice.get(`${row.label}\n${row.model}`) ?? 0;
|
|
7145
|
-
if (reads === 0) {
|
|
7146
|
-
console.log(` ${c.yellow('!')} ${c.bold(wrap(t.profile.singleTurnConfirmed(name, row.modelName, n(row.singleTurnSessions), n(row.sessions), formatUsd(row.singleTurnWriteUsd)), 74, ' '))}`);
|
|
7147
|
-
}
|
|
7148
|
-
else {
|
|
7149
|
-
console.log(` ${c.dim(wrap(t.profile.singleTurnCeiling(name, row.modelName, n(row.singleTurnSessions), n(row.sessions), formatUsd(row.singleTurnWriteUsd)), 74, ' '))}`);
|
|
7150
|
-
}
|
|
7151
|
-
}
|
|
7152
|
-
}
|
|
8066
|
+
const logTexts = await Promise.all(logFiles.map((file) => readUsageLog(file, t)));
|
|
8067
|
+
// A file that does not end in a newline would otherwise glue its last record
|
|
8068
|
+
// to the next file's first one, and both would be reported as unreadable.
|
|
8069
|
+
const raw = logTexts.map((text) => (text.endsWith('\n') ? text : `${text}\n`)).join('');
|
|
7153
8070
|
/**
|
|
7154
|
-
*
|
|
7155
|
-
*
|
|
7156
|
-
*
|
|
7157
|
-
*
|
|
8071
|
+
* The drill-down. A label that matches nothing is an error naming the labels
|
|
8072
|
+
* that exist — the route command's rule, for the route command's reason: a
|
|
8073
|
+
* report over zero calls silently filtered would read as "this workload is
|
|
8074
|
+
* free".
|
|
7158
8075
|
*/
|
|
7159
|
-
|
|
7160
|
-
const name = shape.label === UNLABELLED ? t.profile.unlabelled() : shape.label;
|
|
7161
|
-
console.log();
|
|
7162
|
-
console.log(` ${c.dim(wrap(t.profile.sessionCost(name, shape.modelName, n(shape.sessions), formatUsd(shape.medianUsd), n(shape.medianTurns), formatUsd(shape.p95Usd), formatUsd(shape.maxUsd)), 74, ' '))}`);
|
|
7163
|
-
/**
|
|
7164
|
-
* The tail, when there is one. A p95 far above the median is a shape a
|
|
7165
|
-
* quota can fix; a p95 beside it is a workload that is simply expensive,
|
|
7166
|
-
* and saying "hunt the tail" there would send somebody after nothing.
|
|
7167
|
-
* The threshold is in the sentence rather than hidden here.
|
|
7168
|
-
*/
|
|
7169
|
-
if (shape.medianUsd > 0 && shape.p95Usd > 10 * shape.medianUsd) {
|
|
7170
|
-
console.log(` ${c.yellow('!')} ${c.dim(wrap(t.profile.sessionCostTail((shape.p95Usd / shape.medianUsd).toFixed(0)), 74, ' '))}`);
|
|
7171
|
-
}
|
|
7172
|
-
}
|
|
8076
|
+
const onlyLabel = stringFlag(args, 'label');
|
|
7173
8077
|
/**
|
|
7174
|
-
* The
|
|
7175
|
-
*
|
|
7176
|
-
*
|
|
7177
|
-
*
|
|
7178
|
-
*
|
|
8078
|
+
* The drill-down in time. `--since`/`--until` take a UTC day or a full
|
|
8079
|
+
* timestamp; a bare day means the whole of it — since its first instant,
|
|
8080
|
+
* until its last — because "--until 2026-08-14" excluding the named day is
|
|
8081
|
+
* a trap sprung on everyone who reads dates the way humans do. Internally
|
|
8082
|
+
* the window is half-open `[since, until)`, so two adjacent windows share
|
|
8083
|
+
* no record.
|
|
7179
8084
|
*/
|
|
7180
|
-
|
|
7181
|
-
|
|
7182
|
-
|
|
8085
|
+
const now = Date.now();
|
|
8086
|
+
const sinceWhen = parseWhen(args, 'since', false, t, now);
|
|
8087
|
+
const untilWhen = parseWhen(args, 'until', true, t, now);
|
|
8088
|
+
const relativeWindow = sinceWhen.relative || untilWhen.relative;
|
|
8089
|
+
const sinceMs = sinceWhen.ms;
|
|
8090
|
+
const untilMs = untilWhen.ms;
|
|
8091
|
+
if (sinceMs !== undefined && untilMs !== undefined && sinceMs >= untilMs) {
|
|
8092
|
+
throw new Error(t.profile.sinceAfterUntil());
|
|
7183
8093
|
}
|
|
8094
|
+
const windowed = sinceMs !== undefined || untilMs !== undefined;
|
|
7184
8095
|
/**
|
|
7185
|
-
*
|
|
7186
|
-
*
|
|
7187
|
-
*
|
|
7188
|
-
*
|
|
7189
|
-
*
|
|
7190
|
-
*
|
|
8096
|
+
* How old the price table behind every dollar below is. Stated only when it
|
|
8097
|
+
* is old enough to matter: `models` and `doctor` always print the date, but
|
|
8098
|
+
* a profile is read for its figures, and the one fact that silently
|
|
8099
|
+
* invalidates all of them is a table the provider has re-priced since.
|
|
8100
|
+
* The threshold is in the sentence, and the number behind it is
|
|
8101
|
+
* `STALE_PRICING_DAYS` — shared with the MCP report and the browser's bill,
|
|
8102
|
+
* which used to keep their own copies of it.
|
|
7191
8103
|
*/
|
|
7192
|
-
|
|
7193
|
-
|
|
8104
|
+
const pricingAgeDays = reviewAgeDays(pricing.lastReviewed, new Date());
|
|
8105
|
+
const pricingStale = pricingAgeDays !== null && pricingAgeDays > STALE_PRICING_DAYS
|
|
8106
|
+
? { date: pricing.lastReviewed, days: pricingAgeDays }
|
|
8107
|
+
: null;
|
|
8108
|
+
const report = profileUsage(raw, { catalogue: pricing, label: onlyLabel, sinceMs, untilMs });
|
|
8109
|
+
if (report.total.calls === 0 && report.unpriced.calls === 0) {
|
|
8110
|
+
if (onlyLabel !== undefined || windowed) {
|
|
8111
|
+
// Diagnose against the log without the failed filter, so the error can
|
|
8112
|
+
// name what does exist instead of describing an absence.
|
|
8113
|
+
const unfiltered = profileUsage(raw, { catalogue: pricing });
|
|
8114
|
+
if (unfiltered.total.calls > 0 || unfiltered.unpriced.calls > 0) {
|
|
8115
|
+
if (onlyLabel !== undefined && !unfiltered.byLabel.some((r) => r.label === onlyLabel)) {
|
|
8116
|
+
const available = unfiltered.byLabel
|
|
8117
|
+
.map((r) => (r.label === UNLABELLED ? t.profile.unlabelled() : r.label))
|
|
8118
|
+
.join(', ');
|
|
8119
|
+
throw new Error(t.route.labelNotFound(onlyLabel, available || '—'));
|
|
8120
|
+
}
|
|
8121
|
+
if (windowed) {
|
|
8122
|
+
/**
|
|
8123
|
+
* A window that matches nothing must not become a $0 report — under
|
|
8124
|
+
* `--max-usd` it would pass a budget gate over a period the log
|
|
8125
|
+
* simply does not cover, which is the flattering non-answer. The
|
|
8126
|
+
* error names what the log *does* cover, or says it has no clock at
|
|
8127
|
+
* all, so the fix is visible in the message.
|
|
8128
|
+
*/
|
|
8129
|
+
if (unfiltered.span === null)
|
|
8130
|
+
throw new Error(t.profile.windowNeedsClock());
|
|
8131
|
+
throw new Error(`${t.profile.windowMatchesNothing(dayOf(unfiltered.span.fromMs), dayOf(unfiltered.span.toMs))}${relativeWindow ? ` ${t.profile.windowRelativeEmpty()}` : ''}`);
|
|
8132
|
+
}
|
|
8133
|
+
}
|
|
8134
|
+
}
|
|
7194
8135
|
}
|
|
8136
|
+
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
8137
|
+
const pct = (share) => `${(share * 100).toFixed(1)}%`;
|
|
7195
8138
|
/**
|
|
7196
|
-
*
|
|
7197
|
-
* product has had: on a bill of twenty thousand, the rules recover two hundred.
|
|
8139
|
+
* A gate armed over a report with nothing in it, refused rather than passed.
|
|
7198
8140
|
*
|
|
7199
|
-
*
|
|
7200
|
-
*
|
|
7201
|
-
*
|
|
7202
|
-
*
|
|
7203
|
-
*
|
|
7204
|
-
*
|
|
7205
|
-
*
|
|
8141
|
+
* The product had already made this call once and only once. A `--since` that
|
|
8142
|
+
* matches no record throws, and the comment above it says why: "under
|
|
8143
|
+
* --max-usd it would pass a budget gate over a period the log does not
|
|
8144
|
+
* cover". Two other ways of measuring nothing reached the same gate and were
|
|
8145
|
+
* never given the same answer — a log whose every line was unreadable, and a
|
|
8146
|
+
* log with no records at all. Both exited 0 against a budget, on the human
|
|
8147
|
+
* path and under `--json`.
|
|
7206
8148
|
*
|
|
7207
|
-
*
|
|
7208
|
-
*
|
|
7209
|
-
*
|
|
7210
|
-
*
|
|
7211
|
-
|
|
7212
|
-
const levers = billLevers(report, { catalogue: pricing });
|
|
7213
|
-
console.log();
|
|
7214
|
-
console.log(sectionHeading(t.profile.leversHeading()));
|
|
7215
|
-
/**
|
|
7216
|
-
* Every lever below describes a mixture when nothing carries a label.
|
|
8149
|
+
* That is the doctrine's own rule, verbatim: *a period, or a service, nobody
|
|
8150
|
+
* measured is not one under budget. Name the gap; never report the absence as
|
|
8151
|
+
* a pass.* A pipeline that stopped writing looks exactly like a quiet month,
|
|
8152
|
+
* and the whole reason the rule exists is that the quiet month is the one
|
|
8153
|
+
* that reads as green.
|
|
7217
8154
|
*
|
|
7218
|
-
*
|
|
7219
|
-
*
|
|
7220
|
-
*
|
|
7221
|
-
* covering both. The session case already tells the reader to add the field;
|
|
7222
|
-
* this one named the row `unlabelled` and said nothing, as though that were a
|
|
7223
|
-
* workload.
|
|
8155
|
+
* `--allow-empty` is the way to say a period with no calls is the expected
|
|
8156
|
+
* answer, because that is a real thing a nightly job wants and it should be
|
|
8157
|
+
* said out loud rather than inferred from silence.
|
|
7224
8158
|
*/
|
|
7225
|
-
const
|
|
7226
|
-
|
|
7227
|
-
|
|
8159
|
+
const gateArmed = typeof args.flags.get('max-usd') === 'string' ||
|
|
8160
|
+
typeof args.flags.get('max-growth-usd') === 'string' ||
|
|
8161
|
+
typeof args.flags.get('max-cache-loss-usd') === 'string' ||
|
|
8162
|
+
typeof args.flags.get('max-day-usd') === 'string' ||
|
|
8163
|
+
typeof args.flags.get('max-session-usd') === 'string' ||
|
|
8164
|
+
config.spend !== undefined;
|
|
8165
|
+
if (gateArmed && report.total.calls === 0 && !boolFlag(args, 'allow-empty')) {
|
|
8166
|
+
const gap = report.unpriced.calls > 0
|
|
8167
|
+
? t.profile.nothingUnpriced(report.unpriced.calls)
|
|
8168
|
+
: report.skippedLines.length > 0
|
|
8169
|
+
? t.profile.nothingUnreadable(report.skippedLines.length)
|
|
8170
|
+
: t.profile.nothingEmpty();
|
|
8171
|
+
throw new Error(t.profile.gateNothingMeasured(gap));
|
|
7228
8172
|
}
|
|
7229
|
-
if (
|
|
7230
|
-
|
|
8173
|
+
if (boolFlag(args, 'by-source')) {
|
|
8174
|
+
await profileBySource(args, config, logFiles, logTexts, pricing, { onlyLabel, sinceMs, untilMs }, t);
|
|
8175
|
+
return;
|
|
7231
8176
|
}
|
|
7232
|
-
|
|
7233
|
-
|
|
7234
|
-
|
|
7235
|
-
console.log();
|
|
7236
|
-
/**
|
|
7237
|
-
* The headline is the **combined** figure, and the options underneath are
|
|
7238
|
-
* the ways to reach it — not rows to add up. Batching a routed call
|
|
7239
|
-
* discounts the cheaper model's price, so listing them separately printed
|
|
7240
|
-
* $12.60 and $10.50 against a slice that had spent $21.00: a saving larger
|
|
7241
|
-
* than the bill it came from, in the flattering direction.
|
|
7242
|
-
*/
|
|
7243
|
-
console.log(` ${c.green('→')} ${c.bold(wrap(t.profile.leverSlice(label, slice.modelName, formatUsd(slice.combinedUsd), pct(slice.shareOfBill)), 74, ' '))}`);
|
|
7244
|
-
console.log(` ${c.dim(t.profile.leverCalls(t.profile.calls(slice.calls), formatUsd(slice.spentUsd)))}`);
|
|
7245
|
-
if (slice.route) {
|
|
7246
|
-
console.log(` ${c.dim('·')} ${c.dim(wrap(t.profile.leverRoute(slice.route.candidate.displayName, formatUsd(slice.route.savingUsd)), 74, ' '))}`);
|
|
7247
|
-
}
|
|
7248
|
-
if (slice.batch) {
|
|
7249
|
-
console.log(` ${c.dim('·')} ${c.dim(wrap(t.profile.leverBatch(formatUsd(slice.batch.savingUsd)), 74, ' '))}`);
|
|
7250
|
-
}
|
|
7251
|
-
// The arithmetic is exact and the quality question is untouched by it.
|
|
7252
|
-
// Naming the command is the difference between a saving and a gamble.
|
|
7253
|
-
if (slice.route) {
|
|
7254
|
-
console.log(` ${c.dim(wrap(t.profile.leverRouteVerify(slice.route.candidate.id), 74, ' '))}`);
|
|
7255
|
-
}
|
|
7256
|
-
}
|
|
8177
|
+
if (boolFlag(args, 'dry-run')) {
|
|
8178
|
+
profileDryRun(args, report, t);
|
|
8179
|
+
return;
|
|
7257
8180
|
}
|
|
7258
|
-
console.log();
|
|
7259
|
-
console.log(` ${c.dim(wrap(t.profile.leverPromptCeiling(formatUsd(levers.promptCeilingUsd), pct(levers.promptCeilingShare)), 74, ' '))}`);
|
|
7260
8181
|
/**
|
|
7261
|
-
*
|
|
7262
|
-
*
|
|
7263
|
-
*
|
|
7264
|
-
*
|
|
7265
|
-
|
|
8182
|
+
* The previous log, loaded before the output paths split so the growth gate
|
|
8183
|
+
* exists under `--json` too — a CI step reads the JSON and trusts the exit
|
|
8184
|
+
* code, and a gate that only arms in the human rendering is a gate CI never
|
|
8185
|
+
* had.
|
|
8186
|
+
*/
|
|
8187
|
+
/* Filled by `runProfileGates` on whichever output path runs, and read by
|
|
8188
|
+
the side-file writer, which is why it is declared here rather than
|
|
8189
|
+
inlined at either call. */
|
|
8190
|
+
let gates = { failed: false, verdicts: [] };
|
|
8191
|
+
const againstPath = stringFlag(args, 'against');
|
|
8192
|
+
// The same filter on both sides: comparing one workload's bill against the
|
|
8193
|
+
// whole previous log would report every sibling workload as vanished savings.
|
|
8194
|
+
const previous = againstPath !== undefined
|
|
8195
|
+
// The same reader as the log itself, so `--against last-month.jsonl.gz`
|
|
8196
|
+
// works: a comparison that could only read one of the two formats would
|
|
8197
|
+
// be a flag that fails on exactly the rotated file it exists to read.
|
|
8198
|
+
? profileUsage(await readUsageLog(againstPath, t), {
|
|
8199
|
+
catalogue: pricing,
|
|
8200
|
+
label: onlyLabel,
|
|
8201
|
+
// The same window on both sides, for the same reason as the label:
|
|
8202
|
+
// a windowed bill against an unwindowed one compares a slice to a
|
|
8203
|
+
// whole and calls the difference growth.
|
|
8204
|
+
sinceMs,
|
|
8205
|
+
untilMs,
|
|
8206
|
+
})
|
|
8207
|
+
: null;
|
|
8208
|
+
const againstDelta = previous !== null && previous.total.calls > 0
|
|
8209
|
+
? report.total.totalUsd - previous.total.totalUsd
|
|
8210
|
+
: null;
|
|
8211
|
+
/**
|
|
8212
|
+
* The same tokens at another model's rates, computed before the output paths
|
|
8213
|
+
* split so `--json` carries it too.
|
|
7266
8214
|
*
|
|
7267
|
-
*
|
|
7268
|
-
*
|
|
7269
|
-
*
|
|
7270
|
-
*
|
|
7271
|
-
* - spend already on the target is stated separately, because a difference
|
|
7272
|
-
* computed over money that cannot move is a percentage of the wrong
|
|
7273
|
-
* denominator.
|
|
8215
|
+
* An unknown id **throws** rather than printing nothing. A flag that silently
|
|
8216
|
+
* does nothing is worse than a missing feature: the reader typed a question,
|
|
8217
|
+
* got a report with no answer in it, and has no way to tell a typo from a
|
|
8218
|
+
* model this comparison had nothing to say about.
|
|
7274
8219
|
*/
|
|
7275
|
-
|
|
7276
|
-
|
|
7277
|
-
|
|
7278
|
-
|
|
7279
|
-
|
|
7280
|
-
|
|
7281
|
-
|
|
7282
|
-
|
|
7283
|
-
|
|
7284
|
-
|
|
7285
|
-
const line = t.profile.whatIfTotal(formatUsd(whatIf.currentUsd), formatUsd(whatIf.targetUsd), formatUsd(Math.abs(whatIf.deltaUsd)));
|
|
7286
|
-
console.log(` ${cheaper ? c.green('→') : c.yellow('!')} ${c.bold(wrap(line, 74, ' '))}`);
|
|
7287
|
-
console.log(` ${c.dim(wrap(cheaper ? t.profile.whatIfCheaper() : t.profile.whatIfDearer(), 74, ' '))}`);
|
|
7288
|
-
/**
|
|
7289
|
-
* The other half of the whole decision: the same move with the target's
|
|
7290
|
-
* Batch API on top. Computed against the target's rates, never by
|
|
7291
|
-
* adding two savings, and hedged the only honest way — whether these
|
|
7292
|
-
* calls can wait is not in the log.
|
|
7293
|
-
*/
|
|
7294
|
-
if (whatIf.batchOnTarget !== null) {
|
|
7295
|
-
console.log(` ${c.dim(wrap(t.profile.whatIfBatchOnTarget(formatUsd(whatIf.batchOnTarget.targetUsd), formatUsd(whatIf.targetUsd)), 74, ' '))}`);
|
|
7296
|
-
}
|
|
7297
|
-
for (const slice of whatIf.slices.slice(0, 5)) {
|
|
7298
|
-
const label = slice.label === UNLABELLED ? t.profile.unlabelled() : slice.label;
|
|
7299
|
-
console.log(` ${c.dim('·')} ${c.dim(wrap(t.profile.whatIfSlice(label, slice.model, formatUsd(slice.currentUsd), formatUsd(slice.targetUsd)), 74, ' '))}`);
|
|
7300
|
-
/**
|
|
7301
|
-
* Cache traffic the target could not grant, said loudly and beside
|
|
7302
|
-
* the figure it corrects. The standard row prices cache entries the
|
|
7303
|
-
* target's minimum would refuse to create — an error that flatters
|
|
7304
|
-
* the move, which is the direction this repository refuses.
|
|
7305
|
-
*/
|
|
7306
|
-
if (slice.cacheBeyondTarget !== null) {
|
|
7307
|
-
console.log(` ${c.yellow('!')} ${c.dim(wrap(t.profile.whatIfCacheBeyond(n(slice.maxCallInputTokens), n(slice.cacheBeyondTarget.minTokens), formatUsd(slice.cacheBeyondTarget.noCacheUsd)), 74, ' '))}`);
|
|
7308
|
-
}
|
|
7309
|
-
}
|
|
7310
|
-
}
|
|
7311
|
-
/**
|
|
7312
|
-
* The refusal, and it is loud. A call larger than the target's window is
|
|
7313
|
-
* not a cheaper call, and a comparison that priced it anyway would report
|
|
7314
|
-
* a saving for traffic that would have failed outright.
|
|
7315
|
-
*/
|
|
7316
|
-
for (const slice of whatIf.overContext.slice(0, 3)) {
|
|
7317
|
-
const label = slice.label === UNLABELLED ? t.profile.unlabelled() : slice.label;
|
|
7318
|
-
console.log(` ${c.yellow('!')} ${c.bold(wrap(t.profile.whatIfOverContext(label, n(slice.maxCallInputTokens), n(whatIf.target.contextWindow), formatUsd(slice.currentUsd)), 74, ' '))}`);
|
|
7319
|
-
}
|
|
7320
|
-
// Money that is already there cannot move, and leaving it out of the
|
|
7321
|
-
// totals above is only honest if the reader is told it exists.
|
|
7322
|
-
if (whatIf.alreadyOnTarget.calls > 0) {
|
|
7323
|
-
console.log(` ${c.dim(wrap(t.profile.whatIfAlreadyThere(t.profile.calls(whatIf.alreadyOnTarget.calls), formatUsd(whatIf.alreadyOnTarget.usd)), 74, ' '))}`);
|
|
7324
|
-
}
|
|
7325
|
-
// Models with no current price have no difference to state — their target
|
|
7326
|
-
// cost is knowable and the subtraction is not.
|
|
7327
|
-
if (whatIf.unpricedCalls > 0) {
|
|
7328
|
-
console.log(` ${c.dim(wrap(t.profile.whatIfUnpriced(t.profile.calls(whatIf.unpricedCalls), whatIf.unpricedModels.join(', ')), 74, ' '))}`);
|
|
7329
|
-
}
|
|
8220
|
+
/**
|
|
8221
|
+
* The largest call against each model's window, computed once here so the
|
|
8222
|
+
* terminal, the JSON and the markdown state the same ratio — the
|
|
8223
|
+
* denominator lives in the (possibly overlaid) catalogue.
|
|
8224
|
+
*/
|
|
8225
|
+
const pressures = contextPressure(report, pricing);
|
|
8226
|
+
const whatIfModel = stringFlag(args, 'what-if');
|
|
8227
|
+
const whatIf = whatIfModel !== undefined ? repriceProfile(report, whatIfModel, pricing) : null;
|
|
8228
|
+
if (whatIfModel !== undefined && whatIf === null) {
|
|
8229
|
+
throw new Error(t.profile.whatIfUnknown(whatIfModel, pricing.models.map((m) => m.id).join(', ')));
|
|
7330
8230
|
}
|
|
7331
8231
|
/**
|
|
7332
|
-
*
|
|
7333
|
-
*
|
|
7334
|
-
*
|
|
7335
|
-
*
|
|
7336
|
-
*
|
|
7337
|
-
|
|
7338
|
-
|
|
7339
|
-
|
|
7340
|
-
|
|
7341
|
-
|
|
7342
|
-
|
|
8232
|
+
* The drivers of the change, per label and per model, computed once here so
|
|
8233
|
+
* the terminal, the JSON and any future rendering describe the same change.
|
|
8234
|
+
* The model half answers the question the label half cannot: "the growth is
|
|
8235
|
+
* traffic moving from Haiku to Opus" is a fact about the mix, invisible in
|
|
8236
|
+
* per-workload rows whose names did not change.
|
|
8237
|
+
*/
|
|
8238
|
+
const labelDrivers = previous !== null && previous.total.calls > 0
|
|
8239
|
+
? driversBetween(previous.byLabel.map((r) => ({ key: r.label, usd: r.breakdown.totalUsd })), report.byLabel.map((r) => ({ key: r.label, usd: r.breakdown.totalUsd })))
|
|
8240
|
+
: [];
|
|
8241
|
+
const modelDrivers = previous !== null && previous.total.calls > 0
|
|
8242
|
+
? driversBetween(previous.byModel.map((r) => ({ key: r.model, usd: r.breakdown.totalUsd })), report.byModel.map((r) => ({ key: r.model, usd: r.breakdown.totalUsd })))
|
|
8243
|
+
: [];
|
|
8244
|
+
/**
|
|
8245
|
+
* Whether the two logs share any time at all. This comparison is meant for
|
|
8246
|
+
* disjoint periods or snapshots of different systems; when both spans are
|
|
8247
|
+
* known and intersect, the same calls may sit on both sides of the
|
|
8248
|
+
* subtraction and the "growth" is partly the same money counted twice.
|
|
8249
|
+
* Only decidable when both logs carry a clock — three states, as always:
|
|
8250
|
+
* warned, clear, or unknown, and unknown stays silent rather than clear.
|
|
7343
8251
|
*/
|
|
7344
|
-
|
|
7345
|
-
|
|
7346
|
-
|
|
7347
|
-
|
|
7348
|
-
|
|
7349
|
-
|
|
7350
|
-
|
|
7351
|
-
|
|
8252
|
+
const againstOverlap = previous !== null &&
|
|
8253
|
+
previous.total.calls > 0 &&
|
|
8254
|
+
previous.span !== null &&
|
|
8255
|
+
report.span !== null &&
|
|
8256
|
+
Math.min(report.span.toMs, previous.span.toMs) >=
|
|
8257
|
+
Math.max(report.span.fromMs, previous.span.fromMs)
|
|
8258
|
+
? {
|
|
8259
|
+
fromMs: Math.max(report.span.fromMs, previous.span.fromMs),
|
|
8260
|
+
toMs: Math.min(report.span.toMs, previous.span.toMs),
|
|
7352
8261
|
}
|
|
8262
|
+
: null;
|
|
8263
|
+
// A gate flag that silently does nothing is not an answer — same rule as
|
|
8264
|
+
// --apply-suggestions without --suggest.
|
|
8265
|
+
if (typeof args.flags.get('max-growth-usd') === 'string' && againstPath === undefined) {
|
|
8266
|
+
throw new Error(t.profile.maxGrowthNeedsAgainst());
|
|
7353
8267
|
}
|
|
7354
|
-
|
|
8268
|
+
if (boolFlag(args, 'json')) {
|
|
7355
8269
|
/**
|
|
7356
|
-
*
|
|
7357
|
-
*
|
|
7358
|
-
*
|
|
8270
|
+
* The report, plus everything the human output leads on.
|
|
8271
|
+
*
|
|
8272
|
+
* Additive rather than a reshape: `report` keeps the shape `@trazum/core`
|
|
8273
|
+
* returns. The cache verdict is included because leaving a consumer to
|
|
8274
|
+
* re-derive it means two implementations of a sign convention where positive
|
|
8275
|
+
* means *worse*, and one of them will eventually get it backwards.
|
|
8276
|
+
*
|
|
8277
|
+
* `levers` is included because it was not, and that made the flagship
|
|
8278
|
+
* section terminal-only: "What would actually move this bill" — the reason
|
|
8279
|
+
* the command exists — was invisible to any pipeline, dashboard or CI step
|
|
8280
|
+
* reading the JSON. A finding the machine-readable output omits is a finding
|
|
8281
|
+
* the reader's tooling will never surface.
|
|
7359
8282
|
*/
|
|
7360
|
-
console.log(
|
|
7361
|
-
console.log(sectionHeading(t.profile.historyHeading()));
|
|
7362
|
-
console.log(` ${c.dim(wrap(t.profile.historyNoSessions(), 74, ' '))}`);
|
|
7363
|
-
}
|
|
7364
|
-
/**
|
|
7365
|
-
* Where the output spend concentrates — the actionable half of "output
|
|
7366
|
-
* dominates", which the headline above could only state as a total.
|
|
7367
|
-
*
|
|
7368
|
-
* Two bills with identical output spend want opposite responses. Six per cent
|
|
7369
|
-
* of calls holding half of it is a tail, and a tail has a cause worth a
|
|
7370
|
-
* morning; forty-five per cent is what evenly spread looks like, and the only
|
|
7371
|
-
* lever there is asking every answer to be shorter. The threshold between the
|
|
7372
|
-
* two is a quarter of the calls — far enough from both shapes that rounding
|
|
7373
|
-
* cannot flip the message, and stated here because it is a presentation choice,
|
|
7374
|
-
* not a measurement.
|
|
7375
|
-
*/
|
|
7376
|
-
if (report.outputShapes.length > 0) {
|
|
7377
|
-
console.log();
|
|
7378
|
-
console.log(sectionHeading(t.profile.outputShapeHeading()));
|
|
7379
|
-
for (const shape of report.outputShapes.slice(0, 3)) {
|
|
7380
|
-
const label = shape.label === UNLABELLED ? t.profile.unlabelled() : shape.label;
|
|
7381
|
-
const isTail = shape.heavyCallShare < 0.25;
|
|
7382
|
-
console.log();
|
|
7383
|
-
if (isTail) {
|
|
7384
|
-
console.log(` ${c.bold(wrap(t.profile.outputTail(label, shape.modelName, pct(shape.heavyCallShare), pct(shape.heavySpendShare), n(shape.aboveTokens), formatUsd(shape.outputUsd)), 74, ' '))}`);
|
|
7385
|
-
console.log(` ${c.dim(wrap(t.profile.outputTailAdvice(), 74, ' '))}`);
|
|
7386
|
-
}
|
|
7387
|
-
else {
|
|
7388
|
-
console.log(` ${c.bold(wrap(t.profile.outputFlat(label, shape.modelName, pct(shape.heavyCallShare), pct(shape.heavySpendShare), formatUsd(shape.outputUsd)), 74, ' '))}`);
|
|
7389
|
-
console.log(` ${c.dim(wrap(t.profile.outputFlatAdvice(), 74, ' '))}`);
|
|
7390
|
-
}
|
|
8283
|
+
console.log(JSON.stringify({
|
|
7391
8284
|
/**
|
|
7392
|
-
*
|
|
7393
|
-
*
|
|
7394
|
-
*
|
|
7395
|
-
*
|
|
8285
|
+
* `schemaVersion` arrives inside `report` now, and that is the point.
|
|
8286
|
+
*
|
|
8287
|
+
* It used to be stamped here, which made the profile the one contract
|
|
8288
|
+
* of the ten whose version existed only if you went through this
|
|
8289
|
+
* command. `docs/format.md` says every document carries it and
|
|
8290
|
+
* `conform` rejects one that does not, so a connector author using
|
|
8291
|
+
* `@trazum/core` produced a profile this tool refuses.
|
|
8292
|
+
*
|
|
8293
|
+
* The contract is documented in docs/json-output.md and enforced by
|
|
8294
|
+
* json-contract.test.js. It changes only when a field's meaning
|
|
8295
|
+
* changes or one is removed — new findings arrive as new keys, so a
|
|
8296
|
+
* consumer that ignores unknown ones keeps working.
|
|
7396
8297
|
*/
|
|
7397
|
-
|
|
7398
|
-
|
|
7399
|
-
|
|
7400
|
-
|
|
8298
|
+
...report,
|
|
8299
|
+
cache: cacheEconomics(report.total),
|
|
8300
|
+
cacheByLabel: report.byLabel.map((r) => ({
|
|
8301
|
+
label: r.label,
|
|
8302
|
+
cache: cacheEconomics(r.breakdown),
|
|
8303
|
+
})),
|
|
8304
|
+
// The provenance of every dollar above: which price table, how old.
|
|
8305
|
+
pricing: { lastReviewed: pricing.lastReviewed, ageDays: pricingAgeDays },
|
|
8306
|
+
levers: billLevers(report, { catalogue: pricing }),
|
|
8307
|
+
// Present only when --against was passed: null delta means the
|
|
8308
|
+
// previous log had nothing priced, which is a different answer from
|
|
8309
|
+
// zero growth.
|
|
8310
|
+
...(previous !== null
|
|
8311
|
+
? {
|
|
8312
|
+
against: {
|
|
8313
|
+
previousTotalUsd: previous.total.totalUsd,
|
|
8314
|
+
deltaUsd: againstDelta,
|
|
8315
|
+
// The same drivers the terminal names, as data. A finding
|
|
8316
|
+
// the machine-readable output omits is a finding the
|
|
8317
|
+
// reader's tooling will never surface.
|
|
8318
|
+
byLabel: labelDrivers,
|
|
8319
|
+
byModel: modelDrivers,
|
|
8320
|
+
},
|
|
8321
|
+
}
|
|
8322
|
+
: {}),
|
|
8323
|
+
// How close each slice's largest call is to its model's window —
|
|
8324
|
+
// derived like `cache` and `levers`, so a dashboard need not
|
|
8325
|
+
// re-derive a ratio whose denominator lives in the catalogue.
|
|
8326
|
+
contextPressure: pressures,
|
|
8327
|
+
// Present only when --what-if was passed. `sameTokensAssumed` rides
|
|
8328
|
+
// along inside it so a consumer cannot print the dollar figure
|
|
8329
|
+
// without the caveat being in the same object.
|
|
8330
|
+
...(whatIf !== null ? { whatIf } : {}),
|
|
8331
|
+
}, null, 2));
|
|
8332
|
+
gates = await runProfileGates({
|
|
8333
|
+
args, config, configDir, report, previous, againstDelta, windowed, now, pricing, t,
|
|
8334
|
+
});
|
|
8335
|
+
await writeProfileSideFiles({
|
|
8336
|
+
args, path, report, previous, pressures, labelDrivers, modelDrivers, againstOverlap,
|
|
8337
|
+
whatIf, pricingStale, windowed, gates, pricing, t,
|
|
8338
|
+
});
|
|
8339
|
+
return;
|
|
7401
8340
|
}
|
|
7402
8341
|
/**
|
|
7403
|
-
*
|
|
8342
|
+
* Nothing priced means there is no report, not a report of zero.
|
|
7404
8343
|
*
|
|
7405
|
-
* The
|
|
7406
|
-
*
|
|
7407
|
-
*
|
|
7408
|
-
*
|
|
7409
|
-
*
|
|
8344
|
+
* The guard was `total.calls === 0 && unpriced.calls === 0`, so a log whose every
|
|
8345
|
+
* model was unknown fell through and printed a full report built from a zeroed
|
|
8346
|
+
* total: `0 calls · $0`, four `$0 / 0.0%` rows, a meaningless "Input is 0.0% of
|
|
8347
|
+
* this bill", and — on a log containing a hundred thousand cache-read tokens —
|
|
8348
|
+
* the flatly false "Caching was never used on these calls".
|
|
7410
8349
|
*
|
|
7411
|
-
*
|
|
7412
|
-
*
|
|
7413
|
-
* in the sentence rather than hidden here. Both figures are bucket
|
|
7414
|
-
* ceilings, so the ratio is coarse by construction and the copy says so.
|
|
8350
|
+
* Two affirmatively wrong claims and a $0 headline for a real bill. The trailing
|
|
8351
|
+
* unpriced note was the only correct line on screen, and it was the quietest.
|
|
7415
8352
|
*/
|
|
7416
|
-
if (report.
|
|
8353
|
+
if (report.total.calls === 0) {
|
|
7417
8354
|
console.log();
|
|
7418
|
-
console.log(
|
|
7419
|
-
|
|
7420
|
-
|
|
7421
|
-
console.log();
|
|
7422
|
-
if (shape.medianWithinTokens === null || shape.p95WithinTokens === null || shape.p95OverMedian === null) {
|
|
7423
|
-
/**
|
|
7424
|
-
* The covering bucket is the open-ended last one, so there is no
|
|
7425
|
-
* ceiling to name. Said rather than skipped: a slice whose calls are
|
|
7426
|
-
* all above a million tokens is a finding, and silence would drop it.
|
|
7427
|
-
*/
|
|
7428
|
-
console.log(` ${c.bold(wrap(t.profile.inputHuge(label, shape.modelName, t.profile.calls(shape.calls), formatUsd(shape.inputUsd)), 74, ' '))}`);
|
|
7429
|
-
continue;
|
|
7430
|
-
}
|
|
7431
|
-
const skewed = shape.p95OverMedian >= 4;
|
|
7432
|
-
const line = skewed
|
|
7433
|
-
? t.profile.inputSkewed(label, shape.modelName, n(shape.medianWithinTokens), n(shape.p95WithinTokens), shape.p95OverMedian.toFixed(1), formatUsd(shape.inputUsd))
|
|
7434
|
-
: t.profile.inputEven(label, shape.modelName, n(shape.medianWithinTokens), n(shape.p95WithinTokens), formatUsd(shape.inputUsd));
|
|
7435
|
-
console.log(` ${c.bold(wrap(line, 74, ' '))}`);
|
|
7436
|
-
console.log(` ${c.dim(wrap(skewed ? t.profile.inputSkewedAdvice() : t.profile.inputEvenAdvice(), 74, ' '))}`);
|
|
7437
|
-
/**
|
|
7438
|
-
* What that size actually costs. A cache read is a tenth of input on
|
|
7439
|
-
* Anthropic, so a large slice reading almost everything from cache is a
|
|
7440
|
-
* very different bill from one paying full rate — and the token counts
|
|
7441
|
-
* alone cannot tell them apart.
|
|
7442
|
-
*/
|
|
7443
|
-
if (shape.cachedShare >= 0.5) {
|
|
7444
|
-
console.log(` ${c.dim(wrap(t.profile.inputMostlyCached(pct(shape.cachedShare)), 74, ' '))}`);
|
|
7445
|
-
}
|
|
7446
|
-
else if (shape.cachedShare < 0.1) {
|
|
7447
|
-
console.log(` ${c.dim(wrap(t.profile.inputFullRate(), 74, ' '))}`);
|
|
7448
|
-
}
|
|
7449
|
-
}
|
|
8355
|
+
console.log(c.dim(report.unpriced.calls === 0 ? t.profile.empty() : t.profile.nothingPriced()));
|
|
8356
|
+
reportProfileGaps(report, t, n, pricingStale);
|
|
8357
|
+
return;
|
|
7450
8358
|
}
|
|
8359
|
+
const shares = sharesOf(report.total);
|
|
8360
|
+
const parts = [
|
|
8361
|
+
[t.profile.partInput(), report.total.inputUsd, shares.input, report.total.inputTokens],
|
|
8362
|
+
[t.profile.partCacheRead(), report.total.cacheReadUsd, shares.cacheRead, report.total.cacheReadTokens],
|
|
8363
|
+
[t.profile.partCacheWrite(), report.total.cacheWriteUsd, shares.cacheWrite, report.total.cacheWriteTokens],
|
|
8364
|
+
[t.profile.partOutput(), report.total.outputUsd, shares.output, report.total.outputTokens],
|
|
8365
|
+
];
|
|
8366
|
+
console.log();
|
|
8367
|
+
console.log(sectionHeading(t.profile.heading()));
|
|
8368
|
+
console.log(` ${t.profile.spent(t.profile.calls(report.total.calls), formatUsd(report.total.totalUsd))}`);
|
|
7451
8369
|
/**
|
|
7452
|
-
*
|
|
7453
|
-
*
|
|
7454
|
-
*
|
|
7455
|
-
*
|
|
7456
|
-
*
|
|
7457
|
-
*
|
|
7458
|
-
* and quiet above half, so the reader sees it coming either way. No
|
|
7459
|
-
* prediction of *when*: a straight line through two points is a guess
|
|
7460
|
-
* wearing arithmetic's clothes.
|
|
8370
|
+
* The period, when the log carries a clock — stated, never extrapolated. A
|
|
8371
|
+
* span makes the reader's own monthly arithmetic valid; a per-month figure
|
|
8372
|
+
* printed from a partial month would be this tool doing the guessing it
|
|
8373
|
+
* exists to end. Partial coverage is said in the same breath, because a span
|
|
8374
|
+
* over a third of the calls silently presented as the log's period is a
|
|
8375
|
+
* figure attributed to something it does not describe.
|
|
7461
8376
|
*/
|
|
7462
|
-
{
|
|
7463
|
-
|
|
7464
|
-
|
|
7465
|
-
|
|
7466
|
-
|
|
7467
|
-
|
|
7468
|
-
|
|
7469
|
-
|
|
7470
|
-
|
|
7471
|
-
|
|
7472
|
-
|
|
7473
|
-
}
|
|
7474
|
-
else {
|
|
7475
|
-
console.log(` ${c.dim(wrap(line, 74, ' '))}`);
|
|
7476
|
-
}
|
|
7477
|
-
}
|
|
7478
|
-
}
|
|
8377
|
+
if (report.span !== null) {
|
|
8378
|
+
const totalParsed = report.total.calls + report.unpriced.calls;
|
|
8379
|
+
const partial = report.span.calls < totalParsed
|
|
8380
|
+
? ` ${t.profile.spanPartial(n(report.span.calls), n(totalParsed))}`
|
|
8381
|
+
: '';
|
|
8382
|
+
console.log(` ${c.dim(wrap(`${t.profile.spanLine(dayOf(report.span.fromMs), dayOf(report.span.toMs), spanDays(report.span.fromMs, report.span.toMs))}${partial}`, 74, ' '))}`);
|
|
8383
|
+
}
|
|
8384
|
+
// How many files this report covers, when it covers more than one: a total
|
|
8385
|
+
// over "the logs" that silently skipped one is wrong by an unknown amount.
|
|
8386
|
+
if (logFiles.length > 1) {
|
|
8387
|
+
console.log(` ${c.dim(wrap(t.profile.readFiles(logFiles.length, path), 74, ' '))}`);
|
|
7479
8388
|
}
|
|
7480
8389
|
/**
|
|
7481
|
-
*
|
|
7482
|
-
*
|
|
7483
|
-
*
|
|
8390
|
+
* A doubled bill, said before anything is believed.
|
|
8391
|
+
*
|
|
8392
|
+
* Reading a directory of rotated logs makes double-counting easy — a log
|
|
8393
|
+
* exported twice, an overlapping export, a copy left in the folder — and
|
|
8394
|
+
* the total then reads high with nothing else able to see it. Only counted
|
|
8395
|
+
* over records with a clock, where an identical line is a claim worth
|
|
8396
|
+
* making. It states the count and the money and stops: whether it is a
|
|
8397
|
+
* double export or a genuinely busy millisecond is the reader's to know.
|
|
7484
8398
|
*/
|
|
7485
|
-
if (report.
|
|
7486
|
-
|
|
7487
|
-
if (moved.length > 0) {
|
|
7488
|
-
console.log();
|
|
7489
|
-
console.log(sectionHeading(t.profile.mixDriftHeading()));
|
|
7490
|
-
for (const m of moved.slice(0, 3)) {
|
|
7491
|
-
console.log();
|
|
7492
|
-
console.log(` ${c.yellow('!')} ${c.bold(wrap(t.profile.mixDriftLine(m.model, pct(m.firstShare), pct(m.lastShare), n(report.modelMixDrift.firstDays), n(report.modelMixDrift.lastDays), formatUsd(m.lastUsd)), 74, ' '))}`);
|
|
7493
|
-
}
|
|
7494
|
-
console.log(` ${c.dim(wrap(t.profile.mixDriftNote(), 74, ' '))}`);
|
|
7495
|
-
}
|
|
8399
|
+
if (report.duplicateLines.count > 0) {
|
|
8400
|
+
console.log(` ${c.yellow('!')} ${c.dim(wrap(t.profile.duplicateLines(report.duplicateLines.count, formatUsd(report.duplicateLines.usd)), 74, ' '))}`);
|
|
7496
8401
|
}
|
|
7497
8402
|
/**
|
|
7498
|
-
* The
|
|
7499
|
-
*
|
|
7500
|
-
*
|
|
7501
|
-
*
|
|
7502
|
-
*
|
|
7503
|
-
* step repeating, a loop. Loud, because the money bought nothing, and
|
|
7504
|
-
* hedged, because this reads counts and cannot see content: the sentence
|
|
7505
|
-
* says the pattern is *usually* a retry, never that it is one.
|
|
8403
|
+
* The window, said before any figure is trusted as "the log": everything
|
|
8404
|
+
* below describes a slice, and a slice presented as the whole is a figure
|
|
8405
|
+
* attributed to something it does not describe. The undated count is loud —
|
|
8406
|
+
* those calls' spend is in the log and not in this report, so the window's
|
|
8407
|
+
* figures are a floor on the period, and only this line says so.
|
|
7506
8408
|
*/
|
|
7507
|
-
if (report.
|
|
7508
|
-
console.log();
|
|
7509
|
-
|
|
7510
|
-
|
|
7511
|
-
|
|
7512
|
-
|
|
7513
|
-
console.log(` ${c.yellow(
|
|
7514
|
-
console.log(` ${c.dim(wrap(t.profile.repeatsAdvice(), 74, ' '))}`);
|
|
8409
|
+
if (report.timeWindow !== null) {
|
|
8410
|
+
console.log(` ${c.dim(wrap(t.profile.windowLine(stringFlag(args, 'since') ?? '—', stringFlag(args, 'until') ?? '—'), 74, ' '))}`);
|
|
8411
|
+
if (relativeWindow) {
|
|
8412
|
+
console.log(` ${c.dim(wrap(t.profile.windowRelative(), 74, ' '))}`);
|
|
8413
|
+
}
|
|
8414
|
+
if (report.timeWindow.undatedExcluded > 0) {
|
|
8415
|
+
console.log(` ${c.yellow(wrap(t.profile.windowUndated(report.timeWindow.undatedExcluded), 74, ' '))}`);
|
|
7515
8416
|
}
|
|
7516
8417
|
}
|
|
8418
|
+
console.log();
|
|
8419
|
+
// Every part, including the zero ones. A row missing because it was zero reads
|
|
8420
|
+
// as a row somebody forgot, and "you are not caching at all" is a finding.
|
|
8421
|
+
for (const [name, usd, share, tokens] of parts) {
|
|
8422
|
+
console.log(` ${bar(share)} ${t.profile.part(name, formatUsd(usd), pct(share), n(tokens))}`);
|
|
8423
|
+
}
|
|
7517
8424
|
/**
|
|
7518
|
-
*
|
|
7519
|
-
* a bill that is waste without a counterpart. Paid in full, frequently
|
|
7520
|
-
* retried and billed again, and the truncated attempt bought nothing.
|
|
8425
|
+
* The line the command exists for: which part of the bill to argue with.
|
|
7521
8426
|
*
|
|
7522
|
-
*
|
|
7523
|
-
*
|
|
7524
|
-
*
|
|
7525
|
-
* of health on a question the log never asked.
|
|
8427
|
+
* When output is both the biggest part and over half, the two sentences say the
|
|
8428
|
+
* same thing and the second says more — so only the second prints. Reporting a
|
|
8429
|
+
* fact twice in adjacent lines reads as a bug, and it was one.
|
|
7526
8430
|
*/
|
|
7527
|
-
|
|
7528
|
-
|
|
7529
|
-
|
|
7530
|
-
|
|
7531
|
-
|
|
7532
|
-
* half the total hides. A 40% truncation rate is a max_tokens setting
|
|
7533
|
-
* that is simply wrong; 1% is a long tail, and the two call for opposite
|
|
7534
|
-
* responses.
|
|
7535
|
-
*
|
|
7536
|
-
* The rate is over calls that **recorded a stop reason**, never over all
|
|
7537
|
-
* calls: a workload that logs the field on half its traffic must not be
|
|
7538
|
-
* reported as though the unmeasured half completed. Both numbers print,
|
|
7539
|
-
* so the denominator is visible rather than implied.
|
|
7540
|
-
*/
|
|
7541
|
-
const truncatedLabels = report.byLabel
|
|
7542
|
-
.filter((row) => row.breakdown.truncatedCalls > 0)
|
|
7543
|
-
.sort((a, b) => b.breakdown.truncatedOutputUsd - a.breakdown.truncatedOutputUsd);
|
|
7544
|
-
if (truncatedLabels.length > 0 && report.byLabel.length > 1) {
|
|
7545
|
-
for (const row of truncatedLabels.slice(0, 3)) {
|
|
7546
|
-
const name = row.label === UNLABELLED ? t.profile.unlabelled() : row.label;
|
|
7547
|
-
console.log(` ${c.dim(wrap(t.profile.truncatedBy(name, n(row.breakdown.truncatedCalls), n(row.breakdown.stopReasonCalls), pct(row.breakdown.truncatedCalls / row.breakdown.stopReasonCalls), formatUsd(row.breakdown.truncatedOutputUsd)), 74, ' '))}`);
|
|
7548
|
-
}
|
|
7549
|
-
}
|
|
7550
|
-
/**
|
|
7551
|
-
* The ceiling the completed answers actually needed, when the output
|
|
7552
|
-
* shapes measured it: "95% of the answers that finished fit within N
|
|
7553
|
-
* tokens" is the number a max_tokens cap wants, and it sits next to the
|
|
7554
|
-
* evidence that the current cap is too low. Measured on these calls,
|
|
7555
|
-
* promised for nothing.
|
|
7556
|
-
*/
|
|
7557
|
-
const ceiling = report.outputShapes.find((shape) => shape.p95WithinTokens !== null);
|
|
7558
|
-
if (ceiling !== undefined) {
|
|
7559
|
-
console.log(` ${c.dim(wrap(t.profile.truncatedCeiling(n(ceiling.p95WithinTokens)), 74, ' '))}`);
|
|
7560
|
-
}
|
|
7561
|
-
/**
|
|
7562
|
-
* The "billed again" half, measured. The sentence above has always said
|
|
7563
|
-
* truncated answers are frequently retried; this is the count and the
|
|
7564
|
-
* money, when the log can carry it — a pattern, never a certainty, and
|
|
7565
|
-
* attributed to the truncated call's slice, where the ceiling that
|
|
7566
|
-
* caused it lives.
|
|
7567
|
-
*/
|
|
7568
|
-
for (const row of report.truncationRetries.slice(0, 3)) {
|
|
7569
|
-
const name = row.label === UNLABELLED ? t.profile.unlabelled() : row.label;
|
|
7570
|
-
console.log(` ${c.yellow('!')} ${c.bold(wrap(t.profile.truncationRetryLine(name, row.modelName, n(row.retried), n(row.truncatedCalls), n(Math.round(row.withinMs / 1000)), formatUsd(row.wastedUsd), formatUsd(row.retryUsd)), 74, ' '))}`);
|
|
7571
|
-
}
|
|
7572
|
-
if (report.truncationRetries.length > 0) {
|
|
7573
|
-
console.log(` ${c.dim(wrap(t.profile.truncationRetryNote(), 74, ' '))}`);
|
|
7574
|
-
}
|
|
8431
|
+
const [biggestName, , biggestShare] = parts.reduce((a, b) => (b[1] > a[1] ? b : a));
|
|
8432
|
+
const outputDominates = shares.output > 0.5;
|
|
8433
|
+
console.log();
|
|
8434
|
+
if (outputDominates) {
|
|
8435
|
+
console.log(` ${c.bold(wrap(t.profile.outputDominates(pct(shares.output)), 74, ' '))}`);
|
|
7575
8436
|
}
|
|
7576
|
-
else
|
|
7577
|
-
console.log();
|
|
7578
|
-
console.log(` ${c.dim(wrap(t.profile.truncatedNotRecorded(), 74, ' '))}`);
|
|
8437
|
+
else {
|
|
8438
|
+
console.log(` ${c.bold(t.profile.biggestPart(biggestName, pct(biggestShare)))}`);
|
|
7579
8439
|
}
|
|
7580
8440
|
/**
|
|
7581
|
-
*
|
|
8441
|
+
* The most expensive day, with a suspect attached.
|
|
7582
8442
|
*
|
|
7583
|
-
*
|
|
7584
|
-
*
|
|
7585
|
-
*
|
|
7586
|
-
*
|
|
7587
|
-
*
|
|
7588
|
-
* reader to judge comparability before judging the trend.
|
|
8443
|
+
* The shape of a bill over time is the finding the total hides: a steady $3 a
|
|
8444
|
+
* day and a quiet week broken by one $40 spike sum to the same number and call
|
|
8445
|
+
* for opposite responses. Rendered against the **median** day — a mean would
|
|
8446
|
+
* let the spike inflate its own yardstick — and loud only when it clears twice
|
|
8447
|
+
* the median, a threshold stated in the sentence rather than hidden in code.
|
|
7589
8448
|
*/
|
|
7590
|
-
if (
|
|
7591
|
-
|
|
7592
|
-
|
|
7593
|
-
if (
|
|
7594
|
-
|
|
8449
|
+
if (report.spendByDay.length >= 2) {
|
|
8450
|
+
const medianUsd = median(report.spendByDay.map((d) => d.usd));
|
|
8451
|
+
const peak = report.spendByDay.reduce((a, b) => (b.usd > a.usd ? b : a));
|
|
8452
|
+
if (medianUsd > 0) {
|
|
8453
|
+
const ratio = (peak.usd / medianUsd).toFixed(1);
|
|
8454
|
+
const line = t.profile.dayPeak(peak.day, formatUsd(peak.usd), ratio);
|
|
8455
|
+
const labelClause = peak.topLabel !== null && report.byLabel.length > 1
|
|
8456
|
+
? ` ${t.profile.dayPeakLabel(peak.topLabel === UNLABELLED ? t.profile.unlabelled() : peak.topLabel, formatUsd(peak.topLabelUsd))}`
|
|
8457
|
+
: '';
|
|
8458
|
+
const loud = peak.usd > 2 * medianUsd;
|
|
8459
|
+
const text = wrap(`${line}${labelClause}`, 74, ' ');
|
|
8460
|
+
console.log(` ${loud ? c.yellow(text) : c.dim(text)}`);
|
|
7595
8461
|
}
|
|
7596
|
-
|
|
7597
|
-
|
|
7598
|
-
|
|
7599
|
-
|
|
7600
|
-
|
|
7601
|
-
|
|
7602
|
-
|
|
7603
|
-
|
|
7604
|
-
|
|
7605
|
-
|
|
7606
|
-
|
|
7607
|
-
|
|
7608
|
-
|
|
7609
|
-
|
|
7610
|
-
|
|
7611
|
-
|
|
7612
|
-
|
|
7613
|
-
|
|
7614
|
-
|
|
7615
|
-
|
|
7616
|
-
|
|
7617
|
-
|
|
7618
|
-
|
|
7619
|
-
|
|
7620
|
-
|
|
7621
|
-
console.log(` ${c.dim(t.profile.andMoreLabels(labelDrivers.length - 5))}`);
|
|
7622
|
-
}
|
|
7623
|
-
/**
|
|
7624
|
-
* The same change, by model — where the mix moved. The label rows cannot
|
|
7625
|
-
* show it: a workload that kept its name and switched from Haiku to Opus
|
|
7626
|
-
* reads as "chat grew", and the reason is the model. Only printed when
|
|
7627
|
-
* more than one model is involved; with one model on both sides, this
|
|
7628
|
-
* section restates the totals line and says nothing new.
|
|
7629
|
-
*/
|
|
7630
|
-
const modelsInvolved = new Set([
|
|
7631
|
-
...previous.byModel.map((r) => r.model),
|
|
7632
|
-
...report.byModel.map((r) => r.model),
|
|
7633
|
-
]);
|
|
7634
|
-
if (modelDrivers.length > 0 && modelsInvolved.size > 1) {
|
|
7635
|
-
console.log();
|
|
7636
|
-
console.log(` ${c.dim(t.profile.againstByModel())}`);
|
|
7637
|
-
for (const d of modelDrivers.slice(0, 3)) {
|
|
7638
|
-
const line = driverLine(d, d.key);
|
|
7639
|
-
console.log(` ${d.delta > 0 ? c.yellow(line) : c.dim(line)}`);
|
|
7640
|
-
}
|
|
7641
|
-
}
|
|
7642
|
-
/**
|
|
7643
|
-
* What the comparison stopped being able to see.
|
|
7644
|
-
*
|
|
7645
|
-
* Every figure above is dollars, and dollars cannot tell a finding that
|
|
7646
|
-
* was fixed from a finding whose field the log stopped recording — both
|
|
7647
|
-
* are silence. This is the only section that can, so it is loud: a
|
|
7648
|
-
* collapse in coverage invalidates whichever findings depended on it,
|
|
7649
|
-
* and reading the drop as good news is the specific mistake it exists
|
|
7650
|
-
* to prevent.
|
|
7651
|
-
*/
|
|
7652
|
-
const drifts = coverageDrift(previous.fieldCoverage, report.fieldCoverage);
|
|
7653
|
-
if (drifts.length > 0) {
|
|
7654
|
-
console.log();
|
|
7655
|
-
for (const drift of drifts) {
|
|
7656
|
-
const line = t.profile.coverageDrift(t.profile.coverageField(drift.field), pct(drift.was), pct(drift.now));
|
|
7657
|
-
console.log(drift.delta < 0
|
|
7658
|
-
? ` ${c.yellow('!')} ${c.bold(wrap(line, 74, ' '))}`
|
|
7659
|
-
: ` ${c.dim(wrap(line, 74, ' '))}`);
|
|
7660
|
-
/**
|
|
7661
|
-
* Which findings went with it, named. "Some findings are silent" is
|
|
7662
|
-
* not something a reader can act on; knowing that conversation
|
|
7663
|
-
* growth and the cache-TTL fit are now silence rather than absence
|
|
7664
|
-
* tells them exactly which sections of this report to distrust.
|
|
7665
|
-
*/
|
|
7666
|
-
if (drift.delta < 0) {
|
|
7667
|
-
const silenced = t.profile.coverageSilenced(drift.field);
|
|
7668
|
-
if (silenced !== '')
|
|
7669
|
-
console.log(` ${c.dim(wrap(silenced, 72, ' '))}`);
|
|
7670
|
-
}
|
|
7671
|
-
}
|
|
7672
|
-
if (drifts.some((d) => d.delta < 0)) {
|
|
7673
|
-
console.log(` ${c.dim(wrap(t.profile.coverageDriftWhy(), 74, ' '))}`);
|
|
7674
|
-
}
|
|
7675
|
-
}
|
|
8462
|
+
}
|
|
8463
|
+
/**
|
|
8464
|
+
* The shape of the day, and what it says about batching.
|
|
8465
|
+
*
|
|
8466
|
+
* Spend packed into the hours a country is awake is interactive traffic
|
|
8467
|
+
* somebody is waiting on; spend spread evenly across twenty-four is
|
|
8468
|
+
* background work — and background work is what the Batch API halves. The
|
|
8469
|
+
* measure is exact and needs no threshold to state: the **fewest hours that
|
|
8470
|
+
* hold 80% of the spend**. Two or three means concentrated; sixteen means
|
|
8471
|
+
* flat.
|
|
8472
|
+
*
|
|
8473
|
+
* It says what the shape is and stops. Whether a workload can wait is a
|
|
8474
|
+
* product decision Trazum cannot make from counts, so the sentence names
|
|
8475
|
+
* the lever and never claims the saving — the batch figure the levers
|
|
8476
|
+
* section already prints is the one with money attached.
|
|
8477
|
+
*/
|
|
8478
|
+
if (report.spendByHour.length >= 4 && report.total.totalUsd > 0) {
|
|
8479
|
+
const sorted = [...report.spendByHour].sort((a, b) => b.usd - a.usd);
|
|
8480
|
+
let covered = 0;
|
|
8481
|
+
let hoursForMost = 0;
|
|
8482
|
+
for (const hour of sorted) {
|
|
8483
|
+
covered += hour.usd;
|
|
8484
|
+
hoursForMost += 1;
|
|
8485
|
+
if (covered >= 0.8 * report.total.totalUsd)
|
|
8486
|
+
break;
|
|
7676
8487
|
}
|
|
8488
|
+
const busiest = sorted
|
|
8489
|
+
.slice(0, hoursForMost)
|
|
8490
|
+
.map((hour) => hour.hour)
|
|
8491
|
+
.sort((a, b) => a - b)
|
|
8492
|
+
.map((hour) => `${String(hour).padStart(2, '0')}:00`)
|
|
8493
|
+
.join(', ');
|
|
8494
|
+
console.log();
|
|
8495
|
+
console.log(` ${c.dim(wrap(hoursForMost <= 8 ? t.profile.hoursConcentrated(n(hoursForMost), busiest) : t.profile.hoursFlat(n(hoursForMost)), 74, ' '))}`);
|
|
7677
8496
|
}
|
|
8497
|
+
reportCacheVerdict(report, t);
|
|
8498
|
+
await reportLabelsBudgetsAndCache(report, config, pricing, t);
|
|
8499
|
+
reportLevers(report, pricing, t);
|
|
8500
|
+
reportWhatIf(whatIf, report, t);
|
|
8501
|
+
reportConversationHistory(report, t);
|
|
8502
|
+
reportOutputShape(report, t);
|
|
8503
|
+
reportInputShape(report, t);
|
|
8504
|
+
reportContextPressure(pressures, t);
|
|
8505
|
+
reportMixDrift(report, t);
|
|
8506
|
+
reportRepeatedCalls(report, t);
|
|
8507
|
+
reportAgainstPrevious({
|
|
8508
|
+
report, previous, againstOverlap, labelDrivers, modelDrivers, gates, now, t,
|
|
8509
|
+
});
|
|
7678
8510
|
for (const [heading, rows] of [
|
|
7679
8511
|
[t.profile.byLabelHeading(), report.byLabel.map((r) => [r.label === UNLABELLED ? t.profile.unlabelled() : r.label, r.breakdown])],
|
|
7680
8512
|
[t.profile.byModelHeading(), report.byModel.map((r) => [r.model, r.breakdown])],
|
|
@@ -7688,168 +8520,15 @@ async function commandProfile(args, config, configDir, pricing, t) {
|
|
|
7688
8520
|
console.log(` ${bar(share)} ${t.profile.row(name, formatUsd(breakdown.totalUsd), pct(share), t.profile.calls(breakdown.calls))}`);
|
|
7689
8521
|
}
|
|
7690
8522
|
}
|
|
7691
|
-
|
|
7692
|
-
* What this log cannot answer, and what would fix it.
|
|
7693
|
-
*
|
|
7694
|
-
* Every finding past the totals needs a field the format does not require,
|
|
7695
|
-
* and a reader who never adds them sees a report quietly missing half of
|
|
7696
|
-
* itself — with no way to tell "nothing to report" from "nothing recorded".
|
|
7697
|
-
* Named with counts rather than booleans: twelve labelled records out of
|
|
7698
|
-
* forty thousand is not a labelled log, and a boolean would call it one.
|
|
7699
|
-
*
|
|
7700
|
-
* Only fields that are actually missing are listed. A complete log gets no
|
|
7701
|
-
* section at all, because a paragraph of things that are fine is the
|
|
7702
|
-
* paragraph readers learn to skip.
|
|
7703
|
-
*/
|
|
7704
|
-
/**
|
|
7705
|
-
* Outcomes — the counterpart, where somebody recorded one.
|
|
7706
|
-
*
|
|
7707
|
-
* Printed above the coverage section rather than below it, because when this
|
|
7708
|
-
* section is present it is the most valuable thing on the page: every other
|
|
7709
|
-
* figure in this report is a cost, and this is the only one that says what
|
|
7710
|
-
* the money bought.
|
|
7711
|
-
*
|
|
7712
|
-
* Silent when nothing recorded an outcome. The coverage section below
|
|
7713
|
-
* already names the missing field and what it would unlock, and printing an
|
|
7714
|
-
* empty Outcomes heading above it would be the same sentence twice.
|
|
7715
|
-
*/
|
|
7716
|
-
{
|
|
7717
|
-
const outcomes = outcomeReport(report.outcomeTally, config.outcomes ?? null);
|
|
7718
|
-
if (outcomes.coverage.recorded > 0) {
|
|
7719
|
-
console.log();
|
|
7720
|
-
console.log(sectionHeading(t.profile.outcomeHeading()));
|
|
7721
|
-
const col = t.profile.outcomeColumns;
|
|
7722
|
-
const rows = [...outcomes.slices, ...outcomes.undeclared].map((slice) => ({
|
|
7723
|
-
value: slice.value,
|
|
7724
|
-
verdict: slice.verdict === 'success'
|
|
7725
|
-
? t.profile.verdictSuccess()
|
|
7726
|
-
: slice.verdict === 'undeclared'
|
|
7727
|
-
? t.profile.verdictUndeclared()
|
|
7728
|
-
: t.profile.verdictOther(),
|
|
7729
|
-
calls: n(slice.calls),
|
|
7730
|
-
spend: formatUsd(slice.usd),
|
|
7731
|
-
}));
|
|
7732
|
-
const w = {
|
|
7733
|
-
value: Math.max(...rows.map((r) => r.value.length), col.outcome.length),
|
|
7734
|
-
verdict: Math.max(...rows.map((r) => r.verdict.length), 0),
|
|
7735
|
-
calls: Math.max(...rows.map((r) => r.calls.length), col.calls.length),
|
|
7736
|
-
spend: Math.max(...rows.map((r) => r.spend.length), col.spend.length),
|
|
7737
|
-
};
|
|
7738
|
-
console.log(c.dim(` ${col.outcome.padEnd(w.value)} ${''.padEnd(w.verdict)} ` +
|
|
7739
|
-
`${col.calls.padStart(w.calls)} ${col.spend.padStart(w.spend)}`));
|
|
7740
|
-
for (const row of rows) {
|
|
7741
|
-
const tint = row.verdict === t.profile.verdictUndeclared()
|
|
7742
|
-
? c.yellow
|
|
7743
|
-
: row.verdict === t.profile.verdictSuccess()
|
|
7744
|
-
? c.green
|
|
7745
|
-
: c.dim;
|
|
7746
|
-
console.log(` ${row.value.padEnd(w.value)} ${tint(row.verdict.padEnd(w.verdict))} ` +
|
|
7747
|
-
`${row.calls.padStart(w.calls)} ${row.spend.padStart(w.spend)}`);
|
|
7748
|
-
}
|
|
7749
|
-
console.log();
|
|
7750
|
-
if (outcomes.successShareOfRecordedUsd !== null) {
|
|
7751
|
-
const declaredUsd = outcomes.slices.reduce((sum, slice) => sum + slice.usd, 0);
|
|
7752
|
-
console.log(` ${wrap(t.profile.outcomeRate(pct(outcomes.successShareOfRecordedUsd), formatUsd(declaredUsd)), 74, ' ')}`);
|
|
7753
|
-
}
|
|
7754
|
-
else if (outcomes.noRate !== null) {
|
|
7755
|
-
console.log(` ${c.dim(wrap(t.profile.outcomeNoRate(outcomes.noRate), 74, ' '))}`);
|
|
7756
|
-
}
|
|
7757
|
-
// What the rate does not cover, every time it is printed. A rate over a
|
|
7758
|
-
// twelfth of the bill is a rate about a twelfth of the bill.
|
|
7759
|
-
if (outcomes.coverage.unrecordedUsd > 0 && report.total.totalUsd > 0) {
|
|
7760
|
-
console.log(` ${c.yellow('!')} ${wrap(t.profile.outcomeUnrecorded(pct(outcomes.coverage.unrecordedUsd / report.total.totalUsd), formatUsd(outcomes.coverage.unrecordedUsd)), 74, ' ')}`);
|
|
7761
|
-
}
|
|
7762
|
-
/**
|
|
7763
|
-
* What an outcome costs, per workload — the finding a total cannot make.
|
|
7764
|
-
*
|
|
7765
|
-
* Printed only when at least one workload has enough recorded outcomes
|
|
7766
|
-
* to say something, and every row that cannot state a figure says which
|
|
7767
|
-
* of the five reasons applies rather than showing a blank.
|
|
7768
|
-
*/
|
|
7769
|
-
const ranking = rankPerOutcome(report.outcomeTallyByLabel.map((entry) => ({
|
|
7770
|
-
key: entry.label,
|
|
7771
|
-
calls: entry.calls,
|
|
7772
|
-
totalUsd: entry.totalUsd,
|
|
7773
|
-
tally: entry.tally,
|
|
7774
|
-
})), config.outcomes ?? null);
|
|
7775
|
-
if (ranking.byCall.length > 0) {
|
|
7776
|
-
console.log();
|
|
7777
|
-
console.log(sectionHeading(t.profile.perOutcomeHeading()));
|
|
7778
|
-
const pcol = t.profile.perOutcomeColumns;
|
|
7779
|
-
const prows = ranking.byCall.map((slice) => ({
|
|
7780
|
-
key: slice.key,
|
|
7781
|
-
perCall: formatUsd(slice.usdPerCall),
|
|
7782
|
-
perOutcome: slice.per.usdPerSuccess !== null
|
|
7783
|
-
? formatUsd(slice.per.usdPerSuccess)
|
|
7784
|
-
: t.profile.perOutcomeWithheld(slice.per.withheld ?? 'no-vocabulary', n(slice.per.successes), pct(slice.per.coverage)),
|
|
7785
|
-
recorded: pct(slice.per.coverage),
|
|
7786
|
-
}));
|
|
7787
|
-
const pw = {
|
|
7788
|
-
key: Math.max(...prows.map((r) => r.key.length), pcol.workload.length),
|
|
7789
|
-
perCall: Math.max(...prows.map((r) => r.perCall.length), pcol.perCall.length),
|
|
7790
|
-
perOutcome: Math.max(...prows.map((r) => r.perOutcome.length), pcol.perOutcome.length),
|
|
7791
|
-
recorded: Math.max(...prows.map((r) => r.recorded.length), pcol.recorded.length),
|
|
7792
|
-
};
|
|
7793
|
-
console.log(c.dim(` ${pcol.workload.padEnd(pw.key)} ${pcol.perCall.padStart(pw.perCall)} ` +
|
|
7794
|
-
`${pcol.perOutcome.padStart(pw.perOutcome)} ${pcol.recorded.padStart(pw.recorded)}`));
|
|
7795
|
-
for (const row of prows) {
|
|
7796
|
-
console.log(` ${row.key.padEnd(pw.key)} ${row.perCall.padStart(pw.perCall)} ` +
|
|
7797
|
-
`${row.perOutcome.padStart(pw.perOutcome)} ${c.dim(row.recorded.padStart(pw.recorded))}`);
|
|
7798
|
-
}
|
|
7799
|
-
console.log();
|
|
7800
|
-
console.log(` ${c.dim(wrap(t.profile.perOutcomeNumerator(), 74, ' '))}`);
|
|
7801
|
-
// The disagreement between the two orders, which is itself the finding.
|
|
7802
|
-
if (ranking.disagreements.length > 0) {
|
|
7803
|
-
console.log();
|
|
7804
|
-
console.log(` ${c.dim(wrap(t.profile.perOutcomeBothOrders(), 74, ' '))}`);
|
|
7805
|
-
for (const d of ranking.disagreements) {
|
|
7806
|
-
console.log(` ${c.yellow('→')} ${wrap(t.profile.perOutcomeDisagreement(d.slice.key, n(d.callRank + 1), n(d.outcomeRank + 1)), 74, ' ')}`);
|
|
7807
|
-
}
|
|
7808
|
-
}
|
|
7809
|
-
}
|
|
7810
|
-
if (outcomes.undeclared.length > 0) {
|
|
7811
|
-
console.log(` ${c.yellow('!')} ${wrap(t.profile.outcomeUndeclared(outcomes.undeclared.map((s) => s.value).join(', ')), 74, ' ')}`);
|
|
7812
|
-
}
|
|
7813
|
-
}
|
|
7814
|
-
}
|
|
7815
|
-
const coverage = report.fieldCoverage;
|
|
7816
|
-
if (coverage.parsed > 0) {
|
|
7817
|
-
const missing = [];
|
|
7818
|
-
const partial = (seen) => `${n(seen)}/${n(coverage.parsed)}`;
|
|
7819
|
-
if (coverage.label < coverage.parsed) {
|
|
7820
|
-
missing.push(t.profile.needsLabel(partial(coverage.label)));
|
|
7821
|
-
}
|
|
7822
|
-
if (coverage.session < coverage.parsed) {
|
|
7823
|
-
missing.push(t.profile.needsSession(partial(coverage.session)));
|
|
7824
|
-
}
|
|
7825
|
-
/**
|
|
7826
|
-
* Listed first among the missing when it is missing entirely, because it
|
|
7827
|
-
* is the one field that changes what every other figure here *means*. The
|
|
7828
|
-
* rest sharpen a cost; this one gives it a counterpart.
|
|
7829
|
-
*/
|
|
7830
|
-
if (coverage.outcome < coverage.parsed) {
|
|
7831
|
-
missing.push(t.profile.needsOutcome(partial(coverage.outcome)));
|
|
7832
|
-
}
|
|
7833
|
-
if (coverage.ts < coverage.parsed) {
|
|
7834
|
-
missing.push(t.profile.needsTs(partial(coverage.ts)));
|
|
7835
|
-
}
|
|
7836
|
-
if (coverage.stopReason < coverage.parsed) {
|
|
7837
|
-
missing.push(t.profile.needsStopReason(partial(coverage.stopReason)));
|
|
7838
|
-
}
|
|
7839
|
-
if (coverage.cacheWrites > 0 && coverage.cacheTtl < coverage.cacheWrites) {
|
|
7840
|
-
missing.push(t.profile.needsCacheTtl(`${n(coverage.cacheTtl)}/${n(coverage.cacheWrites)}`));
|
|
7841
|
-
}
|
|
7842
|
-
if (missing.length > 0) {
|
|
7843
|
-
console.log();
|
|
7844
|
-
console.log(sectionHeading(t.profile.coverageHeading()));
|
|
7845
|
-
for (const line of missing)
|
|
7846
|
-
console.log(` ${c.dim(wrap(line, 74, ' '))}`);
|
|
7847
|
-
}
|
|
7848
|
-
}
|
|
8523
|
+
reportOutcomesAndCoverage(report, config, t);
|
|
7849
8524
|
reportProfileGaps(report, t, n, pricingStale);
|
|
7850
|
-
|
|
7851
|
-
|
|
7852
|
-
|
|
8525
|
+
gates = await runProfileGates({
|
|
8526
|
+
args, config, configDir, report, previous, againstDelta, windowed, now, pricing, t,
|
|
8527
|
+
});
|
|
8528
|
+
await writeProfileSideFiles({
|
|
8529
|
+
args, path, report, previous, pressures, labelDrivers, modelDrivers, againstOverlap,
|
|
8530
|
+
whatIf, pricingStale, windowed, gates, pricing, t,
|
|
8531
|
+
});
|
|
7853
8532
|
}
|
|
7854
8533
|
/**
|
|
7855
8534
|
* What the profile could not account for, said out loud.
|
|
@@ -7896,12 +8575,102 @@ function reportProfileGaps(report, t, n, stalePricing = null) {
|
|
|
7896
8575
|
* exactly as `prune` does. A command that can spend somebody's money without
|
|
7897
8576
|
* telling them first is a command they stop trusting.
|
|
7898
8577
|
*/
|
|
8578
|
+
/**
|
|
8579
|
+
* The routing measurement, without the text that produced it.
|
|
8580
|
+
*
|
|
8581
|
+
* `route --json` printed `{ slice, evaluation }` — the whole `EvalReport`,
|
|
8582
|
+
* whose `cases[]` carries `input` (the caller's own case text), `baseline[0]`,
|
|
8583
|
+
* `baseline[1]` and `optimized` (three model answers). It carried no
|
|
8584
|
+
* `schemaVersion` either.
|
|
8585
|
+
*
|
|
8586
|
+
* `docs/json-output.md` opens by promising that **nothing here carries a
|
|
8587
|
+
* session key or prompt text**, and this is the document most likely to be
|
|
8588
|
+
* piped into a dashboard, pasted into a ticket or dropped into a browser. A
|
|
8589
|
+
* tool whose headline is that prompts never leave the machine cannot have its
|
|
8590
|
+
* most portable artefact be the one that carries them.
|
|
8591
|
+
*
|
|
8592
|
+
* The per-case *shape* survives: the two similarity scores are the evidence
|
|
8593
|
+
* behind the verdict, and a reader checking a borderline call needs them. Only
|
|
8594
|
+
* the strings go. Anyone who wants the answers has the human-readable output,
|
|
8595
|
+
* which never left the terminal.
|
|
8596
|
+
*
|
|
8597
|
+
* A named function rather than an inline object so the stripping is a thing a
|
|
8598
|
+
* test can call without spending a provider call to reach it.
|
|
8599
|
+
*/
|
|
8600
|
+
export function routeDocument(slice, result) {
|
|
8601
|
+
return {
|
|
8602
|
+
schemaVersion: 1,
|
|
8603
|
+
slice,
|
|
8604
|
+
evaluation: {
|
|
8605
|
+
provider: result.provider,
|
|
8606
|
+
model: result.model,
|
|
8607
|
+
candidateModel: result.candidateModel,
|
|
8608
|
+
verdict: result.verdict,
|
|
8609
|
+
selfAgreement: result.selfAgreement,
|
|
8610
|
+
crossAgreement: result.crossAgreement,
|
|
8611
|
+
callsMade: result.callsMade,
|
|
8612
|
+
cases: result.cases.map((one) => ({
|
|
8613
|
+
selfSimilarity: one.selfSimilarity,
|
|
8614
|
+
crossSimilarity: one.crossSimilarity,
|
|
8615
|
+
})),
|
|
8616
|
+
},
|
|
8617
|
+
};
|
|
8618
|
+
}
|
|
8619
|
+
/**
|
|
8620
|
+
* The example-pruning measurement, likewise.
|
|
8621
|
+
*
|
|
8622
|
+
* `ExampleContribution.text` is *"the example itself, so a report can quote its
|
|
8623
|
+
* first line"* — true of the terminal rendering and wrong of a document that
|
|
8624
|
+
* promises to carry no prompt text. A few-shot example **is** prompt text, and
|
|
8625
|
+
* this was the second command emitting it.
|
|
8626
|
+
*
|
|
8627
|
+
* `index` and `tokens` say which example without quoting it, which is what a
|
|
8628
|
+
* machine reading this needs: it already has the prompt.
|
|
8629
|
+
*/
|
|
8630
|
+
export function pruneDocument(report) {
|
|
8631
|
+
return {
|
|
8632
|
+
schemaVersion: 1,
|
|
8633
|
+
provider: report.provider,
|
|
8634
|
+
model: report.model,
|
|
8635
|
+
selfAgreement: report.selfAgreement,
|
|
8636
|
+
recoverableTokens: report.recoverableTokens,
|
|
8637
|
+
callsMade: report.callsMade,
|
|
8638
|
+
contributions: report.contributions.map((one) => ({
|
|
8639
|
+
index: one.index,
|
|
8640
|
+
tokens: one.tokens,
|
|
8641
|
+
agreementWithout: one.agreementWithout,
|
|
8642
|
+
verdict: one.verdict,
|
|
8643
|
+
})),
|
|
8644
|
+
};
|
|
8645
|
+
}
|
|
7899
8646
|
async function commandRoute(args, pricing, t) {
|
|
8647
|
+
/**
|
|
8648
|
+
* Under `--json`, stdout carries the document and nothing else.
|
|
8649
|
+
*
|
|
8650
|
+
* It carried the human preamble too — the slice it picked, the call count,
|
|
8651
|
+
* the "nothing has been spent yet" line — straight into the same stream as
|
|
8652
|
+
* the JSON, so `route --json > verdict.json` wrote a file no parser would
|
|
8653
|
+
* read. Found by running the command and feeding the file to the bridge
|
|
8654
|
+
* that reads it, which is the one thing the pure-function guards could not
|
|
8655
|
+
* do: `interchange.test.js` names `route` among the commands it cannot
|
|
8656
|
+
* drive, so nothing checked that its `--json` output was a document.
|
|
8657
|
+
*
|
|
8658
|
+
* The lines go to stderr rather than away. A reader running this in a
|
|
8659
|
+
* terminal still needs to see which slice was picked and what it will cost
|
|
8660
|
+
* before it costs it; a pipe needs the document. stderr gives both.
|
|
8661
|
+
*/
|
|
8662
|
+
const asJson = boolFlag(args, 'json');
|
|
8663
|
+
const say = (line = '') => {
|
|
8664
|
+
if (asJson)
|
|
8665
|
+
console.error(line);
|
|
8666
|
+
else
|
|
8667
|
+
console.log(line);
|
|
8668
|
+
};
|
|
7900
8669
|
const path = args.positional[0];
|
|
7901
8670
|
if (path === undefined) {
|
|
7902
|
-
|
|
7903
|
-
|
|
7904
|
-
|
|
8671
|
+
say();
|
|
8672
|
+
say(c.dim(wrap(t.route.noTarget(), 74, ' ')));
|
|
8673
|
+
say();
|
|
7905
8674
|
return;
|
|
7906
8675
|
}
|
|
7907
8676
|
const promptPath = stringFlag(args, 'prompt-file');
|
|
@@ -7923,16 +8692,16 @@ async function commandRoute(args, pricing, t) {
|
|
|
7923
8692
|
const available = report.byLabel
|
|
7924
8693
|
.map((r) => (r.label === UNLABELLED ? t.profile.unlabelled() : r.label))
|
|
7925
8694
|
.join(', ');
|
|
7926
|
-
|
|
7927
|
-
|
|
7928
|
-
|
|
8695
|
+
say();
|
|
8696
|
+
say(c.dim(wrap(t.route.labelNotFound(wanted, available), 74, ' ')));
|
|
8697
|
+
say();
|
|
7929
8698
|
return;
|
|
7930
8699
|
}
|
|
7931
8700
|
const slice = levers.slices.find((s) => s.route !== null && (wanted === undefined || s.label === wanted));
|
|
7932
8701
|
if (!slice?.route) {
|
|
7933
|
-
|
|
7934
|
-
|
|
7935
|
-
|
|
8702
|
+
say();
|
|
8703
|
+
say(c.dim(wrap(t.route.noRoute(), 74, ' ')));
|
|
8704
|
+
say();
|
|
7936
8705
|
return;
|
|
7937
8706
|
}
|
|
7938
8707
|
const prompt = capInput(await readFile(promptPath, 'utf8'), promptPath, maxInputFlag(args, t), t);
|
|
@@ -7956,8 +8725,8 @@ async function commandRoute(args, pricing, t) {
|
|
|
7956
8725
|
throw new Error(t.errors.llmNotConfigured());
|
|
7957
8726
|
const label = slice.label === UNLABELLED ? t.profile.unlabelled() : slice.label;
|
|
7958
8727
|
const worth = formatUsd(slice.route.savingUsd);
|
|
7959
|
-
|
|
7960
|
-
|
|
8728
|
+
say();
|
|
8729
|
+
say(` ${c.bold(t.route.picked(label, slice.modelName, slice.route.candidate.displayName, worth, `${(slice.shareOfBill * 100).toFixed(1)}%`))}`);
|
|
7961
8730
|
/**
|
|
7962
8731
|
* The money and the measurement have to describe the same calls.
|
|
7963
8732
|
*
|
|
@@ -7968,16 +8737,16 @@ async function commandRoute(args, pricing, t) {
|
|
|
7968
8737
|
* counts, so it is stated rather than guessed at.
|
|
7969
8738
|
*/
|
|
7970
8739
|
if (slice.label === UNLABELLED) {
|
|
7971
|
-
|
|
8740
|
+
say(` ${c.yellow('!')} ${c.dim(wrap(t.route.unlabelledSlice(), 74, ' '))}`);
|
|
7972
8741
|
}
|
|
7973
|
-
|
|
7974
|
-
|
|
8742
|
+
say();
|
|
8743
|
+
say(` ${c.dim(wrap(t.route.willSpend(inputs.length * 3, provider.model, candidate.model), 74, ' '))}`);
|
|
7975
8744
|
if (!boolFlag(args, 'yes')) {
|
|
7976
|
-
|
|
7977
|
-
|
|
8745
|
+
say(` ${c.dim(t.route.dryRun())}`);
|
|
8746
|
+
say();
|
|
7978
8747
|
return;
|
|
7979
8748
|
}
|
|
7980
|
-
|
|
8749
|
+
say(` ${c.dim(t.route.running(inputs.length))}`);
|
|
7981
8750
|
// Same prompt on both sides. The axis under test is the model, and passing the
|
|
7982
8751
|
// prompt twice is what says so at the call site.
|
|
7983
8752
|
const result = await evaluate(prompt, prompt, inputs, provider, {
|
|
@@ -7985,21 +8754,21 @@ async function commandRoute(args, pricing, t) {
|
|
|
7985
8754
|
concurrency: numberFlag(args, 'concurrency', 3, t),
|
|
7986
8755
|
});
|
|
7987
8756
|
if (boolFlag(args, 'json')) {
|
|
7988
|
-
console.log(JSON.stringify(
|
|
8757
|
+
console.log(JSON.stringify(routeDocument(slice, result), null, 2));
|
|
7989
8758
|
return;
|
|
7990
8759
|
}
|
|
7991
8760
|
const asPct = (v) => `${(v * 100).toFixed(0)}%`;
|
|
7992
|
-
|
|
7993
|
-
|
|
7994
|
-
|
|
8761
|
+
say();
|
|
8762
|
+
say(` ${c.dim(wrap(t.route.agreement(asPct(result.crossAgreement), asPct(result.selfAgreement)), 74, ' '))}`);
|
|
8763
|
+
say();
|
|
7995
8764
|
if (result.verdict === 'inconclusive') {
|
|
7996
|
-
|
|
8765
|
+
say(` ${c.bold(wrap(t.route.inconclusive(), 74, ' '))}`);
|
|
7997
8766
|
}
|
|
7998
8767
|
else if (result.verdict === 'diverges') {
|
|
7999
|
-
|
|
8768
|
+
say(` ${c.yellow('!')} ${c.bold(wrap(t.route.diverges(worth), 74, ' '))}`);
|
|
8000
8769
|
}
|
|
8001
8770
|
else {
|
|
8002
|
-
|
|
8771
|
+
say(` ${c.green('✓')} ${c.bold(wrap(t.route.holds(worth), 74, ' '))}`);
|
|
8003
8772
|
}
|
|
8004
8773
|
/**
|
|
8005
8774
|
* Printed on every verdict including the good one. Agreement is not
|
|
@@ -8007,8 +8776,8 @@ async function commandRoute(args, pricing, t) {
|
|
|
8007
8776
|
* ever right, and a green tick that let somebody forget that would be the tool
|
|
8008
8777
|
* overstating what it knows.
|
|
8009
8778
|
*/
|
|
8010
|
-
|
|
8011
|
-
|
|
8779
|
+
say(` ${c.dim(wrap(t.route.yours(), 74, ' '))}`);
|
|
8780
|
+
say();
|
|
8012
8781
|
}
|
|
8013
8782
|
/**
|
|
8014
8783
|
* `trazum baseline <dir>` — record what the estate costs now.
|
|
@@ -8163,8 +8932,12 @@ async function scanPrompts(root, args, flagBudget, config, counter, level, t, lo
|
|
|
8163
8932
|
}
|
|
8164
8933
|
const verdicts = [];
|
|
8165
8934
|
const declined = [];
|
|
8935
|
+
// Every prompt this run judged, so the report's band is the widest of them
|
|
8936
|
+
// rather than one figure asserted over text of several kinds.
|
|
8937
|
+
const judgedTexts = [];
|
|
8166
8938
|
for (const relativePath of files) {
|
|
8167
8939
|
const text = capInput(await readFile(join(root, relativePath), 'utf8'), relativePath, maxInputFlag(args, t), t);
|
|
8940
|
+
judgedTexts.push(text);
|
|
8168
8941
|
// Budgets are keyed on paths as written in the repository, so a pattern like
|
|
8169
8942
|
// `prompts/**` has to be matched against the path including the root the
|
|
8170
8943
|
// user passed — not against the name relative to it.
|
|
@@ -8194,7 +8967,7 @@ async function scanPrompts(root, args, flagBudget, config, counter, level, t, lo
|
|
|
8194
8967
|
if (verdicts.length === 0 && declined.length === 0) {
|
|
8195
8968
|
throw new Error(t.errors.noPromptsFound(root, extensions.join(' ')));
|
|
8196
8969
|
}
|
|
8197
|
-
return { verdicts, declined, truncated, extensions };
|
|
8970
|
+
return { verdicts, declined, truncated, extensions, band: widestBand(judgedTexts) };
|
|
8198
8971
|
}
|
|
8199
8972
|
async function checkDirectory(root, args, flagBudget, config, counter, level, t, locale, pricing, only) {
|
|
8200
8973
|
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
@@ -8213,7 +8986,7 @@ async function checkDirectory(root, args, flagBudget, config, counter, level, t,
|
|
|
8213
8986
|
return;
|
|
8214
8987
|
only = checkable;
|
|
8215
8988
|
}
|
|
8216
|
-
const { verdicts, declined, truncated, extensions } = await scanPrompts(root, args, flagBudget, config, counter, level, t, locale, pricing, only);
|
|
8989
|
+
const { verdicts, declined, truncated, extensions, band } = await scanPrompts(root, args, flagBudget, config, counter, level, t, locale, pricing, only);
|
|
8217
8990
|
/**
|
|
8218
8991
|
* The baseline gate, when the config declares one and `--no-baseline` did not
|
|
8219
8992
|
* switch it off for this run.
|
|
@@ -8270,6 +9043,7 @@ async function checkDirectory(root, args, flagBudget, config, counter, level, t,
|
|
|
8270
9043
|
await writeMarkdown(args, () => renderCheckMarkdown({
|
|
8271
9044
|
target: root,
|
|
8272
9045
|
verdicts,
|
|
9046
|
+
band,
|
|
8273
9047
|
level,
|
|
8274
9048
|
tokenSource: counter.source,
|
|
8275
9049
|
truncated,
|
|
@@ -8411,6 +9185,15 @@ function joinPosix(root, relativePath) {
|
|
|
8411
9185
|
* never "delete this", and nothing here edits the prompt.
|
|
8412
9186
|
*/
|
|
8413
9187
|
async function commandPrune(args, t) {
|
|
9188
|
+
// Same as `route`: under `--json` stdout is the document, and the estimate
|
|
9189
|
+
// a reader has to see before spending goes to stderr rather than away.
|
|
9190
|
+
const asJson = boolFlag(args, 'json');
|
|
9191
|
+
const say = (line = '') => {
|
|
9192
|
+
if (asJson)
|
|
9193
|
+
console.error(line);
|
|
9194
|
+
else
|
|
9195
|
+
console.log(line);
|
|
9196
|
+
};
|
|
8414
9197
|
const prompt = await readInput(args.positional[0], t, maxInputFlag(args, t));
|
|
8415
9198
|
const casesPath = stringFlag(args, 'cases');
|
|
8416
9199
|
if (!casesPath)
|
|
@@ -8424,10 +9207,10 @@ async function commandPrune(args, t) {
|
|
|
8424
9207
|
const calls = plannedCalls(examples.length, inputs.length);
|
|
8425
9208
|
// Printed before the key is even looked up, so somebody weighing it up does not
|
|
8426
9209
|
// need a configured provider to see the number.
|
|
8427
|
-
|
|
8428
|
-
|
|
9210
|
+
say();
|
|
9211
|
+
say(c.bold(t.prune.estimate(examples.length, inputs.length, calls)));
|
|
8429
9212
|
if (!boolFlag(args, 'yes')) {
|
|
8430
|
-
|
|
9213
|
+
say(c.yellow(` ${t.prune.needsConsent()}`));
|
|
8431
9214
|
return;
|
|
8432
9215
|
}
|
|
8433
9216
|
const provider = providerFromEnv();
|
|
@@ -8437,14 +9220,14 @@ async function commandPrune(args, t) {
|
|
|
8437
9220
|
concurrency: numberFlag(args, 'concurrency', 3, t),
|
|
8438
9221
|
});
|
|
8439
9222
|
if (boolFlag(args, 'json')) {
|
|
8440
|
-
console.log(JSON.stringify(report, null, 2));
|
|
9223
|
+
console.log(JSON.stringify(pruneDocument(report), null, 2));
|
|
8441
9224
|
return;
|
|
8442
9225
|
}
|
|
8443
9226
|
const pct = (value) => `${(value * 100).toFixed(0)}%`;
|
|
8444
|
-
|
|
8445
|
-
|
|
8446
|
-
|
|
8447
|
-
|
|
9227
|
+
say();
|
|
9228
|
+
say(sectionHeading(t.prune.heading(provider.model)));
|
|
9229
|
+
say(` ${c.dim(t.prune.selfAgreement(pct(report.selfAgreement)))}`);
|
|
9230
|
+
say();
|
|
8448
9231
|
for (const contribution of report.contributions) {
|
|
8449
9232
|
/**
|
|
8450
9233
|
* The mark points at what the reader can act on, which is the *recoverable*
|
|
@@ -8461,7 +9244,7 @@ async function commandPrune(args, t) {
|
|
|
8461
9244
|
: needed
|
|
8462
9245
|
? t.prune.verdictNeeded()
|
|
8463
9246
|
: t.prune.verdictRecoverable();
|
|
8464
|
-
|
|
9247
|
+
say(` ${mark} ${t.prune.line(contribution.index + 1, contribution.tokens, pct(contribution.agreementWithout))}`
|
|
8465
9248
|
+ ` ${unknown ? c.yellow(label) : needed ? c.dim(label) : c.green(label)}`);
|
|
8466
9249
|
/**
|
|
8467
9250
|
* The first line that is not the header, because the header is the same on
|
|
@@ -8471,15 +9254,15 @@ async function commandPrune(args, t) {
|
|
|
8471
9254
|
*/
|
|
8472
9255
|
const lines = contribution.text.split('\n').filter((line) => line.trim() !== '');
|
|
8473
9256
|
const body = lines.find((line) => !/^\s*(?:#+\s*)?(?:example|ejemplo)\b[\s:.-]*$/i.test(line));
|
|
8474
|
-
|
|
9257
|
+
say(` ${c.dim(truncate((body ?? lines[0] ?? '').trim(), 60))}`);
|
|
8475
9258
|
}
|
|
8476
|
-
|
|
9259
|
+
say();
|
|
8477
9260
|
if (report.recoverableTokens > 0) {
|
|
8478
|
-
|
|
9261
|
+
say(` ${t.prune.recoverable(report.recoverableTokens)}`);
|
|
8479
9262
|
}
|
|
8480
|
-
|
|
8481
|
-
|
|
8482
|
-
|
|
9263
|
+
say(` ${c.dim(wrap(t.prune.caveat(), 74, ' '))}`);
|
|
9264
|
+
say();
|
|
9265
|
+
say(c.dim(` ${t.eval.callsMade(report.callsMade)}`));
|
|
8483
9266
|
}
|
|
8484
9267
|
async function commandEval(args, config, t, locale) {
|
|
8485
9268
|
const prompt = await readInput(args.positional[0], t, maxInputFlag(args, t));
|
|
@@ -8729,6 +9512,18 @@ function printComparison(comparison, beforePath, afterPath, optimized, t) {
|
|
|
8729
9512
|
const DOCTOR_LIST_LIMIT = 8;
|
|
8730
9513
|
const BLAME_DEFAULT_LIMIT = 20;
|
|
8731
9514
|
const BLAME_MAX_LIMIT = 500;
|
|
9515
|
+
/**
|
|
9516
|
+
* The band for a report covering several prompts.
|
|
9517
|
+
*
|
|
9518
|
+
* The widest of them, and the empty case is the widest published band rather
|
|
9519
|
+
* than a narrow default: a report over nothing has measured nothing, and
|
|
9520
|
+
* guessing narrow there is the one direction that produces a false claim.
|
|
9521
|
+
*/
|
|
9522
|
+
function widestBand(texts) {
|
|
9523
|
+
if (texts.length === 0)
|
|
9524
|
+
return ESTIMATE_ERROR_BAND_PCT;
|
|
9525
|
+
return Math.max(...texts.map((text) => bandFor(text)));
|
|
9526
|
+
}
|
|
8732
9527
|
/**
|
|
8733
9528
|
* `trazum blame <file>` — what happened to this prompt's cost, and who did it.
|
|
8734
9529
|
*
|
|
@@ -8803,14 +9598,32 @@ async function commandBlame(args, config, pricing, t) {
|
|
|
8803
9598
|
// put the newest first: this is a history, and histories are read backwards.
|
|
8804
9599
|
const shown = rows.slice(revisions.length > limit ? 1 : 0).reverse();
|
|
8805
9600
|
const truncatedHistory = revisions.length > limit;
|
|
9601
|
+
/*
|
|
9602
|
+
The band of this file's own text, read once and handed to both renderers.
|
|
9603
|
+
|
|
9604
|
+
`blame` walks one file across revisions, so the kind of text is a property
|
|
9605
|
+
of the file, and today's content is the closest thing to it that exists
|
|
9606
|
+
here. A file that is gone from the working tree — renamed away, or deleted
|
|
9607
|
+
in the commit being blamed — falls back to the widest published band, which
|
|
9608
|
+
is exactly what that constant is for: a caller that cannot see the text can
|
|
9609
|
+
only guess in the safe direction.
|
|
9610
|
+
*/
|
|
9611
|
+
let band = ESTIMATE_ERROR_BAND_PCT;
|
|
9612
|
+
try {
|
|
9613
|
+
band = bandFor(await readFile(resolvePath(process.cwd(), target), 'utf8'));
|
|
9614
|
+
}
|
|
9615
|
+
catch {
|
|
9616
|
+
// Not on disk now. The wide band stands and says so by being wide.
|
|
9617
|
+
}
|
|
8806
9618
|
await writeMarkdown(args, () => renderBlameMarkdown({
|
|
8807
9619
|
repoPath,
|
|
8808
9620
|
rows: shown,
|
|
8809
9621
|
truncated: truncatedHistory,
|
|
8810
9622
|
netCost: netCostOf(shown, args, config, pricing, t),
|
|
9623
|
+
band,
|
|
8811
9624
|
t,
|
|
8812
9625
|
}));
|
|
8813
|
-
printBlame(shown, { repoPath, args, config, pricing, t, truncated: truncatedHistory });
|
|
9626
|
+
printBlame(shown, { repoPath, args, config, pricing, t, truncated: truncatedHistory, band });
|
|
8814
9627
|
}
|
|
8815
9628
|
/**
|
|
8816
9629
|
* What the movement across a history costs per month, or `null`.
|
|
@@ -8946,7 +9759,7 @@ function printBlame(rows, context) {
|
|
|
8946
9759
|
console.log(`\n${c.bold(t.blame.biggestRise())}`);
|
|
8947
9760
|
console.log(` ${t.blame.biggestRiseDetail(n(worst.delta), worst.revision.author, truncate(worst.revision.subject, 60), worst.revision.shortSha)}`);
|
|
8948
9761
|
}
|
|
8949
|
-
console.log(`\n${c.dim(t.blame.estimateNote())}\n`);
|
|
9762
|
+
console.log(`\n${c.dim(t.blame.estimateNote(context.band))}\n`);
|
|
8950
9763
|
}
|
|
8951
9764
|
/**
|
|
8952
9765
|
* `trazum rank <dir>` — which of these prompts to fix first.
|
|
@@ -9303,9 +10116,21 @@ async function main() {
|
|
|
9303
10116
|
case 'from-claude-code':
|
|
9304
10117
|
await commandFromClaudeCode(args, t);
|
|
9305
10118
|
break;
|
|
10119
|
+
case 'receipt':
|
|
10120
|
+
await commandReceipt(args, pricing, t);
|
|
10121
|
+
break;
|
|
9306
10122
|
case 'from-otel':
|
|
9307
10123
|
await commandFromOtel(args, t);
|
|
9308
10124
|
break;
|
|
10125
|
+
case 'from-litellm':
|
|
10126
|
+
await commandFromLiteLlm(args, t);
|
|
10127
|
+
break;
|
|
10128
|
+
case 'from-langsmith':
|
|
10129
|
+
await commandFromLangsmith(args, t);
|
|
10130
|
+
break;
|
|
10131
|
+
case 'from-helicone':
|
|
10132
|
+
await commandFromHelicone(args, t);
|
|
10133
|
+
break;
|
|
9309
10134
|
case 'switch':
|
|
9310
10135
|
await commandSwitch(args, pricing, t);
|
|
9311
10136
|
break;
|
|
@@ -9346,9 +10171,47 @@ async function main() {
|
|
|
9346
10171
|
throw new Error(t.errors.unknownCommand(args.command));
|
|
9347
10172
|
}
|
|
9348
10173
|
}
|
|
10174
|
+
/**
|
|
10175
|
+
* A filesystem error, said in this product's voice.
|
|
10176
|
+
*
|
|
10177
|
+
* `optimize`, `check`, `profile`, `position`, `diff`, `semantic` and `conform`
|
|
10178
|
+
* all read a path the reader typed, and all seven let Node's own error through:
|
|
10179
|
+
* `ENOENT: no such file or directory, open '/nope/x.txt'` on the commonest
|
|
10180
|
+
* mistake there is. The five converters `stat` first and answer
|
|
10181
|
+
* `/nope/x.json: not found`, so the CLI disagreed with itself about how to
|
|
10182
|
+
* refuse, and the majority spelling was the one that names a syscall.
|
|
10183
|
+
*
|
|
10184
|
+
* Translated here rather than at each of the seven, because the seven is the
|
|
10185
|
+
* part that changes: a command added next year reads a path too, and a fix
|
|
10186
|
+
* applied per call site is one somebody has to remember. A converter's own
|
|
10187
|
+
* message is more specific and is thrown earlier, so it still wins.
|
|
10188
|
+
*/
|
|
10189
|
+
function filesystemRefusal(error, t) {
|
|
10190
|
+
if (!(error instanceof Error) || !('code' in error))
|
|
10191
|
+
return null;
|
|
10192
|
+
const path = 'path' in error && typeof error.path === 'string' ? error.path : null;
|
|
10193
|
+
/*
|
|
10194
|
+
`readFile` on a directory fails at `read` rather than `open`, and Node
|
|
10195
|
+
attaches no path to that one. The refusal still has to arrive with
|
|
10196
|
+
something in it, so it names what went wrong without naming which file.
|
|
10197
|
+
*/
|
|
10198
|
+
if (path === null)
|
|
10199
|
+
return error.code === 'EISDIR' ? t.errors.fileIsDirectoryUnnamed() : null;
|
|
10200
|
+
switch (error.code) {
|
|
10201
|
+
case 'ENOENT':
|
|
10202
|
+
return t.errors.fileNotFound(path);
|
|
10203
|
+
case 'EISDIR':
|
|
10204
|
+
return t.errors.fileIsDirectory(path);
|
|
10205
|
+
case 'EACCES':
|
|
10206
|
+
case 'EPERM':
|
|
10207
|
+
return t.errors.fileNotReadable(path);
|
|
10208
|
+
default:
|
|
10209
|
+
return null;
|
|
10210
|
+
}
|
|
10211
|
+
}
|
|
9349
10212
|
main().catch((error) => {
|
|
9350
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
9351
10213
|
const t = getCliMessages(localeFromArgv(process.argv.slice(2)));
|
|
10214
|
+
const message = filesystemRefusal(error, t) ?? (error instanceof Error ? error.message : String(error));
|
|
9352
10215
|
console.error(`\n${c.red(t.errors.errorLabel())}: ${message}\n`);
|
|
9353
10216
|
process.exitCode = 1;
|
|
9354
10217
|
});
|