@trazum/cli 1.81.0 → 1.82.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -7,7 +7,7 @@ import { createInterface } from 'node:readline/promises';
7
7
  import { dirname, join, resolve as resolvePath } from 'node:path';
8
8
  import { fileURLToPath } from 'node:url';
9
9
  import { gunzipSync } from 'node:zlib';
10
- import { CONTRACT_NAMES, contractSchema, applyRewrites, BASELINE_FILENAME, BASELINE_VERSION, breaches, cacheableMinimum, analyzeCachePrefix, billLevers, bucketedCacheEconomics, bucketedProfile, buildHistory, buildPlan, connectorFor, CONNECTORS, normalizeAnthropicUsage, normalizeOpenAIUsage, bucketsFromRecords, evaluateWatch, firedKey, pruneRecords, recordsFromBuckets, storeInventory, storedReportFrom, verifyPlan, cacheEconomics, cacheHitRate, contextPressure, comparePrompts, compareToBaseline, computeSavings, bandGoverns, countTokensAnthropic, DEFAULT_USAGE, budgetPositions, conform, BREAK_EVEN_BAND, allocate, annualRecord, replayCommitment, coversTheTerm, runExperiment, qualityGate, semanticPassCost, verifySemanticProposals, SEMANTIC_SYSTEM_PROMPT, ladderPosition, validateLadder, outcomeReport, rankPerOutcome, FAILURE_POLICIES, detectFromSource, matchLocale, parsePlanDocument, waiverDay, waiverHistory, proposeInit, MIN_RATE_DAYS, parseConfig, coverageDrift, driversBetween, explainGateFailure, assignSources, fleetRollup, rollUp, heartbeats, ruleYield, labelCoverage, measuredUsage, gateMargin, GATE_MARGIN_TIGHT, estimateTokens, evaluate, extractPrompts, findExamples, formatBaseline, formatSignedUsd, formatUsd, getMessages, getModel, hasMarker, LOCALES, MAX_BASELINE_BYTES, matchGlob, MAX_INPUT_CHARS, moneyIsComparable, mostSpecificMatch, assemble, interview, nearestName, slot, SLOT_IDS, optimize, parseBaseline, PHRASE_LANGUAGES, detectTextLanguage, dictionaryStanding, languagesWithStanding, indexUsage, parseUsageLine, plannedCalls, claudeCodeRecords, otelRecords, ownRate, positionAt, positionReport, profilePrompt, profileToCsv, profileUsage, promptId, providerFromEnv, pruneExamples, refineWithLlm, rejectionText, reorderForCache, repriceProfile, switchAnalysis, reviewAgeDays, reviewExamples, RULES, sharedPrefixes, sharesOf, SOURCE_EXTENSIONS, suggestRewrites, toOtlpMetrics, toPromptfoo, TTL_1H_MS, UNLABELLED, withExactTokenCounts, } from '@trazum/core';
10
+ import { CONTRACT_NAMES, contractSchema, applyRewrites, BASELINE_FILENAME, BASELINE_VERSION, breaches, cacheableMinimum, analyzeCachePrefix, billLevers, bucketedCacheEconomics, bucketedProfile, buildHistory, buildPlan, connectorFor, CONNECTORS, normalizeAnthropicUsage, normalizeOpenAIUsage, bucketsFromRecords, evaluateWatch, firedKey, pruneRecords, recordsFromBuckets, storeInventory, storedReportFrom, verifyPlan, cacheEconomics, cacheHitRate, contextPressure, comparePrompts, compareToBaseline, computeSavings, bandGoverns, countTokensAnthropic, DEFAULT_USAGE, budgetPositions, conform, BREAK_EVEN_BAND, allocate, annualRecord, replayCommitment, coversTheTerm, runExperiment, qualityGate, semanticPassCost, verifySemanticProposals, SEMANTIC_SYSTEM_PROMPT, ladderPosition, validateLadder, outcomeReport, rankPerOutcome, FAILURE_POLICIES, detectFromSource, matchLocale, parsePlanDocument, waiverDay, waiverHistory, proposeInit, MIN_RATE_DAYS, parseConfig, coverageDrift, driversBetween, explainGateFailure, assignSources, fleetRollup, rollUp, heartbeats, ruleYield, labelCoverage, measuredUsage, gateMargin, GATE_MARGIN_TIGHT, estimateTokens, evaluate, extractPrompts, findExamples, formatBaseline, formatSignedUsd, formatUsd, getMessages, getModel, hasMarker, LOCALES, MAX_BASELINE_BYTES, matchGlob, MAX_INPUT_CHARS, moneyIsComparable, mostSpecificMatch, assemble, interview, nearestName, slot, SLOT_IDS, optimize, parseBaseline, PHRASE_LANGUAGES, detectTextLanguage, dictionaryStanding, languagesWithStanding, indexUsage, parseUsageLine, plannedCalls, claudeCodeRecords, ESTIMATE_ERROR_BAND_PCT, bandFor, foreignTokenizer, measuredForeignError, heliconeRecords, litellmRecords, otelRecords, ownRate, positionAt, positionReport, profilePrompt, profileToCsv, profileUsage, promptId, providerFromEnv, pruneExamples, refineWithLlm, rejectionText, reorderForCache, repriceProfile, switchAnalysis, reviewAgeDays, reviewExamples, RULES, sharedPrefixes, sharesOf, SOURCE_EXTENSIONS, suggestRewrites, toOtlpMetrics, toPromptfoo, TTL_1H_MS, UNLABELLED, withExactTokenCounts, } from '@trazum/core';
11
11
  import { cacheDir, cacheStats, cachingProvider, clearCache } from './suggest-cache.js';
12
12
  import { dayOf, formatGap, median, spanDays } from './time.js';
13
13
  // Everything that reads the filesystem, on its own entry point so the web
@@ -383,6 +383,8 @@ const COMMAND_FLAGS = {
383
383
  position: ['json', 'html-out', 'pricing', 'pricing-live'],
384
384
  'from-claude-code': ['label', 'label-from-project', 'out', 'o', 'state'],
385
385
  'from-otel': ['label-from-service', 'out', 'o'],
386
+ 'from-litellm': ['out', 'o'],
387
+ 'from-helicone': ['out', 'o'],
386
388
  switch: ['to', 'migration-usd', 'cases'],
387
389
  ownrate: ['gpu-usd-hour', 'tokens-per-second', 'utilization'],
388
390
  pulse: ['json', 'max-stale-hours'],
@@ -483,11 +485,11 @@ function renderDiff(before, after, t) {
483
485
  /**
484
486
  * The provider's name when the estimator was not calibrated for it.
485
487
  *
486
- * `estimateTokens` is a heuristic tuned against Claude's tokenizer, and the
487
- * ±10% band descends from that. Printing the same band beside a GPT or Kimi
488
- * figure states a precision nobody has measured for that family — and since the
489
- * catalogue grew past Anthropic, that is most of it. Returns null when the model
490
- * is Anthropic's, where the band is at least the claim it was written for.
488
+ * `estimateTokens` is a heuristic tuned against Claude's tokenizer, and every
489
+ * band descends from that. Printing one beside a GPT or Kimi figure states a
490
+ * precision nobody has measured for that family — and since the catalogue grew
491
+ * past Anthropic, that is most of it. Returns null when the model is
492
+ * Anthropic's, where the band is at least the claim it was written for.
491
493
  */
492
494
  function offFamilyName(modelId) {
493
495
  const provider = getModel(modelId).provider;
@@ -495,6 +497,17 @@ function offFamilyName(modelId) {
495
497
  return null;
496
498
  return getModel(modelId).displayName;
497
499
  }
500
+ /**
501
+ * How far off the estimator has been measured on this model's family.
502
+ *
503
+ * Read from the catalogue's provider rather than from the display name, because
504
+ * the name is what a reader sees and the id is what was measured. Null on the
505
+ * calibrated family and on every family nobody has run — an unmeasured one gets
506
+ * a sentence saying so, never a borrowed figure.
507
+ */
508
+ function offFamilyError(modelId) {
509
+ return measuredForeignError(foreignTokenizer(getModel(modelId).provider ?? null));
510
+ }
498
511
  /**
499
512
  * Language codes as names, in the reader's language.
500
513
  *
@@ -559,7 +572,7 @@ namedScenario = false,
559
572
  measured = null) {
560
573
  const n = (value) => value.toLocaleString(t.numberLocale);
561
574
  const sourceNote = result.tokenSource === 'heuristic'
562
- ? c.dim(t.report.estimated(offFamilyName(result.usage.model)))
575
+ ? c.dim(t.report.estimated(offFamilyName(result.usage.model), bandFor(result.optimized), offFamilyError(result.usage.model)))
563
576
  : c.dim(t.report.exactCount());
564
577
  /**
565
578
  * The largest lever, first.
@@ -2477,6 +2490,184 @@ async function commandFromOtel(args, t) {
2477
2490
  if (unparseable > 0)
2478
2491
  console.error(t.fromOtel.unparseable(unparseable));
2479
2492
  }
2493
+ /**
2494
+ * `trazum from-litellm <file|dir>` — a LiteLLM spend log as a usage log.
2495
+ *
2496
+ * The same pure-converter pattern again, pointed at the gateway a great many
2497
+ * teams already put in front of every provider: `LiteLLM_SpendLogs` is the
2498
+ * export most likely to already exist on somebody's disk. `litellmRecords` in
2499
+ * core does the reading; this is the walk and the honesty.
2500
+ *
2501
+ * Three of the four counts it prints exist because the alternative is a
2502
+ * flattering silence. Rows naming no model are not priced by the route they
2503
+ * took, rows with no tokens are logged calls nobody can price, and a
2504
+ * `cache_hit` flag is not a token split — so the cache verdicts read "cannot
2505
+ * tell" rather than a fabricated one, exactly as `from-otel` does.
2506
+ *
2507
+ * The fourth is the one that matters most: LiteLLM prices the same calls with
2508
+ * its own table, and that figure is printed beside Trazum's and never merged
2509
+ * into it. Two price tables summed into one total is how a report becomes
2510
+ * quietly wrong.
2511
+ *
2512
+ * The row carries `messages`, `response`, `api_key`, `requester_ip_address`
2513
+ * and `end_user`. None of it is read; a fixture plants a marker in each and
2514
+ * greps the whole output.
2515
+ */
2516
+ async function commandFromLiteLlm(args, t) {
2517
+ const target = args.positional[0];
2518
+ if (target === undefined)
2519
+ throw new Error(t.fromLiteLlm.noPath());
2520
+ let info;
2521
+ try {
2522
+ info = await stat(target);
2523
+ }
2524
+ catch {
2525
+ throw new Error(t.fromLiteLlm.notFound(target));
2526
+ }
2527
+ const files = [];
2528
+ if (info.isDirectory()) {
2529
+ const entries = await readdir(target, { recursive: true, withFileTypes: true });
2530
+ for (const entry of entries) {
2531
+ if (entry.isFile() && /\.(json|jsonl|ndjson)$/i.test(entry.name)) {
2532
+ files.push(join(entry.parentPath, entry.name));
2533
+ }
2534
+ }
2535
+ files.sort();
2536
+ if (files.length === 0)
2537
+ throw new Error(t.fromLiteLlm.noExports(target));
2538
+ }
2539
+ else {
2540
+ files.push(target);
2541
+ }
2542
+ const lines = [];
2543
+ let rows = 0;
2544
+ let unnamedModel = 0;
2545
+ let noTokens = 0;
2546
+ let cacheFlagged = 0;
2547
+ let unparseable = 0;
2548
+ let reportedSpend = 0;
2549
+ let sawSpend = false;
2550
+ for (const file of files) {
2551
+ const conversion = litellmRecords(await readFile(file, 'utf8'));
2552
+ for (const record of conversion.records)
2553
+ lines.push(JSON.stringify(record));
2554
+ rows += conversion.rows;
2555
+ unnamedModel += conversion.unnamedModel;
2556
+ noTokens += conversion.noTokens;
2557
+ cacheFlagged += conversion.cacheFlagged;
2558
+ unparseable += conversion.unparseable;
2559
+ if (conversion.reportedSpendUsd !== null) {
2560
+ reportedSpend += conversion.reportedSpendUsd;
2561
+ sawSpend = true;
2562
+ }
2563
+ }
2564
+ const out = stringFlag(args, 'out') ?? stringFlag(args, 'o');
2565
+ if (out !== undefined) {
2566
+ await writeFile(out, lines.join('\n') + (lines.length > 0 ? '\n' : ''), 'utf8');
2567
+ console.error(t.fromLiteLlm.written(out));
2568
+ }
2569
+ else {
2570
+ for (const line of lines)
2571
+ console.log(line);
2572
+ }
2573
+ console.error(t.fromLiteLlm.summary(files.length, rows));
2574
+ if (unnamedModel > 0)
2575
+ console.error(t.fromLiteLlm.unnamedModel(unnamedModel));
2576
+ if (noTokens > 0)
2577
+ console.error(t.fromLiteLlm.noTokens(noTokens));
2578
+ if (cacheFlagged > 0)
2579
+ console.error(t.fromLiteLlm.cacheFlagged(cacheFlagged));
2580
+ if (sawSpend)
2581
+ console.error(t.fromLiteLlm.reportedSpend(formatUsd(reportedSpend)));
2582
+ if (unparseable > 0)
2583
+ console.error(t.fromLiteLlm.unparseable(unparseable));
2584
+ }
2585
+ /**
2586
+ * `trazum from-helicone <file|dir>` — a Helicone request export as a usage log.
2587
+ *
2588
+ * The third converter, and the one that needs three columns where the others
2589
+ * need one: Helicone carries `request_model`, `model_override` and
2590
+ * `response_model`, and they can disagree. The response wins, because a bill
2591
+ * is about what was billed rather than what was intended, and the
2592
+ * disagreements are counted so a substitution is something the reader sees.
2593
+ *
2594
+ * Two absences are stated rather than filled in. There is no cache token
2595
+ * split on the row, only a flag; and a Helicone request id names one call and
2596
+ * never a conversation, so the records carry no session and the
2597
+ * conversation-shaped findings stay unavailable. Both are printed, because a
2598
+ * gap that is not said reads as a gap that is not there.
2599
+ */
2600
+ async function commandFromHelicone(args, t) {
2601
+ const target = args.positional[0];
2602
+ if (target === undefined)
2603
+ throw new Error(t.fromHelicone.noPath());
2604
+ let info;
2605
+ try {
2606
+ info = await stat(target);
2607
+ }
2608
+ catch {
2609
+ throw new Error(t.fromHelicone.notFound(target));
2610
+ }
2611
+ const files = [];
2612
+ if (info.isDirectory()) {
2613
+ const entries = await readdir(target, { recursive: true, withFileTypes: true });
2614
+ for (const entry of entries) {
2615
+ if (entry.isFile() && /\.(json|jsonl|ndjson)$/i.test(entry.name)) {
2616
+ files.push(join(entry.parentPath, entry.name));
2617
+ }
2618
+ }
2619
+ files.sort();
2620
+ if (files.length === 0)
2621
+ throw new Error(t.fromHelicone.noExports(target));
2622
+ }
2623
+ else {
2624
+ files.push(target);
2625
+ }
2626
+ const lines = [];
2627
+ let rows = 0;
2628
+ let unnamedModel = 0;
2629
+ let disagreements = 0;
2630
+ let noTokens = 0;
2631
+ let cacheFlagged = 0;
2632
+ let unparseable = 0;
2633
+ for (const file of files) {
2634
+ const conversion = heliconeRecords(await readFile(file, 'utf8'));
2635
+ for (const record of conversion.records)
2636
+ lines.push(JSON.stringify(record));
2637
+ rows += conversion.rows;
2638
+ unnamedModel += conversion.unnamedModel;
2639
+ disagreements += conversion.modelDisagreements;
2640
+ noTokens += conversion.noTokens;
2641
+ cacheFlagged += conversion.cacheFlagged;
2642
+ unparseable += conversion.unparseable;
2643
+ }
2644
+ const out = stringFlag(args, 'out') ?? stringFlag(args, 'o');
2645
+ if (out !== undefined) {
2646
+ await writeFile(out, lines.join('\n') + (lines.length > 0 ? '\n' : ''), 'utf8');
2647
+ console.error(t.fromHelicone.written(out));
2648
+ }
2649
+ else {
2650
+ for (const line of lines)
2651
+ console.log(line);
2652
+ }
2653
+ console.error(t.fromHelicone.summary(files.length, rows));
2654
+ if (unnamedModel > 0)
2655
+ console.error(t.fromHelicone.unnamedModel(unnamedModel));
2656
+ if (disagreements > 0)
2657
+ console.error(t.fromHelicone.disagreements(disagreements));
2658
+ if (noTokens > 0)
2659
+ console.error(t.fromHelicone.noTokens(noTokens));
2660
+ if (cacheFlagged > 0)
2661
+ console.error(t.fromHelicone.cacheFlagged(cacheFlagged));
2662
+ // Said on every run that produced records, not only when something is
2663
+ // missing: the absence is a property of the format, and a reader who has
2664
+ // just converted a month needs to know which questions this log cannot
2665
+ // answer before they go looking for the answers.
2666
+ if (rows > 0)
2667
+ console.error(t.fromHelicone.noSessions());
2668
+ if (unparseable > 0)
2669
+ console.error(t.fromHelicone.unparseable(unparseable));
2670
+ }
2480
2671
  /**
2481
2672
  * `trazum position <usage.jsonl>` — where the month stands, measured.
2482
2673
  *
@@ -4547,6 +4738,7 @@ async function commandCheck(args, config, pricing, t, locale) {
4547
4738
  verdicts: [verdict],
4548
4739
  level,
4549
4740
  tokenSource: counter.source,
4741
+ band: bandFor(prompt),
4550
4742
  truncated: false,
4551
4743
  t,
4552
4744
  }));
@@ -4654,11 +4846,16 @@ async function checkEmbedded(path, extraction, budget, args, counter, level, t,
4654
4846
  }
4655
4847
  const failures = verdicts.filter(isOverBudget);
4656
4848
  const ok = failures.length === 0 && extraction.declined.length === 0;
4849
+ // Widest across the prompts this report covers, never an average: a run over
4850
+ // prose and a CSV covers text the estimator is 6% and 33% out on, and a
4851
+ // figure between them would describe neither.
4852
+ const band = widestBand(extraction.prompts.map((prompt) => prompt.text));
4657
4853
  await writeMarkdown(args, () => renderCheckMarkdown({
4658
4854
  target: path,
4659
4855
  verdicts,
4660
4856
  level,
4661
4857
  tokenSource: counter.source,
4858
+ band,
4662
4859
  truncated: false,
4663
4860
  t,
4664
4861
  }));
@@ -7896,12 +8093,102 @@ function reportProfileGaps(report, t, n, stalePricing = null) {
7896
8093
  * exactly as `prune` does. A command that can spend somebody's money without
7897
8094
  * telling them first is a command they stop trusting.
7898
8095
  */
8096
+ /**
8097
+ * The routing measurement, without the text that produced it.
8098
+ *
8099
+ * `route --json` printed `{ slice, evaluation }` — the whole `EvalReport`,
8100
+ * whose `cases[]` carries `input` (the caller's own case text), `baseline[0]`,
8101
+ * `baseline[1]` and `optimized` (three model answers). It carried no
8102
+ * `schemaVersion` either.
8103
+ *
8104
+ * `docs/json-output.md` opens by promising that **nothing here carries a
8105
+ * session key or prompt text**, and this is the document most likely to be
8106
+ * piped into a dashboard, pasted into a ticket or dropped into a browser. A
8107
+ * tool whose headline is that prompts never leave the machine cannot have its
8108
+ * most portable artefact be the one that carries them.
8109
+ *
8110
+ * The per-case *shape* survives: the two similarity scores are the evidence
8111
+ * behind the verdict, and a reader checking a borderline call needs them. Only
8112
+ * the strings go. Anyone who wants the answers has the human-readable output,
8113
+ * which never left the terminal.
8114
+ *
8115
+ * A named function rather than an inline object so the stripping is a thing a
8116
+ * test can call without spending a provider call to reach it.
8117
+ */
8118
+ export function routeDocument(slice, result) {
8119
+ return {
8120
+ schemaVersion: 1,
8121
+ slice,
8122
+ evaluation: {
8123
+ provider: result.provider,
8124
+ model: result.model,
8125
+ candidateModel: result.candidateModel,
8126
+ verdict: result.verdict,
8127
+ selfAgreement: result.selfAgreement,
8128
+ crossAgreement: result.crossAgreement,
8129
+ callsMade: result.callsMade,
8130
+ cases: result.cases.map((one) => ({
8131
+ selfSimilarity: one.selfSimilarity,
8132
+ crossSimilarity: one.crossSimilarity,
8133
+ })),
8134
+ },
8135
+ };
8136
+ }
8137
+ /**
8138
+ * The example-pruning measurement, likewise.
8139
+ *
8140
+ * `ExampleContribution.text` is *"the example itself, so a report can quote its
8141
+ * first line"* — true of the terminal rendering and wrong of a document that
8142
+ * promises to carry no prompt text. A few-shot example **is** prompt text, and
8143
+ * this was the second command emitting it.
8144
+ *
8145
+ * `index` and `tokens` say which example without quoting it, which is what a
8146
+ * machine reading this needs: it already has the prompt.
8147
+ */
8148
+ export function pruneDocument(report) {
8149
+ return {
8150
+ schemaVersion: 1,
8151
+ provider: report.provider,
8152
+ model: report.model,
8153
+ selfAgreement: report.selfAgreement,
8154
+ recoverableTokens: report.recoverableTokens,
8155
+ callsMade: report.callsMade,
8156
+ contributions: report.contributions.map((one) => ({
8157
+ index: one.index,
8158
+ tokens: one.tokens,
8159
+ agreementWithout: one.agreementWithout,
8160
+ verdict: one.verdict,
8161
+ })),
8162
+ };
8163
+ }
7899
8164
  async function commandRoute(args, pricing, t) {
8165
+ /**
8166
+ * Under `--json`, stdout carries the document and nothing else.
8167
+ *
8168
+ * It carried the human preamble too — the slice it picked, the call count,
8169
+ * the "nothing has been spent yet" line — straight into the same stream as
8170
+ * the JSON, so `route --json > verdict.json` wrote a file no parser would
8171
+ * read. Found by running the command and feeding the file to the bridge
8172
+ * that reads it, which is the one thing the pure-function guards could not
8173
+ * do: `interchange.test.js` names `route` among the commands it cannot
8174
+ * drive, so nothing checked that its `--json` output was a document.
8175
+ *
8176
+ * The lines go to stderr rather than away. A reader running this in a
8177
+ * terminal still needs to see which slice was picked and what it will cost
8178
+ * before it costs it; a pipe needs the document. stderr gives both.
8179
+ */
8180
+ const asJson = boolFlag(args, 'json');
8181
+ const say = (line = '') => {
8182
+ if (asJson)
8183
+ console.error(line);
8184
+ else
8185
+ console.log(line);
8186
+ };
7900
8187
  const path = args.positional[0];
7901
8188
  if (path === undefined) {
7902
- console.log();
7903
- console.log(c.dim(wrap(t.route.noTarget(), 74, ' ')));
7904
- console.log();
8189
+ say();
8190
+ say(c.dim(wrap(t.route.noTarget(), 74, ' ')));
8191
+ say();
7905
8192
  return;
7906
8193
  }
7907
8194
  const promptPath = stringFlag(args, 'prompt-file');
@@ -7923,16 +8210,16 @@ async function commandRoute(args, pricing, t) {
7923
8210
  const available = report.byLabel
7924
8211
  .map((r) => (r.label === UNLABELLED ? t.profile.unlabelled() : r.label))
7925
8212
  .join(', ');
7926
- console.log();
7927
- console.log(c.dim(wrap(t.route.labelNotFound(wanted, available), 74, ' ')));
7928
- console.log();
8213
+ say();
8214
+ say(c.dim(wrap(t.route.labelNotFound(wanted, available), 74, ' ')));
8215
+ say();
7929
8216
  return;
7930
8217
  }
7931
8218
  const slice = levers.slices.find((s) => s.route !== null && (wanted === undefined || s.label === wanted));
7932
8219
  if (!slice?.route) {
7933
- console.log();
7934
- console.log(c.dim(wrap(t.route.noRoute(), 74, ' ')));
7935
- console.log();
8220
+ say();
8221
+ say(c.dim(wrap(t.route.noRoute(), 74, ' ')));
8222
+ say();
7936
8223
  return;
7937
8224
  }
7938
8225
  const prompt = capInput(await readFile(promptPath, 'utf8'), promptPath, maxInputFlag(args, t), t);
@@ -7956,8 +8243,8 @@ async function commandRoute(args, pricing, t) {
7956
8243
  throw new Error(t.errors.llmNotConfigured());
7957
8244
  const label = slice.label === UNLABELLED ? t.profile.unlabelled() : slice.label;
7958
8245
  const worth = formatUsd(slice.route.savingUsd);
7959
- console.log();
7960
- console.log(` ${c.bold(t.route.picked(label, slice.modelName, slice.route.candidate.displayName, worth, `${(slice.shareOfBill * 100).toFixed(1)}%`))}`);
8246
+ say();
8247
+ say(` ${c.bold(t.route.picked(label, slice.modelName, slice.route.candidate.displayName, worth, `${(slice.shareOfBill * 100).toFixed(1)}%`))}`);
7961
8248
  /**
7962
8249
  * The money and the measurement have to describe the same calls.
7963
8250
  *
@@ -7968,16 +8255,16 @@ async function commandRoute(args, pricing, t) {
7968
8255
  * counts, so it is stated rather than guessed at.
7969
8256
  */
7970
8257
  if (slice.label === UNLABELLED) {
7971
- console.log(` ${c.yellow('!')} ${c.dim(wrap(t.route.unlabelledSlice(), 74, ' '))}`);
8258
+ say(` ${c.yellow('!')} ${c.dim(wrap(t.route.unlabelledSlice(), 74, ' '))}`);
7972
8259
  }
7973
- console.log();
7974
- console.log(` ${c.dim(wrap(t.route.willSpend(inputs.length * 3, provider.model, candidate.model), 74, ' '))}`);
8260
+ say();
8261
+ say(` ${c.dim(wrap(t.route.willSpend(inputs.length * 3, provider.model, candidate.model), 74, ' '))}`);
7975
8262
  if (!boolFlag(args, 'yes')) {
7976
- console.log(` ${c.dim(t.route.dryRun())}`);
7977
- console.log();
8263
+ say(` ${c.dim(t.route.dryRun())}`);
8264
+ say();
7978
8265
  return;
7979
8266
  }
7980
- console.log(` ${c.dim(t.route.running(inputs.length))}`);
8267
+ say(` ${c.dim(t.route.running(inputs.length))}`);
7981
8268
  // Same prompt on both sides. The axis under test is the model, and passing the
7982
8269
  // prompt twice is what says so at the call site.
7983
8270
  const result = await evaluate(prompt, prompt, inputs, provider, {
@@ -7985,21 +8272,21 @@ async function commandRoute(args, pricing, t) {
7985
8272
  concurrency: numberFlag(args, 'concurrency', 3, t),
7986
8273
  });
7987
8274
  if (boolFlag(args, 'json')) {
7988
- console.log(JSON.stringify({ slice, evaluation: result }, null, 2));
8275
+ console.log(JSON.stringify(routeDocument(slice, result), null, 2));
7989
8276
  return;
7990
8277
  }
7991
8278
  const asPct = (v) => `${(v * 100).toFixed(0)}%`;
7992
- console.log();
7993
- console.log(` ${c.dim(wrap(t.route.agreement(asPct(result.crossAgreement), asPct(result.selfAgreement)), 74, ' '))}`);
7994
- console.log();
8279
+ say();
8280
+ say(` ${c.dim(wrap(t.route.agreement(asPct(result.crossAgreement), asPct(result.selfAgreement)), 74, ' '))}`);
8281
+ say();
7995
8282
  if (result.verdict === 'inconclusive') {
7996
- console.log(` ${c.bold(wrap(t.route.inconclusive(), 74, ' '))}`);
8283
+ say(` ${c.bold(wrap(t.route.inconclusive(), 74, ' '))}`);
7997
8284
  }
7998
8285
  else if (result.verdict === 'diverges') {
7999
- console.log(` ${c.yellow('!')} ${c.bold(wrap(t.route.diverges(worth), 74, ' '))}`);
8286
+ say(` ${c.yellow('!')} ${c.bold(wrap(t.route.diverges(worth), 74, ' '))}`);
8000
8287
  }
8001
8288
  else {
8002
- console.log(` ${c.green('✓')} ${c.bold(wrap(t.route.holds(worth), 74, ' '))}`);
8289
+ say(` ${c.green('✓')} ${c.bold(wrap(t.route.holds(worth), 74, ' '))}`);
8003
8290
  }
8004
8291
  /**
8005
8292
  * Printed on every verdict including the good one. Agreement is not
@@ -8007,8 +8294,8 @@ async function commandRoute(args, pricing, t) {
8007
8294
  * ever right, and a green tick that let somebody forget that would be the tool
8008
8295
  * overstating what it knows.
8009
8296
  */
8010
- console.log(` ${c.dim(wrap(t.route.yours(), 74, ' '))}`);
8011
- console.log();
8297
+ say(` ${c.dim(wrap(t.route.yours(), 74, ' '))}`);
8298
+ say();
8012
8299
  }
8013
8300
  /**
8014
8301
  * `trazum baseline <dir>` — record what the estate costs now.
@@ -8163,8 +8450,12 @@ async function scanPrompts(root, args, flagBudget, config, counter, level, t, lo
8163
8450
  }
8164
8451
  const verdicts = [];
8165
8452
  const declined = [];
8453
+ // Every prompt this run judged, so the report's band is the widest of them
8454
+ // rather than one figure asserted over text of several kinds.
8455
+ const judgedTexts = [];
8166
8456
  for (const relativePath of files) {
8167
8457
  const text = capInput(await readFile(join(root, relativePath), 'utf8'), relativePath, maxInputFlag(args, t), t);
8458
+ judgedTexts.push(text);
8168
8459
  // Budgets are keyed on paths as written in the repository, so a pattern like
8169
8460
  // `prompts/**` has to be matched against the path including the root the
8170
8461
  // user passed — not against the name relative to it.
@@ -8194,7 +8485,7 @@ async function scanPrompts(root, args, flagBudget, config, counter, level, t, lo
8194
8485
  if (verdicts.length === 0 && declined.length === 0) {
8195
8486
  throw new Error(t.errors.noPromptsFound(root, extensions.join(' ')));
8196
8487
  }
8197
- return { verdicts, declined, truncated, extensions };
8488
+ return { verdicts, declined, truncated, extensions, band: widestBand(judgedTexts) };
8198
8489
  }
8199
8490
  async function checkDirectory(root, args, flagBudget, config, counter, level, t, locale, pricing, only) {
8200
8491
  const n = (value) => value.toLocaleString(t.numberLocale);
@@ -8213,7 +8504,7 @@ async function checkDirectory(root, args, flagBudget, config, counter, level, t,
8213
8504
  return;
8214
8505
  only = checkable;
8215
8506
  }
8216
- const { verdicts, declined, truncated, extensions } = await scanPrompts(root, args, flagBudget, config, counter, level, t, locale, pricing, only);
8507
+ const { verdicts, declined, truncated, extensions, band } = await scanPrompts(root, args, flagBudget, config, counter, level, t, locale, pricing, only);
8217
8508
  /**
8218
8509
  * The baseline gate, when the config declares one and `--no-baseline` did not
8219
8510
  * switch it off for this run.
@@ -8270,6 +8561,7 @@ async function checkDirectory(root, args, flagBudget, config, counter, level, t,
8270
8561
  await writeMarkdown(args, () => renderCheckMarkdown({
8271
8562
  target: root,
8272
8563
  verdicts,
8564
+ band,
8273
8565
  level,
8274
8566
  tokenSource: counter.source,
8275
8567
  truncated,
@@ -8411,6 +8703,15 @@ function joinPosix(root, relativePath) {
8411
8703
  * never "delete this", and nothing here edits the prompt.
8412
8704
  */
8413
8705
  async function commandPrune(args, t) {
8706
+ // Same as `route`: under `--json` stdout is the document, and the estimate
8707
+ // a reader has to see before spending goes to stderr rather than away.
8708
+ const asJson = boolFlag(args, 'json');
8709
+ const say = (line = '') => {
8710
+ if (asJson)
8711
+ console.error(line);
8712
+ else
8713
+ console.log(line);
8714
+ };
8414
8715
  const prompt = await readInput(args.positional[0], t, maxInputFlag(args, t));
8415
8716
  const casesPath = stringFlag(args, 'cases');
8416
8717
  if (!casesPath)
@@ -8424,10 +8725,10 @@ async function commandPrune(args, t) {
8424
8725
  const calls = plannedCalls(examples.length, inputs.length);
8425
8726
  // Printed before the key is even looked up, so somebody weighing it up does not
8426
8727
  // need a configured provider to see the number.
8427
- console.log();
8428
- console.log(c.bold(t.prune.estimate(examples.length, inputs.length, calls)));
8728
+ say();
8729
+ say(c.bold(t.prune.estimate(examples.length, inputs.length, calls)));
8429
8730
  if (!boolFlag(args, 'yes')) {
8430
- console.log(c.yellow(` ${t.prune.needsConsent()}`));
8731
+ say(c.yellow(` ${t.prune.needsConsent()}`));
8431
8732
  return;
8432
8733
  }
8433
8734
  const provider = providerFromEnv();
@@ -8437,14 +8738,14 @@ async function commandPrune(args, t) {
8437
8738
  concurrency: numberFlag(args, 'concurrency', 3, t),
8438
8739
  });
8439
8740
  if (boolFlag(args, 'json')) {
8440
- console.log(JSON.stringify(report, null, 2));
8741
+ console.log(JSON.stringify(pruneDocument(report), null, 2));
8441
8742
  return;
8442
8743
  }
8443
8744
  const pct = (value) => `${(value * 100).toFixed(0)}%`;
8444
- console.log();
8445
- console.log(sectionHeading(t.prune.heading(provider.model)));
8446
- console.log(` ${c.dim(t.prune.selfAgreement(pct(report.selfAgreement)))}`);
8447
- console.log();
8745
+ say();
8746
+ say(sectionHeading(t.prune.heading(provider.model)));
8747
+ say(` ${c.dim(t.prune.selfAgreement(pct(report.selfAgreement)))}`);
8748
+ say();
8448
8749
  for (const contribution of report.contributions) {
8449
8750
  /**
8450
8751
  * The mark points at what the reader can act on, which is the *recoverable*
@@ -8461,7 +8762,7 @@ async function commandPrune(args, t) {
8461
8762
  : needed
8462
8763
  ? t.prune.verdictNeeded()
8463
8764
  : t.prune.verdictRecoverable();
8464
- console.log(` ${mark} ${t.prune.line(contribution.index + 1, contribution.tokens, pct(contribution.agreementWithout))}`
8765
+ say(` ${mark} ${t.prune.line(contribution.index + 1, contribution.tokens, pct(contribution.agreementWithout))}`
8465
8766
  + ` ${unknown ? c.yellow(label) : needed ? c.dim(label) : c.green(label)}`);
8466
8767
  /**
8467
8768
  * The first line that is not the header, because the header is the same on
@@ -8471,15 +8772,15 @@ async function commandPrune(args, t) {
8471
8772
  */
8472
8773
  const lines = contribution.text.split('\n').filter((line) => line.trim() !== '');
8473
8774
  const body = lines.find((line) => !/^\s*(?:#+\s*)?(?:example|ejemplo)\b[\s:.-]*$/i.test(line));
8474
- console.log(` ${c.dim(truncate((body ?? lines[0] ?? '').trim(), 60))}`);
8775
+ say(` ${c.dim(truncate((body ?? lines[0] ?? '').trim(), 60))}`);
8475
8776
  }
8476
- console.log();
8777
+ say();
8477
8778
  if (report.recoverableTokens > 0) {
8478
- console.log(` ${t.prune.recoverable(report.recoverableTokens)}`);
8779
+ say(` ${t.prune.recoverable(report.recoverableTokens)}`);
8479
8780
  }
8480
- console.log(` ${c.dim(wrap(t.prune.caveat(), 74, ' '))}`);
8481
- console.log();
8482
- console.log(c.dim(` ${t.eval.callsMade(report.callsMade)}`));
8781
+ say(` ${c.dim(wrap(t.prune.caveat(), 74, ' '))}`);
8782
+ say();
8783
+ say(c.dim(` ${t.eval.callsMade(report.callsMade)}`));
8483
8784
  }
8484
8785
  async function commandEval(args, config, t, locale) {
8485
8786
  const prompt = await readInput(args.positional[0], t, maxInputFlag(args, t));
@@ -8729,6 +9030,18 @@ function printComparison(comparison, beforePath, afterPath, optimized, t) {
8729
9030
  const DOCTOR_LIST_LIMIT = 8;
8730
9031
  const BLAME_DEFAULT_LIMIT = 20;
8731
9032
  const BLAME_MAX_LIMIT = 500;
9033
+ /**
9034
+ * The band for a report covering several prompts.
9035
+ *
9036
+ * The widest of them, and the empty case is the widest published band rather
9037
+ * than a narrow default: a report over nothing has measured nothing, and
9038
+ * guessing narrow there is the one direction that produces a false claim.
9039
+ */
9040
+ function widestBand(texts) {
9041
+ if (texts.length === 0)
9042
+ return ESTIMATE_ERROR_BAND_PCT;
9043
+ return Math.max(...texts.map((text) => bandFor(text)));
9044
+ }
8732
9045
  /**
8733
9046
  * `trazum blame <file>` — what happened to this prompt's cost, and who did it.
8734
9047
  *
@@ -8803,14 +9116,32 @@ async function commandBlame(args, config, pricing, t) {
8803
9116
  // put the newest first: this is a history, and histories are read backwards.
8804
9117
  const shown = rows.slice(revisions.length > limit ? 1 : 0).reverse();
8805
9118
  const truncatedHistory = revisions.length > limit;
9119
+ /*
9120
+ The band of this file's own text, read once and handed to both renderers.
9121
+
9122
+ `blame` walks one file across revisions, so the kind of text is a property
9123
+ of the file, and today's content is the closest thing to it that exists
9124
+ here. A file that is gone from the working tree — renamed away, or deleted
9125
+ in the commit being blamed — falls back to the widest published band, which
9126
+ is exactly what that constant is for: a caller that cannot see the text can
9127
+ only guess in the safe direction.
9128
+ */
9129
+ let band = ESTIMATE_ERROR_BAND_PCT;
9130
+ try {
9131
+ band = bandFor(await readFile(resolvePath(process.cwd(), target), 'utf8'));
9132
+ }
9133
+ catch {
9134
+ // Not on disk now. The wide band stands and says so by being wide.
9135
+ }
8806
9136
  await writeMarkdown(args, () => renderBlameMarkdown({
8807
9137
  repoPath,
8808
9138
  rows: shown,
8809
9139
  truncated: truncatedHistory,
8810
9140
  netCost: netCostOf(shown, args, config, pricing, t),
9141
+ band,
8811
9142
  t,
8812
9143
  }));
8813
- printBlame(shown, { repoPath, args, config, pricing, t, truncated: truncatedHistory });
9144
+ printBlame(shown, { repoPath, args, config, pricing, t, truncated: truncatedHistory, band });
8814
9145
  }
8815
9146
  /**
8816
9147
  * What the movement across a history costs per month, or `null`.
@@ -8946,7 +9277,7 @@ function printBlame(rows, context) {
8946
9277
  console.log(`\n${c.bold(t.blame.biggestRise())}`);
8947
9278
  console.log(` ${t.blame.biggestRiseDetail(n(worst.delta), worst.revision.author, truncate(worst.revision.subject, 60), worst.revision.shortSha)}`);
8948
9279
  }
8949
- console.log(`\n${c.dim(t.blame.estimateNote())}\n`);
9280
+ console.log(`\n${c.dim(t.blame.estimateNote(context.band))}\n`);
8950
9281
  }
8951
9282
  /**
8952
9283
  * `trazum rank <dir>` — which of these prompts to fix first.
@@ -9306,6 +9637,12 @@ async function main() {
9306
9637
  case 'from-otel':
9307
9638
  await commandFromOtel(args, t);
9308
9639
  break;
9640
+ case 'from-litellm':
9641
+ await commandFromLiteLlm(args, t);
9642
+ break;
9643
+ case 'from-helicone':
9644
+ await commandFromHelicone(args, t);
9645
+ break;
9309
9646
  case 'switch':
9310
9647
  await commandSwitch(args, pricing, t);
9311
9648
  break;