@trazum/cli 1.81.0 → 1.82.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.ts CHANGED
@@ -112,6 +112,12 @@ import {
112
112
  parseUsageLine,
113
113
  plannedCalls,
114
114
  claudeCodeRecords,
115
+ ESTIMATE_ERROR_BAND_PCT,
116
+ bandFor,
117
+ foreignTokenizer,
118
+ measuredForeignError,
119
+ heliconeRecords,
120
+ litellmRecords,
115
121
  otelRecords,
116
122
  ownRate,
117
123
  positionAt,
@@ -145,9 +151,11 @@ import { cacheDir, cacheStats, cachingProvider, clearCache } from './suggest-cac
145
151
  import { dayOf, formatGap, median, spanDays } from './time.js';
146
152
  import type {
147
153
  BucketedReport,
154
+ EvalReport,
148
155
  FleetSource,
149
156
  HistoryRun,
150
157
  MeasuredUsage,
158
+ PruneReport,
151
159
  PlanDocument,
152
160
  StoredReport,
153
161
  VerifiedAction,
@@ -678,6 +686,8 @@ const COMMAND_FLAGS: Record<string, string[]> = {
678
686
  position: ['json', 'html-out', 'pricing', 'pricing-live'],
679
687
  'from-claude-code': ['label', 'label-from-project', 'out', 'o', 'state'],
680
688
  'from-otel': ['label-from-service', 'out', 'o'],
689
+ 'from-litellm': ['out', 'o'],
690
+ 'from-helicone': ['out', 'o'],
681
691
  switch: ['to', 'migration-usd', 'cases'],
682
692
  ownrate: ['gpu-usd-hour', 'tokens-per-second', 'utilization'],
683
693
  pulse: ['json', 'max-stale-hours'],
@@ -789,11 +799,11 @@ function renderDiff(before: string, after: string, t: CliMessages): string {
789
799
  /**
790
800
  * The provider's name when the estimator was not calibrated for it.
791
801
  *
792
- * `estimateTokens` is a heuristic tuned against Claude's tokenizer, and the
793
- * ±10% band descends from that. Printing the same band beside a GPT or Kimi
794
- * figure states a precision nobody has measured for that family — and since the
795
- * catalogue grew past Anthropic, that is most of it. Returns null when the model
796
- * is Anthropic's, where the band is at least the claim it was written for.
802
+ * `estimateTokens` is a heuristic tuned against Claude's tokenizer, and every
803
+ * band descends from that. Printing one beside a GPT or Kimi figure states a
804
+ * precision nobody has measured for that family — and since the catalogue grew
805
+ * past Anthropic, that is most of it. Returns null when the model is
806
+ * Anthropic's, where the band is at least the claim it was written for.
797
807
  */
798
808
  function offFamilyName(modelId: string): string | null {
799
809
  const provider = getModel(modelId).provider;
@@ -801,6 +811,18 @@ function offFamilyName(modelId: string): string | null {
801
811
  return getModel(modelId).displayName;
802
812
  }
803
813
 
814
+ /**
815
+ * How far off the estimator has been measured on this model's family.
816
+ *
817
+ * Read from the catalogue's provider rather than from the display name, because
818
+ * the name is what a reader sees and the id is what was measured. Null on the
819
+ * calibrated family and on every family nobody has run — an unmeasured one gets
820
+ * a sentence saying so, never a borrowed figure.
821
+ */
822
+ function offFamilyError(modelId: string): number | null {
823
+ return measuredForeignError(foreignTokenizer(getModel(modelId).provider ?? null));
824
+ }
825
+
804
826
  /**
805
827
  * Language codes as names, in the reader's language.
806
828
  *
@@ -883,7 +905,13 @@ function printReport(
883
905
  const n = (value: number): string => value.toLocaleString(t.numberLocale);
884
906
  const sourceNote =
885
907
  result.tokenSource === 'heuristic'
886
- ? c.dim(t.report.estimated(offFamilyName(result.usage.model)))
908
+ ? c.dim(
909
+ t.report.estimated(
910
+ offFamilyName(result.usage.model),
911
+ bandFor(result.optimized),
912
+ offFamilyError(result.usage.model),
913
+ ),
914
+ )
887
915
  : c.dim(t.report.exactCount());
888
916
 
889
917
  /**
@@ -3082,6 +3110,169 @@ async function commandFromOtel(args: Args, t: CliMessages): Promise<void> {
3082
3110
  if (unparseable > 0) console.error(t.fromOtel.unparseable(unparseable));
3083
3111
  }
3084
3112
 
3113
+ /**
3114
+ * `trazum from-litellm <file|dir>` — a LiteLLM spend log as a usage log.
3115
+ *
3116
+ * The same pure-converter pattern again, pointed at the gateway a great many
3117
+ * teams already put in front of every provider: `LiteLLM_SpendLogs` is the
3118
+ * export most likely to already exist on somebody's disk. `litellmRecords` in
3119
+ * core does the reading; this is the walk and the honesty.
3120
+ *
3121
+ * Three of the four counts it prints exist because the alternative is a
3122
+ * flattering silence. Rows naming no model are not priced by the route they
3123
+ * took, rows with no tokens are logged calls nobody can price, and a
3124
+ * `cache_hit` flag is not a token split — so the cache verdicts read "cannot
3125
+ * tell" rather than a fabricated one, exactly as `from-otel` does.
3126
+ *
3127
+ * The fourth is the one that matters most: LiteLLM prices the same calls with
3128
+ * its own table, and that figure is printed beside Trazum's and never merged
3129
+ * into it. Two price tables summed into one total is how a report becomes
3130
+ * quietly wrong.
3131
+ *
3132
+ * The row carries `messages`, `response`, `api_key`, `requester_ip_address`
3133
+ * and `end_user`. None of it is read; a fixture plants a marker in each and
3134
+ * greps the whole output.
3135
+ */
3136
+ async function commandFromLiteLlm(args: Args, t: CliMessages): Promise<void> {
3137
+ const target = args.positional[0];
3138
+ if (target === undefined) throw new Error(t.fromLiteLlm.noPath());
3139
+
3140
+ let info;
3141
+ try {
3142
+ info = await stat(target);
3143
+ } catch {
3144
+ throw new Error(t.fromLiteLlm.notFound(target));
3145
+ }
3146
+ const files: string[] = [];
3147
+ if (info.isDirectory()) {
3148
+ const entries = await readdir(target, { recursive: true, withFileTypes: true });
3149
+ for (const entry of entries) {
3150
+ if (entry.isFile() && /\.(json|jsonl|ndjson)$/i.test(entry.name)) {
3151
+ files.push(join(entry.parentPath, entry.name));
3152
+ }
3153
+ }
3154
+ files.sort();
3155
+ if (files.length === 0) throw new Error(t.fromLiteLlm.noExports(target));
3156
+ } else {
3157
+ files.push(target);
3158
+ }
3159
+
3160
+ const lines: string[] = [];
3161
+ let rows = 0;
3162
+ let unnamedModel = 0;
3163
+ let noTokens = 0;
3164
+ let cacheFlagged = 0;
3165
+ let unparseable = 0;
3166
+ let reportedSpend = 0;
3167
+ let sawSpend = false;
3168
+ for (const file of files) {
3169
+ const conversion = litellmRecords(await readFile(file, 'utf8'));
3170
+ for (const record of conversion.records) lines.push(JSON.stringify(record));
3171
+ rows += conversion.rows;
3172
+ unnamedModel += conversion.unnamedModel;
3173
+ noTokens += conversion.noTokens;
3174
+ cacheFlagged += conversion.cacheFlagged;
3175
+ unparseable += conversion.unparseable;
3176
+ if (conversion.reportedSpendUsd !== null) {
3177
+ reportedSpend += conversion.reportedSpendUsd;
3178
+ sawSpend = true;
3179
+ }
3180
+ }
3181
+
3182
+ const out = stringFlag(args, 'out') ?? stringFlag(args, 'o');
3183
+ if (out !== undefined) {
3184
+ await writeFile(out, lines.join('\n') + (lines.length > 0 ? '\n' : ''), 'utf8');
3185
+ console.error(t.fromLiteLlm.written(out));
3186
+ } else {
3187
+ for (const line of lines) console.log(line);
3188
+ }
3189
+
3190
+ console.error(t.fromLiteLlm.summary(files.length, rows));
3191
+ if (unnamedModel > 0) console.error(t.fromLiteLlm.unnamedModel(unnamedModel));
3192
+ if (noTokens > 0) console.error(t.fromLiteLlm.noTokens(noTokens));
3193
+ if (cacheFlagged > 0) console.error(t.fromLiteLlm.cacheFlagged(cacheFlagged));
3194
+ if (sawSpend) console.error(t.fromLiteLlm.reportedSpend(formatUsd(reportedSpend)));
3195
+ if (unparseable > 0) console.error(t.fromLiteLlm.unparseable(unparseable));
3196
+ }
3197
+
3198
+ /**
3199
+ * `trazum from-helicone <file|dir>` — a Helicone request export as a usage log.
3200
+ *
3201
+ * The third converter, and the one that needs three columns where the others
3202
+ * need one: Helicone carries `request_model`, `model_override` and
3203
+ * `response_model`, and they can disagree. The response wins, because a bill
3204
+ * is about what was billed rather than what was intended, and the
3205
+ * disagreements are counted so a substitution is something the reader sees.
3206
+ *
3207
+ * Two absences are stated rather than filled in. There is no cache token
3208
+ * split on the row, only a flag; and a Helicone request id names one call and
3209
+ * never a conversation, so the records carry no session and the
3210
+ * conversation-shaped findings stay unavailable. Both are printed, because a
3211
+ * gap that is not said reads as a gap that is not there.
3212
+ */
3213
+ async function commandFromHelicone(args: Args, t: CliMessages): Promise<void> {
3214
+ const target = args.positional[0];
3215
+ if (target === undefined) throw new Error(t.fromHelicone.noPath());
3216
+
3217
+ let info;
3218
+ try {
3219
+ info = await stat(target);
3220
+ } catch {
3221
+ throw new Error(t.fromHelicone.notFound(target));
3222
+ }
3223
+ const files: string[] = [];
3224
+ if (info.isDirectory()) {
3225
+ const entries = await readdir(target, { recursive: true, withFileTypes: true });
3226
+ for (const entry of entries) {
3227
+ if (entry.isFile() && /\.(json|jsonl|ndjson)$/i.test(entry.name)) {
3228
+ files.push(join(entry.parentPath, entry.name));
3229
+ }
3230
+ }
3231
+ files.sort();
3232
+ if (files.length === 0) throw new Error(t.fromHelicone.noExports(target));
3233
+ } else {
3234
+ files.push(target);
3235
+ }
3236
+
3237
+ const lines: string[] = [];
3238
+ let rows = 0;
3239
+ let unnamedModel = 0;
3240
+ let disagreements = 0;
3241
+ let noTokens = 0;
3242
+ let cacheFlagged = 0;
3243
+ let unparseable = 0;
3244
+ for (const file of files) {
3245
+ const conversion = heliconeRecords(await readFile(file, 'utf8'));
3246
+ for (const record of conversion.records) lines.push(JSON.stringify(record));
3247
+ rows += conversion.rows;
3248
+ unnamedModel += conversion.unnamedModel;
3249
+ disagreements += conversion.modelDisagreements;
3250
+ noTokens += conversion.noTokens;
3251
+ cacheFlagged += conversion.cacheFlagged;
3252
+ unparseable += conversion.unparseable;
3253
+ }
3254
+
3255
+ const out = stringFlag(args, 'out') ?? stringFlag(args, 'o');
3256
+ if (out !== undefined) {
3257
+ await writeFile(out, lines.join('\n') + (lines.length > 0 ? '\n' : ''), 'utf8');
3258
+ console.error(t.fromHelicone.written(out));
3259
+ } else {
3260
+ for (const line of lines) console.log(line);
3261
+ }
3262
+
3263
+ console.error(t.fromHelicone.summary(files.length, rows));
3264
+ if (unnamedModel > 0) console.error(t.fromHelicone.unnamedModel(unnamedModel));
3265
+ if (disagreements > 0) console.error(t.fromHelicone.disagreements(disagreements));
3266
+ if (noTokens > 0) console.error(t.fromHelicone.noTokens(noTokens));
3267
+ if (cacheFlagged > 0) console.error(t.fromHelicone.cacheFlagged(cacheFlagged));
3268
+ // Said on every run that produced records, not only when something is
3269
+ // missing: the absence is a property of the format, and a reader who has
3270
+ // just converted a month needs to know which questions this log cannot
3271
+ // answer before they go looking for the answers.
3272
+ if (rows > 0) console.error(t.fromHelicone.noSessions());
3273
+ if (unparseable > 0) console.error(t.fromHelicone.unparseable(unparseable));
3274
+ }
3275
+
3085
3276
  /**
3086
3277
  * `trazum position <usage.jsonl>` — where the month stands, measured.
3087
3278
  *
@@ -5682,6 +5873,7 @@ async function commandCheck(
5682
5873
  verdicts: [verdict],
5683
5874
  level,
5684
5875
  tokenSource: counter.source,
5876
+ band: bandFor(prompt),
5685
5877
  truncated: false,
5686
5878
  t,
5687
5879
  }),
@@ -5817,12 +6009,18 @@ async function checkEmbedded(
5817
6009
  const failures = verdicts.filter(isOverBudget);
5818
6010
  const ok = failures.length === 0 && extraction.declined.length === 0;
5819
6011
 
6012
+ // Widest across the prompts this report covers, never an average: a run over
6013
+ // prose and a CSV covers text the estimator is 6% and 33% out on, and a
6014
+ // figure between them would describe neither.
6015
+ const band = widestBand(extraction.prompts.map((prompt) => prompt.text));
6016
+
5820
6017
  await writeMarkdown(args, () =>
5821
6018
  renderCheckMarkdown({
5822
6019
  target: path,
5823
6020
  verdicts,
5824
6021
  level,
5825
6022
  tokenSource: counter.source,
6023
+ band,
5826
6024
  truncated: false,
5827
6025
  t,
5828
6026
  }),
@@ -9643,12 +9841,102 @@ function reportProfileGaps(
9643
9841
  * exactly as `prune` does. A command that can spend somebody's money without
9644
9842
  * telling them first is a command they stop trusting.
9645
9843
  */
9844
+ /**
9845
+ * The routing measurement, without the text that produced it.
9846
+ *
9847
+ * `route --json` printed `{ slice, evaluation }` — the whole `EvalReport`,
9848
+ * whose `cases[]` carries `input` (the caller's own case text), `baseline[0]`,
9849
+ * `baseline[1]` and `optimized` (three model answers). It carried no
9850
+ * `schemaVersion` either.
9851
+ *
9852
+ * `docs/json-output.md` opens by promising that **nothing here carries a
9853
+ * session key or prompt text**, and this is the document most likely to be
9854
+ * piped into a dashboard, pasted into a ticket or dropped into a browser. A
9855
+ * tool whose headline is that prompts never leave the machine cannot have its
9856
+ * most portable artefact be the one that carries them.
9857
+ *
9858
+ * The per-case *shape* survives: the two similarity scores are the evidence
9859
+ * behind the verdict, and a reader checking a borderline call needs them. Only
9860
+ * the strings go. Anyone who wants the answers has the human-readable output,
9861
+ * which never left the terminal.
9862
+ *
9863
+ * A named function rather than an inline object so the stripping is a thing a
9864
+ * test can call without spending a provider call to reach it.
9865
+ */
9866
+ export function routeDocument(slice: unknown, result: EvalReport): unknown {
9867
+ return {
9868
+ schemaVersion: 1,
9869
+ slice,
9870
+ evaluation: {
9871
+ provider: result.provider,
9872
+ model: result.model,
9873
+ candidateModel: result.candidateModel,
9874
+ verdict: result.verdict,
9875
+ selfAgreement: result.selfAgreement,
9876
+ crossAgreement: result.crossAgreement,
9877
+ callsMade: result.callsMade,
9878
+ cases: result.cases.map((one: EvalReport['cases'][number]) => ({
9879
+ selfSimilarity: one.selfSimilarity,
9880
+ crossSimilarity: one.crossSimilarity,
9881
+ })),
9882
+ },
9883
+ };
9884
+ }
9885
+
9886
+ /**
9887
+ * The example-pruning measurement, likewise.
9888
+ *
9889
+ * `ExampleContribution.text` is *"the example itself, so a report can quote its
9890
+ * first line"* — true of the terminal rendering and wrong of a document that
9891
+ * promises to carry no prompt text. A few-shot example **is** prompt text, and
9892
+ * this was the second command emitting it.
9893
+ *
9894
+ * `index` and `tokens` say which example without quoting it, which is what a
9895
+ * machine reading this needs: it already has the prompt.
9896
+ */
9897
+ export function pruneDocument(report: PruneReport): unknown {
9898
+ return {
9899
+ schemaVersion: 1,
9900
+ provider: report.provider,
9901
+ model: report.model,
9902
+ selfAgreement: report.selfAgreement,
9903
+ recoverableTokens: report.recoverableTokens,
9904
+ callsMade: report.callsMade,
9905
+ contributions: report.contributions.map((one: PruneReport['contributions'][number]) => ({
9906
+ index: one.index,
9907
+ tokens: one.tokens,
9908
+ agreementWithout: one.agreementWithout,
9909
+ verdict: one.verdict,
9910
+ })),
9911
+ };
9912
+ }
9913
+
9646
9914
  async function commandRoute(args: Args, pricing: PricingCatalogue, t: CliMessages): Promise<void> {
9915
+ /**
9916
+ * Under `--json`, stdout carries the document and nothing else.
9917
+ *
9918
+ * It carried the human preamble too — the slice it picked, the call count,
9919
+ * the "nothing has been spent yet" line — straight into the same stream as
9920
+ * the JSON, so `route --json > verdict.json` wrote a file no parser would
9921
+ * read. Found by running the command and feeding the file to the bridge
9922
+ * that reads it, which is the one thing the pure-function guards could not
9923
+ * do: `interchange.test.js` names `route` among the commands it cannot
9924
+ * drive, so nothing checked that its `--json` output was a document.
9925
+ *
9926
+ * The lines go to stderr rather than away. A reader running this in a
9927
+ * terminal still needs to see which slice was picked and what it will cost
9928
+ * before it costs it; a pipe needs the document. stderr gives both.
9929
+ */
9930
+ const asJson = boolFlag(args, 'json');
9931
+ const say = (line = ''): void => {
9932
+ if (asJson) console.error(line);
9933
+ else console.log(line);
9934
+ };
9647
9935
  const path = args.positional[0];
9648
9936
  if (path === undefined) {
9649
- console.log();
9650
- console.log(c.dim(wrap(t.route.noTarget(), 74, ' ')));
9651
- console.log();
9937
+ say();
9938
+ say(c.dim(wrap(t.route.noTarget(), 74, ' ')));
9939
+ say();
9652
9940
  return;
9653
9941
  }
9654
9942
 
@@ -9671,18 +9959,18 @@ async function commandRoute(args: Args, pricing: PricingCatalogue, t: CliMessage
9671
9959
  const available = report.byLabel
9672
9960
  .map((r) => (r.label === UNLABELLED ? t.profile.unlabelled() : r.label))
9673
9961
  .join(', ');
9674
- console.log();
9675
- console.log(c.dim(wrap(t.route.labelNotFound(wanted, available), 74, ' ')));
9676
- console.log();
9962
+ say();
9963
+ say(c.dim(wrap(t.route.labelNotFound(wanted, available), 74, ' ')));
9964
+ say();
9677
9965
  return;
9678
9966
  }
9679
9967
  const slice = levers.slices.find(
9680
9968
  (s) => s.route !== null && (wanted === undefined || s.label === wanted),
9681
9969
  );
9682
9970
  if (!slice?.route) {
9683
- console.log();
9684
- console.log(c.dim(wrap(t.route.noRoute(), 74, ' ')));
9685
- console.log();
9971
+ say();
9972
+ say(c.dim(wrap(t.route.noRoute(), 74, ' ')));
9973
+ say();
9686
9974
  return;
9687
9975
  }
9688
9976
 
@@ -9706,8 +9994,8 @@ async function commandRoute(args: Args, pricing: PricingCatalogue, t: CliMessage
9706
9994
 
9707
9995
  const label = slice.label === UNLABELLED ? t.profile.unlabelled() : slice.label;
9708
9996
  const worth = formatUsd(slice.route.savingUsd);
9709
- console.log();
9710
- console.log(
9997
+ say();
9998
+ say(
9711
9999
  ` ${c.bold(t.route.picked(label, slice.modelName, slice.route.candidate.displayName, worth, `${(slice.shareOfBill * 100).toFixed(1)}%`))}`,
9712
10000
  );
9713
10001
  /**
@@ -9720,20 +10008,20 @@ async function commandRoute(args: Args, pricing: PricingCatalogue, t: CliMessage
9720
10008
  * counts, so it is stated rather than guessed at.
9721
10009
  */
9722
10010
  if (slice.label === UNLABELLED) {
9723
- console.log(` ${c.yellow('!')} ${c.dim(wrap(t.route.unlabelledSlice(), 74, ' '))}`);
10011
+ say(` ${c.yellow('!')} ${c.dim(wrap(t.route.unlabelledSlice(), 74, ' '))}`);
9724
10012
  }
9725
- console.log();
9726
- console.log(
10013
+ say();
10014
+ say(
9727
10015
  ` ${c.dim(wrap(t.route.willSpend(inputs.length * 3, provider.model, candidate.model), 74, ' '))}`,
9728
10016
  );
9729
10017
 
9730
10018
  if (!boolFlag(args, 'yes')) {
9731
- console.log(` ${c.dim(t.route.dryRun())}`);
9732
- console.log();
10019
+ say(` ${c.dim(t.route.dryRun())}`);
10020
+ say();
9733
10021
  return;
9734
10022
  }
9735
10023
 
9736
- console.log(` ${c.dim(t.route.running(inputs.length))}`);
10024
+ say(` ${c.dim(t.route.running(inputs.length))}`);
9737
10025
  // Same prompt on both sides. The axis under test is the model, and passing the
9738
10026
  // prompt twice is what says so at the call site.
9739
10027
  const result = await evaluate(prompt, prompt, inputs, provider, {
@@ -9742,22 +10030,22 @@ async function commandRoute(args: Args, pricing: PricingCatalogue, t: CliMessage
9742
10030
  });
9743
10031
 
9744
10032
  if (boolFlag(args, 'json')) {
9745
- console.log(JSON.stringify({ slice, evaluation: result }, null, 2));
10033
+ console.log(JSON.stringify(routeDocument(slice, result), null, 2));
9746
10034
  return;
9747
10035
  }
9748
10036
 
9749
10037
  const asPct = (v: number): string => `${(v * 100).toFixed(0)}%`;
9750
- console.log();
9751
- console.log(
10038
+ say();
10039
+ say(
9752
10040
  ` ${c.dim(wrap(t.route.agreement(asPct(result.crossAgreement), asPct(result.selfAgreement)), 74, ' '))}`,
9753
10041
  );
9754
- console.log();
10042
+ say();
9755
10043
  if (result.verdict === 'inconclusive') {
9756
- console.log(` ${c.bold(wrap(t.route.inconclusive(), 74, ' '))}`);
10044
+ say(` ${c.bold(wrap(t.route.inconclusive(), 74, ' '))}`);
9757
10045
  } else if (result.verdict === 'diverges') {
9758
- console.log(` ${c.yellow('!')} ${c.bold(wrap(t.route.diverges(worth), 74, ' '))}`);
10046
+ say(` ${c.yellow('!')} ${c.bold(wrap(t.route.diverges(worth), 74, ' '))}`);
9759
10047
  } else {
9760
- console.log(` ${c.green('✓')} ${c.bold(wrap(t.route.holds(worth), 74, ' '))}`);
10048
+ say(` ${c.green('✓')} ${c.bold(wrap(t.route.holds(worth), 74, ' '))}`);
9761
10049
  }
9762
10050
  /**
9763
10051
  * Printed on every verdict including the good one. Agreement is not
@@ -9765,8 +10053,8 @@ async function commandRoute(args: Args, pricing: PricingCatalogue, t: CliMessage
9765
10053
  * ever right, and a green tick that let somebody forget that would be the tool
9766
10054
  * overstating what it knows.
9767
10055
  */
9768
- console.log(` ${c.dim(wrap(t.route.yours(), 74, ' '))}`);
9769
- console.log();
10056
+ say(` ${c.dim(wrap(t.route.yours(), 74, ' '))}`);
10057
+ say();
9770
10058
  }
9771
10059
 
9772
10060
  /**
@@ -9933,6 +10221,14 @@ interface PromptScan {
9933
10221
  truncated: boolean;
9934
10222
  /** The extensions actually walked, so an error can name them. */
9935
10223
  extensions: string[];
10224
+ /**
10225
+ * The widest measured band across the prompts this scan judged.
10226
+ *
10227
+ * Computed here because this is where the texts are, and widest rather than
10228
+ * averaged: a scan over a prose prompt and a CSV covers text the estimator
10229
+ * is 6% and 33% out on, and a figure between them describes neither.
10230
+ */
10231
+ band: number;
9936
10232
  }
9937
10233
 
9938
10234
  /**
@@ -10012,6 +10308,9 @@ async function scanPrompts(
10012
10308
  const verdicts: FileVerdict[] = [];
10013
10309
  const declined: Array<{ path: string; line: number; detail: string }> = [];
10014
10310
 
10311
+ // Every prompt this run judged, so the report's band is the widest of them
10312
+ // rather than one figure asserted over text of several kinds.
10313
+ const judgedTexts: string[] = [];
10015
10314
  for (const relativePath of files) {
10016
10315
  const text = capInput(
10017
10316
  await readFile(join(root, relativePath), 'utf8'),
@@ -10019,6 +10318,7 @@ async function scanPrompts(
10019
10318
  maxInputFlag(args, t),
10020
10319
  t,
10021
10320
  );
10321
+ judgedTexts.push(text);
10022
10322
  // Budgets are keyed on paths as written in the repository, so a pattern like
10023
10323
  // `prompts/**` has to be matched against the path including the root the
10024
10324
  // user passed — not against the name relative to it.
@@ -10052,7 +10352,7 @@ async function scanPrompts(
10052
10352
  throw new Error(t.errors.noPromptsFound(root, extensions.join(' ')));
10053
10353
  }
10054
10354
 
10055
- return { verdicts, declined, truncated, extensions };
10355
+ return { verdicts, declined, truncated, extensions, band: widestBand(judgedTexts) };
10056
10356
  }
10057
10357
 
10058
10358
  async function checkDirectory(
@@ -10084,7 +10384,7 @@ async function checkDirectory(
10084
10384
  only = checkable;
10085
10385
  }
10086
10386
 
10087
- const { verdicts, declined, truncated, extensions } = await scanPrompts(
10387
+ const { verdicts, declined, truncated, extensions, band } = await scanPrompts(
10088
10388
  root,
10089
10389
  args,
10090
10390
  flagBudget,
@@ -10162,6 +10462,7 @@ async function checkDirectory(
10162
10462
  renderCheckMarkdown({
10163
10463
  target: root,
10164
10464
  verdicts,
10465
+ band,
10165
10466
  level,
10166
10467
  tokenSource: counter.source,
10167
10468
  truncated,
@@ -10342,6 +10643,13 @@ function joinPosix(root: string, relativePath: string): string {
10342
10643
  * never "delete this", and nothing here edits the prompt.
10343
10644
  */
10344
10645
  async function commandPrune(args: Args, t: CliMessages): Promise<void> {
10646
+ // Same as `route`: under `--json` stdout is the document, and the estimate
10647
+ // a reader has to see before spending goes to stderr rather than away.
10648
+ const asJson = boolFlag(args, 'json');
10649
+ const say = (line = ''): void => {
10650
+ if (asJson) console.error(line);
10651
+ else console.log(line);
10652
+ };
10345
10653
  const prompt = await readInput(args.positional[0], t, maxInputFlag(args, t));
10346
10654
 
10347
10655
  const casesPath = stringFlag(args, 'cases');
@@ -10356,11 +10664,11 @@ async function commandPrune(args: Args, t: CliMessages): Promise<void> {
10356
10664
 
10357
10665
  // Printed before the key is even looked up, so somebody weighing it up does not
10358
10666
  // need a configured provider to see the number.
10359
- console.log();
10360
- console.log(c.bold(t.prune.estimate(examples.length, inputs.length, calls)));
10667
+ say();
10668
+ say(c.bold(t.prune.estimate(examples.length, inputs.length, calls)));
10361
10669
 
10362
10670
  if (!boolFlag(args, 'yes')) {
10363
- console.log(c.yellow(` ${t.prune.needsConsent()}`));
10671
+ say(c.yellow(` ${t.prune.needsConsent()}`));
10364
10672
  return;
10365
10673
  }
10366
10674
 
@@ -10372,15 +10680,15 @@ async function commandPrune(args: Args, t: CliMessages): Promise<void> {
10372
10680
  });
10373
10681
 
10374
10682
  if (boolFlag(args, 'json')) {
10375
- console.log(JSON.stringify(report, null, 2));
10683
+ console.log(JSON.stringify(pruneDocument(report), null, 2));
10376
10684
  return;
10377
10685
  }
10378
10686
 
10379
10687
  const pct = (value: number): string => `${(value * 100).toFixed(0)}%`;
10380
- console.log();
10381
- console.log(sectionHeading(t.prune.heading(provider.model)));
10382
- console.log(` ${c.dim(t.prune.selfAgreement(pct(report.selfAgreement)))}`);
10383
- console.log();
10688
+ say();
10689
+ say(sectionHeading(t.prune.heading(provider.model)));
10690
+ say(` ${c.dim(t.prune.selfAgreement(pct(report.selfAgreement)))}`);
10691
+ say();
10384
10692
 
10385
10693
  for (const contribution of report.contributions) {
10386
10694
  /**
@@ -10399,7 +10707,7 @@ async function commandPrune(args: Args, t: CliMessages): Promise<void> {
10399
10707
  ? t.prune.verdictNeeded()
10400
10708
  : t.prune.verdictRecoverable();
10401
10709
 
10402
- console.log(
10710
+ say(
10403
10711
  ` ${mark} ${t.prune.line(contribution.index + 1, contribution.tokens, pct(contribution.agreementWithout))}`
10404
10712
  + ` ${unknown ? c.yellow(label) : needed ? c.dim(label) : c.green(label)}`,
10405
10713
  );
@@ -10412,16 +10720,16 @@ async function commandPrune(args: Args, t: CliMessages): Promise<void> {
10412
10720
  */
10413
10721
  const lines = contribution.text.split('\n').filter((line) => line.trim() !== '');
10414
10722
  const body = lines.find((line) => !/^\s*(?:#+\s*)?(?:example|ejemplo)\b[\s:.-]*$/i.test(line));
10415
- console.log(` ${c.dim(truncate((body ?? lines[0] ?? '').trim(), 60))}`);
10723
+ say(` ${c.dim(truncate((body ?? lines[0] ?? '').trim(), 60))}`);
10416
10724
  }
10417
10725
 
10418
- console.log();
10726
+ say();
10419
10727
  if (report.recoverableTokens > 0) {
10420
- console.log(` ${t.prune.recoverable(report.recoverableTokens)}`);
10728
+ say(` ${t.prune.recoverable(report.recoverableTokens)}`);
10421
10729
  }
10422
- console.log(` ${c.dim(wrap(t.prune.caveat(), 74, ' '))}`);
10423
- console.log();
10424
- console.log(c.dim(` ${t.eval.callsMade(report.callsMade)}`));
10730
+ say(` ${c.dim(wrap(t.prune.caveat(), 74, ' '))}`);
10731
+ say();
10732
+ say(c.dim(` ${t.eval.callsMade(report.callsMade)}`));
10425
10733
  }
10426
10734
 
10427
10735
  async function commandEval(
@@ -10740,6 +11048,18 @@ const DOCTOR_LIST_LIMIT = 8;
10740
11048
  const BLAME_DEFAULT_LIMIT = 20;
10741
11049
  const BLAME_MAX_LIMIT = 500;
10742
11050
 
11051
+ /**
11052
+ * The band for a report covering several prompts.
11053
+ *
11054
+ * The widest of them, and the empty case is the widest published band rather
11055
+ * than a narrow default: a report over nothing has measured nothing, and
11056
+ * guessing narrow there is the one direction that produces a false claim.
11057
+ */
11058
+ function widestBand(texts: readonly string[]): number {
11059
+ if (texts.length === 0) return ESTIMATE_ERROR_BAND_PCT;
11060
+ return Math.max(...texts.map((text) => bandFor(text)));
11061
+ }
11062
+
10743
11063
  interface BlameRow {
10744
11064
  revision: Revision;
10745
11065
  /** `null` when the file did not exist at that commit, or held no marked prompt. */
@@ -10838,17 +11158,35 @@ async function commandBlame(
10838
11158
  const shown = rows.slice(revisions.length > limit ? 1 : 0).reverse();
10839
11159
  const truncatedHistory = revisions.length > limit;
10840
11160
 
11161
+ /*
11162
+ The band of this file's own text, read once and handed to both renderers.
11163
+
11164
+ `blame` walks one file across revisions, so the kind of text is a property
11165
+ of the file, and today's content is the closest thing to it that exists
11166
+ here. A file that is gone from the working tree — renamed away, or deleted
11167
+ in the commit being blamed — falls back to the widest published band, which
11168
+ is exactly what that constant is for: a caller that cannot see the text can
11169
+ only guess in the safe direction.
11170
+ */
11171
+ let band = ESTIMATE_ERROR_BAND_PCT;
11172
+ try {
11173
+ band = bandFor(await readFile(resolvePath(process.cwd(), target), 'utf8'));
11174
+ } catch {
11175
+ // Not on disk now. The wide band stands and says so by being wide.
11176
+ }
11177
+
10841
11178
  await writeMarkdown(args, () =>
10842
11179
  renderBlameMarkdown({
10843
11180
  repoPath,
10844
11181
  rows: shown,
10845
11182
  truncated: truncatedHistory,
10846
11183
  netCost: netCostOf(shown, args, config, pricing, t),
11184
+ band,
10847
11185
  t,
10848
11186
  }),
10849
11187
  );
10850
11188
 
10851
- printBlame(shown, { repoPath, args, config, pricing, t, truncated: truncatedHistory });
11189
+ printBlame(shown, { repoPath, args, config, pricing, t, truncated: truncatedHistory, band });
10852
11190
  }
10853
11191
 
10854
11192
  /**
@@ -10905,6 +11243,15 @@ function printBlame(
10905
11243
  pricing: PricingCatalogue;
10906
11244
  t: CliMessages;
10907
11245
  truncated: boolean;
11246
+ /**
11247
+ * The band of this file's own text, computed once by the caller.
11248
+ *
11249
+ * `blame` walks one file across revisions, so the kind of text is a
11250
+ * property of the file rather than of any one revision. The caller has
11251
+ * today's content and this renderer does not, which is why it arrives
11252
+ * here rather than being read again.
11253
+ */
11254
+ band: number;
10908
11255
  },
10909
11256
  ): void {
10910
11257
  const { repoPath, args, config, pricing, t, truncated } = context;
@@ -11039,7 +11386,7 @@ function printBlame(
11039
11386
  );
11040
11387
  }
11041
11388
 
11042
- console.log(`\n${c.dim(t.blame.estimateNote())}\n`);
11389
+ console.log(`\n${c.dim(t.blame.estimateNote(context.band))}\n`);
11043
11390
  }
11044
11391
 
11045
11392
  // --------------------------------------------------------------------------
@@ -11469,6 +11816,12 @@ async function main(): Promise<void> {
11469
11816
  case 'from-otel':
11470
11817
  await commandFromOtel(args, t);
11471
11818
  break;
11819
+ case 'from-litellm':
11820
+ await commandFromLiteLlm(args, t);
11821
+ break;
11822
+ case 'from-helicone':
11823
+ await commandFromHelicone(args, t);
11824
+ break;
11472
11825
  case 'switch':
11473
11826
  await commandSwitch(args, pricing, t);
11474
11827
  break;